diff --git a/.env.sample b/.env.sample index 7ab64b6..61488f6 100644 --- a/.env.sample +++ b/.env.sample @@ -7,17 +7,9 @@ # Required # ----------------------------------------------------------------------------- -# PostgreSQL connection string for the accounts database -# (stores identities, orgs, engines, sessions, API keys) -ACCOUNTS_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me - -# 32-byte hex key for encrypting API keys at rest -# Generate with: openssl rand -hex 32 -ACCOUNTS_MASTER_KEY= - -# PostgreSQL connection string for the engine database -# (stores memories — each engine gets its own schema) -ENGINE_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me +# PostgreSQL connection string for the application database. One database holds +# the auth + core control plane and every per-space me_ schema. +DATABASE_URL=postgres://postgres:postgres@localhost:5432/me # Public base URL for OAuth callbacks API_BASE_URL=http://localhost:3000 @@ -51,8 +43,9 @@ GOOGLE_CLIENT_SECRET= # Raise for memory.batchCreate-heavy workloads (e.g. transcript imports). # MAX_REQUEST_BODY_BYTES= -# Schema name in the accounts database -# ACCOUNTS_SCHEMA=accounts +# Schema names +# AUTH_SCHEMA=auth +# CORE_SCHEMA=core # Cron schedule for cleaning up expired device authorizations (UTC) # DEVICE_FLOW_CLEANUP_CRON=*/15 * * * * @@ -73,60 +66,20 @@ GOOGLE_CLIENT_SECRET= # LOGFIRE_SCRUBBING=false # ----------------------------------------------------------------------------- -# Optional — Accounts Database Connection Pool -# ----------------------------------------------------------------------------- - -# Maximum connections in pool -# ACCOUNTS_POOL_MAX=10 - -# Close idle pooled connections after N seconds -# ACCOUNTS_POOL_IDLE_REAP_SECONDS=300 - -# Max connection lifetime in seconds (0 = forever) -# ACCOUNTS_POOL_MAX_LIFETIME=0 - -# Timeout for establishing new connections (seconds) -# ACCOUNTS_POOL_CONNECTION_TIMEOUT=30 - -# PostgreSQL statement timeout for accounts DB transactions -# ACCOUNTS_STATEMENT_TIMEOUT=25s - -# PostgreSQL lock wait timeout for accounts DB transactions -# ACCOUNTS_LOCK_TIMEOUT=5s - -# PostgreSQL transaction timeout for accounts DB transactions -# ACCOUNTS_TRANSACTION_TIMEOUT=30s - -# PostgreSQL idle-in-transaction timeout for accounts DB transactions -# ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s - -# ----------------------------------------------------------------------------- -# Optional — Engine Database Connection Pool +# Optional — Database Connection Pool # ----------------------------------------------------------------------------- # Maximum connections in pool -# ENGINE_POOL_MAX=20 +# DB_POOL_MAX=20 # Close idle pooled connections after N seconds -# ENGINE_POOL_IDLE_REAP_SECONDS=300 +# DB_POOL_IDLE_REAP_SECONDS=300 # Max connection lifetime in seconds (0 = forever) -# ENGINE_POOL_MAX_LIFETIME=0 +# DB_POOL_MAX_LIFETIME=0 # Timeout for establishing new connections (seconds) -# ENGINE_POOL_CONNECTION_TIMEOUT=30 - -# PostgreSQL statement timeout for engine DB transactions -# ENGINE_STATEMENT_TIMEOUT=25s - -# PostgreSQL lock wait timeout for engine DB transactions -# ENGINE_LOCK_TIMEOUT=5s - -# PostgreSQL transaction timeout for engine DB transactions -# ENGINE_TRANSACTION_TIMEOUT=30s - -# PostgreSQL idle-in-transaction timeout for engine DB transactions -# ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s +# DB_POOL_CONNECTION_TIMEOUT=30 # ----------------------------------------------------------------------------- # Optional — Embedding @@ -163,45 +116,23 @@ GOOGLE_CLIENT_SECRET= # Max error backoff (ms) # WORKER_MAX_BACKOFF_MS=60000 -# Engine re-discovery interval (ms) +# Space re-discovery interval (ms) # WORKER_REFRESH_INTERVAL_MS=60000 # ----------------------------------------------------------------------------- -# Optional — Embedding Worker Engine Database +# Optional — Embedding Worker Database Pool # ----------------------------------------------------------------------------- +# The embedding worker uses a dedicated pool. Each setting defaults to the +# corresponding application-pool value (or DATABASE_URL) when unset. -# PostgreSQL connection string for embedding worker engine traffic. -# Defaults to ENGINE_DATABASE_URL when unset. -# WORKER_ENGINE_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me - -# Maximum connections in the dedicated embedding worker engine pool. -# Defaults to max(WORKER_COUNT, 1) when unset. -# WORKER_ENGINE_POOL_MAX=2 - -# Close idle embedding worker engine connections after N seconds. -# Defaults to ENGINE_POOL_IDLE_REAP_SECONDS when unset. -# WORKER_ENGINE_POOL_IDLE_REAP_SECONDS=300 - -# Max embedding worker engine connection lifetime in seconds (0 = forever). -# Defaults to ENGINE_POOL_MAX_LIFETIME when unset. -# WORKER_ENGINE_POOL_MAX_LIFETIME=0 - -# Timeout for establishing embedding worker engine connections (seconds). -# Defaults to ENGINE_POOL_CONNECTION_TIMEOUT when unset. -# WORKER_ENGINE_POOL_CONNECTION_TIMEOUT=30 - -# PostgreSQL statement timeout for embedding worker engine DB transactions -# Defaults to ENGINE_STATEMENT_TIMEOUT when unset. -# WORKER_ENGINE_STATEMENT_TIMEOUT=25s - -# PostgreSQL lock wait timeout for embedding worker engine DB transactions -# Defaults to ENGINE_LOCK_TIMEOUT when unset. -# WORKER_ENGINE_LOCK_TIMEOUT=5s - -# PostgreSQL transaction timeout for embedding worker engine DB transactions -# Defaults to ENGINE_TRANSACTION_TIMEOUT when unset. -# WORKER_ENGINE_TRANSACTION_TIMEOUT=30s +# WORKER_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me +# WORKER_DB_POOL_MAX=2 +# WORKER_DB_POOL_IDLE_REAP_SECONDS=300 +# WORKER_DB_POOL_MAX_LIFETIME=0 +# WORKER_DB_POOL_CONNECTION_TIMEOUT=30 -# PostgreSQL idle-in-transaction timeout for embedding worker engine DB transactions -# Defaults to ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT when unset. -# WORKER_ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s +# Per-transaction timeouts for the worker pool (default to built-in values). +# WORKER_DB_STATEMENT_TIMEOUT=25s +# WORKER_DB_LOCK_TIMEOUT=5s +# WORKER_DB_TRANSACTION_TIMEOUT=30s +# WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=30s diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eda52a..7eb79c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,13 @@ on: jobs: ci: runs-on: ubuntu-latest + env: + # TEST_CI disables conditional test skips: every gated suite runs, and + # missing prerequisites fail loudly instead of skipping silently. The + # OpenAI key drives the live embedding suite; a fork PR (no secrets) + # therefore fails rather than silently testing less. + TEST_CI: "1" + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} steps: - uses: actions/checkout@v6 @@ -25,10 +32,19 @@ jobs: - name: Typecheck run: ./bun run typecheck - name: Test (unit) - run: find packages -name '*.test.ts' ! -name '*.integration.test.ts' -print0 | xargs -0 ./bun test + # Same script (and process model — --parallel) as local runs, so CI + # can't diverge from what developers verify against. + run: ./bun run test:unit integration: runs-on: ubuntu-latest + env: + # See the ci job for TEST_CI semantics. The e2e suite needs the database + # URL (the container below) and a real OpenAI key for its embedding + # worker (a few thousand text-embedding-3-small tokens per run). + TEST_CI: "1" + TEST_DATABASE_URL: postgresql://postgres@127.0.0.1:5432/postgres + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} steps: - uses: actions/checkout@v6 @@ -51,6 +67,14 @@ jobs: sleep 1 done - name: Test (integration) - run: find packages -name '*.integration.test.ts' -print0 | xargs -0 ./bun test + # Same script (and process model — --parallel) as local runs; the + # suites default to postgresql://postgres@127.0.0.1:5432/postgres, + # which is the container above. The schema cleaner that test:db runs + # first is a no-op against the fresh container. + run: ./bun run test:db + - name: Test (e2e) + # Real CLI subprocesses against a real in-process server and the + # container above, with real OpenAI embeddings. + run: ./bun run test:e2e - name: Stop Postgres run: docker stop me-postgres diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml index 4e07219..ee905d7 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-dev.yaml @@ -9,7 +9,8 @@ on: - packages/embedding/** - packages/worker/** - packages/protocol/** - - packages/accounts/** + - packages/auth/** + - packages/database/** - scripts/** - package.json - bun.lock diff --git a/CLAUDE.md b/CLAUDE.md index 69bd850..9c37ab0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ All project documentation lives in `docs/`: - [Getting Started](docs/getting-started.md) -- install, login, first memory - [Core Concepts](docs/concepts.md) -- memories, tree paths, metadata, search modes - [File Formats](docs/formats.md) -- JSON, YAML, Markdown, NDJSON import/export schemas -- [Access Control](docs/access-control.md) -- users, roles, grants, ownership +- [Access Control](docs/access-control.md) -- principals, groups, tree-access grants - [Memory Packs](docs/memory-packs.md) -- pre-built knowledge collections - [MCP Integration](docs/mcp-integration.md) -- connecting AI agents - [CLI Reference](docs/cli/) -- full command reference @@ -17,34 +17,57 @@ All project documentation lives in `docs/`: Read the relevant docs before starting work on a subsystem. +> **Note**: the authoritative summary of the current model (principals / spaces / +> the auth+core+space schemas) is in this file. Some `docs/` pages still describe +> the retired engine/org/role model and may lag — trust this file when they +> disagree, and fix the docs as you touch them. + ## Quick Reference -- **Tech stack**: Bun, TypeScript, PostgreSQL 18 (pgvector, pg_textsearch, ltree, JSONB) -- **Core schema**: Single table `memory` per engine schema (`me_`) -- content, meta (JSONB), tree (ltree), temporal (tstzrange), embedding (halfvec(1536)) -- **Search**: Hybrid BM25 + semantic via Reciprocal Rank Fusion -- **API**: JSON-RPC 2.0 over HTTP -- engine RPC (`/api/v1/engine/rpc`) and accounts RPC (`/api/v1/accounts/rpc`), plus REST auth endpoints (OAuth device flow) -- **Auth**: Tree-grant RBAC with PostgreSQL RLS; OAuth (GitHub, Google) for hosted accounts -- **Embedding**: Vercel AI SDK; OpenAI `text-embedding-3-small` (1536-dim) in production; Ollama supported for local dev -- **CLI**: `me` binary (login, logout, whoami, org, engine, invitation, memory, mcp, user, grant, role, owner, apikey, pack) +- **Tech stack**: Bun, TypeScript, PostgreSQL 18 (pgvector/halfvec, pg_textsearch BM25, ltree, citext, JSONB), **postgres.js** driver. One database, one pool. +- **Schemas** (three, one database): `auth` (better-auth-shaped: `users`, `sessions`, `accounts`, `device_authorization`), `core` (control plane: `principal`, `space`, `principal_space`, `group_member`, `tree_access`, `api_key`), and per-space `me_` (data plane: the single `memory` table). `auth.users.id == core.principal.id` for user principals. +- **Memory table** (per space): `content`, `meta` (JSONB), `tree` (ltree), `temporal` (tstzrange), `embedding` (halfvec(1536)). +- **Search**: hybrid BM25 + semantic via Reciprocal Rank Fusion, computed in SQL functions. +- **Access**: no RLS. `core.build_tree_access(principalId, spaceId)` produces a `_tree_access` jsonb (rows of `tree_path` + `access`) passed into the space SQL functions (`search_memory`, `get_memory`, …). Three additive levels: **1 = read, 2 = write, 3 = owner**; `owner@root` (the empty ltree path) owns the whole space, and an owner grant at any path delegates access-management within that subtree. Two axes: **structural** authority (`principal_space.admin` — roster mutations, groups, invitations) vs **data** authority (owner@path); an admin may also grant data and can self-grant `owner@root`. The auth gate is a non-empty `build_tree_access` (every member holds ≥1 grant). +- **Tree conventions**: two reserved roots — per-member `home.` (`~` is input sugar for it; a joining **user** is granted `owner@home`) and the shared `share`. A space **creator** gets `admin` + `owner@home` + `owner@share`, **not** `owner@root` — so it sees `share` and its own `~` but not other members' homes (as an admin it can self-grant `owner@root`). `memory.create`/`batchCreate` **require** an explicit `tree` (callers choose `share` vs `~` deliberately); only the file importers (`me import memories`, the `me_memory_import` MCP tool) default a tree-less record to `share` (`SHARE_NAMESPACE`, canonically defined in `@memory.build/protocol` and re-exported by `@memory.build/database`). +- **API**: JSON-RPC 2.0 over HTTP, two endpoints: + - `/api/v1/memory/rpc` — session **or** api-key bearer + required `X-Me-Space: ` header. Memory data plane (`memory.*`) + space management (`principal.*`, `group.*`, `grant.*`, `invite.*`). + - `/api/v1/user/rpc` — session only (an api key never authenticates here; agents can't manage agents). `whoami`, `agent.*`, `apiKey.*`, `space.*`. + - Plus REST OAuth device-flow endpoints under `/api/v1/auth/*`. +- **Auth**: humans use a **session token** (OAuth device flow, GitHub/Google); agents use an **api key** (`me..`). Api keys are **global** per-principal credentials, not space-bound: the same key works in any space the agent has been admitted to (the space comes from `X-Me-Space`, gated by `build_tree_access`). Session + api-key secrets are sha256 (compared by equality in SQL), not argon2. +- **Embedding**: Vercel AI SDK; OpenAI `text-embedding-3-small` (1536-dim) in production; Ollama supported for local dev. +- **CLI**: `me` binary — `login`, `logout`, `whoami`, `space`, `group`, `access`, `agent`, `apikey`, `memory` (+ top-level aliases like `me search`, `me create` — except `import`), `import` (the source group: `memories`/`claude`/`codex`/`opencode`/`git`; `me memory import` and `me import` remain as aliases), `mcp`, `claude`/`codex`/`gemini`/`opencode`, `serve`, `pack`. + +## Principals, members, spaces (terminology) + +- **Principal** = the union **user | agent | group** (`principal.kind` = `'u'` | `'a'` | `'g'`). The space roster (`principal_space`) holds principals. `principal.member_id` is a generated column equal to `id` for users/agents (NOT groups). +- **Member** = the **user/agent** sense only — group members and api-key holders. So params split as `principalId` (roster / grants, any kind) vs `memberId` (group membership, api keys; u|a only). The space-roster surface is principal-centric (`principal.*` methods, `SpacePrincipal` type), reserving "member" for u|a. +- **Space**: identified by an immutable 12-char `slug` (which is the `me_` schema name and the `X-Me-Space` value) and a renamable `name`. `me space rename` changes only the name. No org / engine / shard concepts. +- **Admin**: `principal_space.admin` is *structural* authority — roster mutations (`principal.add`/`remove`), groups, and invitations (`invite.*`) — distinct from data ownership (owner@path via `tree_access`). Enumerating the whole roster (`principal.list`) is admin-only; **any member** may `principal.resolve`/`lookup` (a targeted name↔id lookup, not enumeration). Admin transfers **transitively** through a group whose own `principal_space.admin` is true; agents are never admins. A space must always keep ≥1 *effective* admin (a **user** who is a direct admin or a member of an admin group — an empty admin group doesn't count) — the `enforce_last_admin` trigger on `principal_space` + `group_member` rejects any remove/demote/group-member-removal that would drop the last one (SQLSTATE `ME001` → `LAST_ADMIN`), but exempts whole-space deletion. +- **Transitive membership** (Model 2): a group member gains the group's space membership, its space-admin (if the group is admin), and its tree-access grants. ## Project Structure ``` packages/ - cli/ # CLI and MCP server (the `me` binary) - client/ # TypeScript client for the engine API - engine/ # Core engine (database operations, search, embedding) - protocol/ # Shared types and Zod schemas (JSON-RPC methods) - hosted/ # Hosted/multi-tenant provisioning + auth/ # auth-schema store: users, sessions, oauth accounts, device flow + cli/ # CLI + MCP server (the `me` binary) + claude-plugin/# Claude Code plugin (capture hooks, slash commands) + client/ # TS client: createMemoryClient + createUserClient (+ auth device flow) + database/ # schema migrations (auth, core, space) + shared migrate kit docs-site/ # Next.js static site that renders `docs/` for docs.memory.build + embedding/ # vector embedding providers (OpenAI, Ollama) + engine/ # runtime stores over the SQL functions: core (control plane) + space (data plane) + protocol/ # shared Zod schemas + types: memory + space + user contracts; auth/fields/headers/jsonrpc/errors/version + server/ # HTTP server, routing, RPC handlers, OAuth, first-login provisioning + web/ # React UI served by `me serve` (talks to the same-origin /rpc proxy) + worker/ # background embedding queue processor packs/ # Memory packs (pre-built knowledge collections) docs/ cli/ # CLI command reference (one file per command group) mcp/ # MCP tool reference (one file per tool) ``` -> **Note**: `packages/hosted` is the target package name; the current implementation is split across `packages/accounts` (org/member/engine management, OAuth), `packages/server` (HTTP server, routing, RPC handlers), `packages/embedding` (vector embedding providers), and `packages/worker` (background embedding queue processor). - ## Build, Lint, and Test Always use the `./bun` wrapper script (auto-installs the pinned Bun version): @@ -59,23 +82,91 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): # Linting and formatting (auto-fix) ./bun run lint --write -# Run unit tests -./bun test - -# Run a single test file +# Run a test file directly (uses TEST_DATABASE_URL; default local 127.0.0.1) — +# fast, for iterating on one file: ./bun test packages/cli/mcp/install.test.ts -# Shorthand for all checks (typecheck + lint + test) +# Full suite (unit + integration) — defaults to the LOCAL Postgres container +# at 127.0.0.1:5432 (--parallel=2, 30s timeout); TEST_DATABASE_URL overrides. +./bun run test + +# Fast inner loop (typecheck + lint + unit tests; no database, ~15s) ./bun run check + +# Everything: check + full suite + the e2e suite (~30s against local Postgres) +./bun run check:full +``` + +**Important — verification runs against the local Postgres**: after making +code changes, run `./bun run check` (fast, no DB). Before committing, run +`./bun run check:full` — it defaults to the `me-postgres` Docker container +(if it isn't running: `docker start me-postgres || ./bun run pg`). Only run +against ghost when explicitly asked to test against ghost. CI is the strict +gate: it runs every suite with `TEST_CI=1`, which disables conditional skips +— any new `describe.skipIf` gate **must** include `!process.env.TEST_CI` in +its condition (pattern: `packages/embedding/generate.test.ts`, +`e2e/cli.e2e.test.ts`) so CI never silently skips it. + +> `packages/web` and `packages/docs-site` are excluded from the root typecheck +> (they have their own); `check`/`check:full` do not cover them. + +### Database integration tests + +`*.integration.test.ts` files run against a real PostgreSQL 18 with the +required extensions (citext, ltree, pgvector, pg_textsearch). Everything +defaults to the **local `me-postgres` Docker container** at 127.0.0.1:5432 +(same image CI builds; `./bun run pg` creates it). `test:db` is the focused +variant: it first reclaims orphaned test schemas, then runs **every** +`*.integration.test.ts` under `packages/` (the auth/core/space migration +suites plus the engine/server/worker suites), `--parallel=2`, 30s timeout: + +```bash +./bun run test:db +``` + +A single integration file runs in seconds locally: + +```bash +./bun test --timeout 30000 packages/database/core/migrate/migrate.integration.test.ts +``` + +**Ghost (only when explicitly asked to test against ghost)**: `testing_me` is +the dedicated ghost database — point `TEST_DATABASE_URL` at it explicitly. +Expect minutes instead of seconds (every statement pays WAN latency), and +always pass `--timeout 30000` for single files — bun's default 5s isn't +enough over the remote connection (a migrating `beforeAll` overruns it, +surfacing as a misleading "beforeEach/afterEach hook timed out"): + +```bash +TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:db ``` -**Important**: After making code changes, always run `./bun run check`. +Isolation is **schema-level** (ghost forbids `create database`): each test +provisions its own throwaway schema(s) — `core_test_` for core, +`auth_test_` for auth, `metest_` for the space *migration* tests — so +the suites are fully concurrent and parallel-safe across files. All migrations are +templated, so production uses `core` / `auth` / `me_` while these tests +target throwaway schemas and never touch real data. The space migration tests +deliberately use the `metest_` prefix (not production `me_`) so leftovers are +distinguishable by name alone. The **server** integration tests are the exception: +they exercise the real `provisionUser` / `provisionSpace`, so they create genuine +`me_` schemas and drop them in `afterAll` (only a hard-interrupted server +test leaks one — see below). + +`test:db` first runs `test:db:clean` (`scripts/clean-test-schemas.ts`) to +reclaim orphaned `core_test_*` / `auth_test_*` / `metest_*` schemas left by +hard-interrupted runs. It is age-gated (only drops schemas older than 60 min, so +a concurrent `test:db` sharing the database is safe) and a no-op against a +production database — no pattern can match a real schema, **including `me_`**: +a server test's leaked `me_` is therefore *not* auto-reclaimed, so drop it +by hand if a run is killed mid-test. Use `test:db:clean:all` for a deliberate full +reset when nothing else is using the database. ## Style Guides **TypeScript**: Biome for linting and formatting. Config in `biome.json`. -**SQL**: Lowercase keywords, leading-comma table definitions, inline comments after columns, native `uuid` with `uuidv7()`. +**SQL**: Lowercase keywords, leading-comma table definitions, inline comments after columns, native `uuid` with `uuidv7()`. Logic lives in SQL functions; the TS stores call functions rather than querying tables directly. ```sql create table me.memory @@ -90,7 +181,30 @@ create table me.memory ## Key Design Decisions -- **Single table**: All memory types live in `me.memory`. Complexity comes from conventions in `meta` and `tree`, not schema proliferation. -- **Database-native**: Uses PostgreSQL extensions (ltree, pgvector, JSONB GIN, tstzrange, BM25) instead of application-layer abstractions. -- **Flexibility over prescription**: `meta` accepts any JSON, `tree` paths are user-defined, `temporal` is optional. No enforced conventions. -- **MCP compatibility**: All tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). +- **One DB, one pool**: `auth` + `core` + every `me_` live in one Postgres database behind one postgres.js pool (plus a dedicated worker pool). Sharding / pgdog distribution is deferred; the per-slug schema model keeps a future re-split cheap. +- **Single memory table per space**: all memory lives in `me_.memory`. Complexity comes from conventions in `meta` and `tree`, not schema proliferation. +- **Database-native**: PostgreSQL extensions (ltree, pgvector/halfvec, JSONB GIN, tstzrange, BM25) instead of application-layer abstractions. +- **Access via `tree_access`, not RLS**: RLS was unperformant. `build_tree_access` produces a `_tree_access` jsonb passed into the space functions; there is no `me.user_id` GUC. Three levels (read/write/owner); an owner grant delegates access-management within its subtree (owner@root = the whole space). +- **Two endpoints, two auth modes**: memory RPC (session or api key + `X-Me-Space`) vs user RPC (session only). `extractBearerToken` is the one shared auth helper. +- **Principal vs member** terminology (see above): principal = u|a|g; member/`memberId` = u|a. +- **CLI credentials**: split across `~/.config/me/` — **`config.yaml`** (non-secret: default server + per-server **active space** / the X-Me-Space) and **`credentials.yaml`** (0600, secret session-token *fallback* only). The **session token** lives in the OS keychain when available (macOS `security`, Linux `secret-tool` via libsecret; `ME_NO_KEYCHAIN=1` forces off), else in `credentials.yaml` (empty/absent on keychain hosts); a pre-split `credentials.yaml` is migrated on first read. `me logout` clears the session secret but keeps the non-secret config (so re-login resumes). **Api keys are never persisted** — an agent key only ever comes from `ME_API_KEY` (humans authenticate with sessions; `apiKey.create` prints the key once for the operator to place where the agent runs). Env: `ME_SERVER` / `ME_API_KEY` / `ME_SPACE` / `ME_SESSION_TOKEN` / `ME_NO_KEYCHAIN`. +- **Header constants** (`CLIENT_VERSION_HEADER`, `SPACE_HEADER`) live in `@memory.build/protocol/headers`. +- **MCP compatibility**: all tool parameters are required (nullable for optional). Uses `z.record(z.string(), z.any())` for meta instead of `z.record(z.unknown())` (which crashes the MCP SDK). +- **batchCreate conflict semantics**: a duplicate explicit id is skipped, or — with `replaceIfMetaDiffers: ""` — replaced in place when the stored row's value for that key differs (the session importers pass `importer_version` so version bumps re-render server-side). Result is `{ids, updatedIds}` (inserted / replaced); ids in neither were skipped. Single `memory.create` on a duplicate id errors with CONFLICT. + +## Database driver: postgres.js + +The runtime is fully on **postgres.js**. We moved off `Bun.SQL` because it does +not return a pooled connection after a query or `begin()` callback errors — +after `max` such errors the pool drains and the next acquire hangs forever (Bun +bug [oven-sh/bun#22395](https://github.com/oven-sh/bun/issues/22395)). The single +application pool + the worker pool + all stores and migrations use postgres.js. +The only remaining `Bun.SQL` use is `scripts/setup.ts`, a dev-only +create-database helper (short-lived, no long pool — the bug doesn't bite). + +Gotchas when writing DB code / tests: + +- Pass jsonb to SQL functions via `sql.json(v)` — a raw `JSON.stringify` double-encodes and a raw array sends as a PG array. +- `noUncheckedIndexedAccess` makes `rows[0]` possibly-`undefined` → `const [row] = ...; row?.col`; don't annotate `.map((r: {col}) => …)` (the row is a typed `Row`). +- `expect(sql\`…\`).rejects` **hangs** in bun:test — it doesn't drive postgres.js's lazy `PendingQuery`. Assert query failures with try/catch (see `expectReject` in `packages/database/migrate/test-utils.ts`). `expect(migrateX(…)).rejects` is fine (a real async-fn Promise). +- `to_bm25query(text, index_name text)` — the index name is `text`, not `regclass`. citext function params: compare with `_x::citext` or it silently degrades to case-sensitive `text = text`. diff --git a/DECISIONS_FOR_REVIEW.md b/DECISIONS_FOR_REVIEW.md new file mode 100644 index 0000000..16e8e87 --- /dev/null +++ b/DECISIONS_FOR_REVIEW.md @@ -0,0 +1,353 @@ +# Decisions for review + +Design/behavior decisions made during implementation that warrant a maintainer's +sign-off. Each entry records the decision, the alternative(s), why the call was +made, and how to change it. Once you've reviewed an entry, either fold it into +`CLAUDE.md` / `docs/` (ratified) or open a change (overridden), and delete it +here. + +--- + +## `~` (home) resolves to the authenticated principal — an agent gets its *own* home + +**Date:** 2026-06-05 · **Area:** tree-path normalization (`a94cfb0`) + +A leading `~` in a tree path expands to `home.` (UUID with hyphens +stripped) where the principal is **whoever the bearer token authenticates as**: +a human session → that user; an agent api key → that agent. So an agent's `~` is +`home.` — the agent's own isolated home — **not** its owner's home. + +**Alternative considered:** an agent's `~` maps to its owner's home +(`home.`), so an agent acting on a user's behalf writes into the +user's home tree. + +**Why this call:** `~` consistently means "me" (the authenticated principal) — +simplest mental model, no owner lookup, and agent homes stay isolated. `~` is +opt-in sugar; an agent that wants a shared/space-wide location just uses an +explicit path (e.g. `projects/x`) instead of `~`. + +**How to change it:** the home id is `ctx.principalId`, passed to the +normalizer/serializer in `packages/server/rpc/memory/support.ts` +(`inputTreePath` / `inputTreeFilter` / `displayTreePath`). To make an agent's `~` +resolve to its owner, resolve the owner id for agent principals there (the agent +principal has `ownerId`) and use it for both expansion and reverse-display. Note: +paths already stored under the current rule (`home..…`) would not +migrate automatically. + +**Status:** needs review. + +--- + +## Destructive space ops (`space.delete` / `space.rename`) gated on admin — no separate owner flag + +**Date:** 2026-06-05 · **Area:** core authority model + +`space.delete` and `space.rename` are gated on **space-admin** +(`principal_space.admin`, which is transitive through admin groups). `delete` +drops the whole `me_` schema — all of the space's memories — so **any** +space-admin, including one who inherited admin via a group, can destroy +everything. + +**Decision:** leave it as-is for now. Admins can delete; we will **not** add a +distinct space-**owner** notion to protect destructive ops until someone +actually asks for it. + +**Alternative (deferred):** a separate owner gate for the truly destructive ops +— e.g. a `principal_space.owner` flag, or treating owner@root as the gate — +keeping plain admin for routine structural management (groups, members, grants). +Would also need decisions on whether owner is transitive through groups +(probably not) and how ownership transfers. + +**Revisit when:** there's a request for delete protection / "are you sure" +beyond the CLI's type-the-name confirmation, or the first report of an admin +nuking a space. At that point implement the owner gate above. + +**Status:** decided (defer); revisit on request. + +--- + +## Home grant at join is for users only — agents get no auto home + +**Date:** 2026-06-05 · **Area:** membership (`add_principal_to_space`, INV-1) + +`add_principal_to_space` now writes a real `owner @ home.` grant when a +**user** joins a space (the single chokepoint every join path goes through: +provisioning, invite redemption, direct add). **Agents are deliberately excluded.** + +**Why exclude agents:** `agent_tree_access` clamps an agent's effective grants to +its owner's — an agent can never exceed what its owner can reach. A typical owner +(an invited user) holds `owner@home.` and maybe `share`, but **nothing** +over `home.`. So an auto `owner@home.` grant would be clamped to +nothing: an inert, misleading row in `tree_access` that `build_tree_access` never +returns. Users have no clamp, so their home grant is always effective. + +**Tension with the `~` decision above:** that entry frames an agent's `~` as +`home.` — its own isolated home. With agents excluded here, an agent's +`~` still *resolves* to `home.` but carries **no access by default**; the +agent can only use it if its owner explicitly grants it there (and, because of the +clamp, the owner must hold that access too). + +**How to change it (give agents real homes):** options — (a) nest agent homes +under the owner (`home..…`) so the owner's home grant covers them; or +(b) in `add_principal_to_space` for an agent, also grant the **owner** +`owner@home.` so the clamp passes (owner can then see into agent homes); +or (c) relax the clamp for the agent's own home subtree. Each needs a deliberate +call on owner visibility into agent data. The gate is `and p.kind = 'u'` in +`packages/database/core/migrate/idempotent/006_membership.sql`. + +**Status:** needs review. + +--- + +## Should users be able to mint their own API keys? (currently agent-only) + +**Date:** 2026-06-05 · **Area:** auth / api keys + +API keys are currently **agent-only**: `apiKey.create` is gated by +`requireOwnedAgent`, and humans authenticate via session. But the intended CLI +surface treats `ME_API_KEY` as pointing to a "user | agent" and `me apikey +create` as defaulting to self — which implies users can mint their own keys. + +**The decision:** allow user-owned api keys, or keep "humans use sessions only"? + +**Cost if yes (small):** `validate_api_key` already returns the principal +regardless of kind and `authenticateSpace` works unchanged, so it's mostly +relaxing the `apiKey.create` gate to allow `member == self` (a user) in addition +to agents the caller owns. + +**Why it's a real decision:** weigh CLI ergonomics (a user scripting against their +own space without a browser session) against the security stance that human auth +stays interactive/session-only — an api key is a long-lived bearer secret, so +making them mintable for users widens that surface. + +**Status:** needs decision. + +--- + +## Rolling sessions (7d, refreshed daily, no absolute cap) — copied from better-auth + +**Date:** 2026-06-08 · **Area:** auth / session lifetime + +Sessions were a **fixed** 30-day expiry with no renewal — an actively-used login +died 30 days after `me login` regardless of activity. That became user-visible +once `me install` started defaulting to the login session (a logged-in +editor's MCP integration would silently break monthly). Changed to **rolling** +sessions matching better-auth's defaults: `validate_session` slides `expires_at` +to `now + 7d` on use, **throttled to ~once/day** (only when <6d remains, i.e. +window − updateAge), with **no absolute cap**. The function is now `volatile` +(was `stable`) and does at most ~one write/session/day on the hot path. + +**Decision:** adopt better-auth's model verbatim (expiresIn=7d, updateAge=1d, no +cap). Initial window also dropped 30d → 7d (`SESSION_EXPIRY_DAYS`). + +**The open tradeoff — no absolute cap:** OWASP recommends pairing an idle timeout +(which we now have: 7d) with an **absolute timeout** (a hard ceiling regardless of +activity) so a leaked-but-actively-used session can't roll forever. better-auth +omits this by default and we followed suit, prioritizing "never log out an active +user." A continuously-used (or exfiltrated-and-used) session never expires; +mitigations remain `me logout` / `deleteSessionsByUser` (revoke-all). + +**How to add a cap later:** store `absolute_expires_at = created_at + ` (or +compute from the existing `sessions.created_at`) and `least(now()+7d, +absolute_expires_at)` in the `validate_session` bump; force re-login past it. +Window/throttle live in `packages/database/auth/migrate/idempotent/002_session.sql` +(`validate_session`) and `SESSION_EXPIRY_DAYS` in `packages/auth/db.ts` — keep the +two windows in sync. + +**Status:** decided (copy better-auth); revisit the absolute cap if the +long-lived-bearer surface becomes a concern. + +--- + +## Should an agent get `share` access on join by default, or no grants (as now)? + +**Date:** 2026-06-08 · **Area:** membership (`me agent add` / `principal.add`) + +Surfaced by the e2e suite: `me agent add` puts the agent on the roster but +grants it **nothing**, so a freshly-added agent (with a minted key) gets +`No access to this space` on its first `me search` — the auth gate is a +non-empty `build_tree_access`, and an agent joins with zero grants (see the +"Home grant at join is for users only" entry: agents get no auto home because +the `agent_tree_access` clamp would make it inert). To make the agent usable the +owner must run an explicit `me access grant share r` (or similar) after +adding it. The e2e api-key scenario does exactly that. + +**The decision:** when an agent is added to a space, should it automatically +receive a default grant — most naturally **read on `share`**, the shared root — +so it's immediately usable, or should it keep getting **no grants** (today), +requiring the owner to grant access explicitly? + +**Why it's a real decision:** weigh ergonomics (an added agent that can do +nothing until a second, easily-forgotten grant command is surprising) against +least-privilege (an agent should see only what its owner deliberately shares). +Note the clamp: an agent's effective access is bounded by its owner's, so a +default `read@share` would only take effect when the owner themselves can read +`share` (the space creator owns it; an invited member may or may not). A default +also raises "which level/path" (read vs write, `share` vs space-root) and whether +it should apply to all join paths (`principal.add`, invite redemption) or only +self-service `me agent add`. + +**How to change it (add a default):** in `add_principal_to_space` +(`packages/database/core/migrate/idempotent/006_membership.sql`) add an +agent-branch that writes a `read @ share` grant (mirroring the user home-grant +branch gated on `p.kind = 'u'`), or do it at the RPC layer in `principal.add` +(`packages/server/rpc/memory/principal.ts`). Keeping it in the SQL chokepoint +makes it uniform across every join path. + +**Status:** needs decision. + +--- + +## No cross-schema FK between `core.principal` and `auth.users` + +**Date:** 2026-06-06 · **Area:** auth / core schema boundary + +For a user principal, `auth.users.id == core.principal.id`. That invariant is +**app-enforced only** — `provisionUser` writes both rows with the same id in one +`sql.begin` transaction (`packages/server/provision.ts:80,89`), and the two +schemas reference each other nowhere (`core.principal` has no FK to `auth.users`; +the `auth` migrations never mention `core`). **Decision: keep it app-enforced — +do not add a DB-level cross-schema FK now.** + +**Alternative considered:** add `core.principal.user_id references auth.users(id) +on delete cascade`. This is clean in shape — `user_id` is the generated column +(`= id` when `kind='u'`, else null) and FKs ignore null columns, so it would +constrain *only* user principals and leave agents/groups untouched; the cascade +would also make "delete an identity" tear down the principal + its grant graph in +one statement. + +**Why defer:** + +- **It makes migration order load-bearing, and today it isn't.** `auth` and + `core` are independent migrate runners; call sites order them inconsistently + (`authenticate-space` migrates auth→core; the agent/api-key integration tests + migrate core→auth). A core→auth FK forces auth-before-core everywhere and would + require standardizing production orchestration + fixing those test setups. +- **It forecloses the deliberate split-DB hedge.** The no-FK decoupling is + intentional — `packages/database/index.ts` notes `auth` could be "distributed + across databases again" (it *was* a separate DB before the recent + consolidation). A cross-schema FK only works within one database. +- **The drift it guards against is near-zero today.** The invariant has exactly + one writer (`provisionUser`, atomic), there's no user-deletion flow yet, and in + v1 every user principal is created via OAuth login (so always has an + `auth.users` row). +- **It would prematurely settle a deferred design question** — "standalone + non-OAuth users" (service accounts) are deferred; a hard FK bakes in "every user + principal has an `auth.users` row," which should be decided when that lands. + +**How to change it (add the FK):** add `core.principal.user_id references +auth.users(id) on delete cascade` (uses the existing u-only generated column), +standardize the migration order to **auth-first** (production + the integration +test `beforeAll`s), and decide whether standalone users get an `auth.users` +identity row. The natural moment is when adding a `user delete` flow or finalizing +standalone users — the cascade-on-identity-delete becomes a concrete win then. A +cheap interim guard: a test asserting every `core.principal` `kind='u'` has a +matching `auth.users` and vice versa. + +**Status:** decided (defer); revisit with user-deletion / standalone users. + +--- + +## Claude Code plugin captures via the import path (Stop/SessionEnd transcript), not per-event + +**Date:** 2026-06-09 · **Area:** claude-plugin / capture hook (`dd28e26`) + +The plugin previously registered **`UserPromptSubmit`** (store your prompt) and +**`Stop`** (store the assistant's *final* message via `last_assistant_message`), +producing two memories per turn with a **bespoke metadata schema**. It now +registers **`Stop`** (after each turn) and **`SessionEnd`** (final flush), and +each fire reads `transcript_path` and runs the session through +`importTranscriptFile` — the *same* parse + write as `me … import`. So live +captures and bulk imports are **identical by construction**: same tree +(`share.projects..agent_sessions`), deterministic ids, and `source_*` +metadata, one memory per message. + +**Why:** the two capture paths had drifted (different metadata vocab — even a +conflicting `type` value — and the hook only caught the prompt + final message, +missing intermediate messages / tool calls / reasoning). Reusing the importer +gives parity, completeness, idempotency (deterministic message ids), and one +code path. Incremental + stateless: each fire does one `limit 1` newest-first +search for the session's high-water message (relies on the `orderBy`-desc default +fixed in `e9a6eec`) and writes only the delta; it falls back to the full +reconcile for a new session / `importer_version` bump / lost anchor / write error. + +**Maintainer decisions baked in:** + +- **Lost prompt-on-submit durability.** Dropping `UserPromptSubmit` means a turn + that never reaches `Stop`/`SessionEnd` (interrupt-and-quit, kill, API error) + isn't captured. Narrow: `Stop` fires per turn and re-imports the + session-so-far, so only the *last in-flight* turn is at risk. Keeping a + lightweight `UserPromptSubmit` safety-net was rejected — it reintroduces dual + paths and double-captures the prompt (the live copy has no message id to dedupe + against the transcript copy). +- **`SessionEnd` in addition to `Stop`** (vs `Stop`-only): a cheap final flush; + no local state to clean up since the watermark is server-derived. +- **`content_mode` default `"default"`** (user + assistant text). `full_transcript` + (reasoning + tool calls/results) is an opt-in plugin userConfig — off by default + because it's much larger/noisier and may capture sensitive tool output. + +**How to change it:** the hook command is `me claude hook --event stop|session-end` +([packages/cli/commands/claude.ts]) → `importTranscriptFile` +([packages/cli/importers/index.ts]); registered events live in +[packages/claude-plugin/hooks/hooks.json]. To restore prompt-on-submit capture, +re-add a `UserPromptSubmit` hook (and a dedupe story). To make `full_transcript` +the default, flip the `content_mode` userConfig default in +[packages/claude-plugin/.claude-plugin/plugin.json] (and the hook's +`resolveHookConfigFromEnv`). + +**Status:** decided (per request); document the capture model when the docs are +refreshed. + +--- + +## Imports reorganized under one `me import ` group; bare `me import ` removed + +**Date:** 2026-06-10 · **Area:** CLI command surface (`fc02772`) + +All imports now live under a single top-level umbrella group — `me import +memories | claude | codex | opencode | git` — one subcommand per **source**. +The pre-existing spellings remain registered as aliases built from the same +factories: `me memory import` ⇒ `me import memories`, and `me claude|codex| +opencode import` ⇒ `me import `. Two breaking consequences, both +deliberate: + +- **Bare `me import ` no longer parses.** `import` was previously the + auto-generated top-level alias of `me memory import`; the group now owns the + name, has **no default subcommand**, and its help text redirects old muscle + memory to `me import memories `. (Verified at the time: nothing in the + repo — tests, hooks, packs, docs — used the bare spelling.) +- `import` is excluded from the memory group's top-level auto-aliases + (`createMemoryAliasCommands`); every other memory subcommand (`me search`, + `me create`, …) still gets one. + +**Alternatives considered:** + +- *Per-source top-level groups* (`me git import`, matching `me claude import`): + rejected — every new source (gemini sessions, GitHub issues, Slack, …) would + cost a top-level command group, most containing only `import`. With the + umbrella, a new source is one subcommand; the integration groups (`me claude` + etc.) keep only genuine setup verbs (install/init/hook). +- *`files` as the umbrella's default subcommand* so bare `me import ` + keeps working (Commander `isDefault`): rejected — backward compatibility for + the bare spelling wasn't wanted, and a default reintroduces the ambiguity the + group exists to remove (`me import git` = the git source or a file named + `git`?). +- *Dropping the old spellings entirely*: rejected — `me memory import` is kept + for symmetry with `me memory export` (the data-plane inverse), and the + per-agent `import` aliases are kept since those groups exist anyway. +- Subcommand name `memories` over `files` — records are memories, not generic + files. + +**How to change it:** the group is assembled in +`packages/cli/commands/import-group.ts` (`createImportCommand`), from factories +that take a name parameter (`createMemoryImportCommand(name)` in +`commands/memory-import.ts`; `createClaudeImportCommand` etc. in +`commands/import.ts`); the alias exclusion is the `c.name() !== "import"` +filter in `commands/memory.ts:createMemoryAliasCommands`. To restore a bare +default, register the memories subcommand with Commander's `isDefault`; to drop +the legacy aliases, remove the `addCommand` calls in `memory.ts` / +`claude.ts` / `codex.ts` / `opencode.ts`. Docs: `docs/cli/me-import.md` is the +group page; the per-group pages note their alias status. + +**Status:** decided (user-directed, pre-release); recorded for rationale — +already reflected in `CLAUDE.md` and `docs/cli/`. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ff80e6b..6ee1043 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -5,6 +5,39 @@ - [Bun](https://bun.sh) (latest) - [Docker](https://www.docker.com/) (for PostgreSQL) +## Quick Start against dev + +### 1. Clone the repo + +```bash +git clone git@github.com:timescale/memory-engine.git +cd memory-engine +``` + +### 2. Install + +```bash +./bun install +./bun run install:local +``` + +### 3. Log in and install the Claude Code plugin + +```bash +me --server https://me.dev-us-east-1.ops.dev.timescale.com login +me claude install --dev +``` + +Login must come first: `me claude install` needs your session and the +stored server URL. `--dev` installs the Claude Code plugin from your local +checkout (run it from inside the repo) instead of the published +marketplace — with it installed, `me claude init` won't offer to install +the published plugin over it. + +After that follow the instructions from login. The next step will probably +be `me claude init` in whatever project you are working in. Don't test on +memory-engine itself as that can be confusing for the model. + ## Quick Start ```bash @@ -52,24 +85,12 @@ cp .env.sample .env #### Required variables -**Database connections** — both point to the local Docker Postgres, but use separate databases: +**Database connection** — one database holds the `auth` + `core` control plane and every per-space `me_` schema: ``` -ACCOUNTS_DATABASE_URL=postgres://postgres@localhost:5432/accounts -ENGINE_DATABASE_URL=postgres://postgres@localhost:5432/shard1 -``` - -**Encryption master key** — 32-byte hex string for encrypting API keys at rest: - -```bash -./bun run generate:master-key +DATABASE_URL=postgres://postgres@localhost:5432/memory_engine ``` -Paste the output into `.env`: - -``` -ACCOUNTS_MASTER_KEY= -``` **Server URL and port** — `API_BASE_URL` is used to construct OAuth callback URLs. `PORT` controls which port the server listens on. They must be consistent: @@ -123,9 +144,7 @@ GITHUB_CLIENT_SECRET=... ```bash # Database -ACCOUNTS_DATABASE_URL=postgres://postgres@localhost:5432/accounts -ENGINE_DATABASE_URL=postgres://postgres@localhost:5432/shard1 -ACCOUNTS_MASTER_KEY=<./bun run generate:master-key> +DATABASE_URL=postgres://postgres@localhost:5432/memory_engine # Server API_BASE_URL=http://localhost:3000 @@ -190,8 +209,9 @@ After login, the server URL is stored as the default in `~/.config/me/credential | `./bun run pg` | Build and start PostgreSQL in Docker | | `./bun run pg:rm` | Stop and remove the PostgreSQL container | | `./bun run psql` | Connect to PostgreSQL with psql | -| `./bun run test` | Run tests | -| `./bun run check` | Format + lint + typecheck | +| `./bun run test` | Run all package tests (unit + integration, vs local Postgres by default) | +| `./bun run check` | Fast inner loop: typecheck + lint + unit tests (no database) | +| `./bun run check:full` | Everything: check + full suite + e2e (vs local Postgres by default) | | `./bun run build` | Compile CLI binary (current platform) | | `./bun run build:all` | Cross-compile CLI for all platforms | | `./bun run install:local` | Build and install local CLI binary to your PATH | @@ -387,16 +407,21 @@ Implications: ### Adding a migration -Database migrations run at server startup using `SERVER_VERSION` as the -`serverVersion` passed to the runners. When you add a new migration file under -`packages/engine/migrate/migrations/` or `packages/accounts/migrate/migrations/`: - -- Cut a **server** release afterwards (`./bun run release:server`) to advance - `SERVER_VERSION` and propagate the migration to prod. -- The migration's `applied_at_version` column and the schema's `version` - row will reflect the new `SERVER_VERSION`. -- Rolling back to an older server image will then trip the downgrade guard - in the migration runners — by design. +Migrations live under `packages/database/{auth,core,space}/migrate/` in two +flavors: `incremental/` (versioned DDL, applied exactly once per schema, +tracked by name) and `idempotent/` (function/index definitions, re-applied on +every migrate run via `create or replace`). + +Migrations run at server startup (`startServer`, unless `migrate: false`): +the `auth` and `core` schemas are migrated, then **every existing space +schema is re-migrated** (enumerated from `core.space`) so changes to the +idempotent space SQL reach already-provisioned spaces on the next deploy — +spaces are otherwise only migrated once, when provisioned. A failed space +re-migration aborts boot; concurrent replica boots are serialized by a +per-schema advisory lock. + +Rolling back to an older server image trips the downgrade guard in the +migration runners (stamped schema version newer than the app's) — by design. ## Troubleshooting diff --git a/README.md b/README.md index 5abdcbb..79d1087 100644 --- a/README.md +++ b/README.md @@ -30,21 +30,36 @@ npm i -g @memory.build/cli # Authenticate me login +# Set up Claude Code memory for a project — run at the project root +cd ~/code/your-project +me claude init +``` + +`me claude init` does the whole setup in one shot: installs the Claude Code +plugin (hooks + slash commands + MCP) if it isn't already, backfills the +project's past Claude Code sessions and git commit history as searchable +memories, and records the project's memory location in `CLAUDE.md` so agents +consult it. From then on, new sessions are captured automatically. + +## Usage + +```bash # Store a memory -me memory create "Auth uses bcrypt with cost 12" --tree design.auth +me memory create "Auth uses bcrypt with cost 12" --tree share.design.auth # Search by meaning + keywords me memory search "how does authentication work" -# Connect to your AI tools +# Import memories, agent sessions, and git history +me import memories notes.md # md / yaml / json / ndjson records +me import claude # all Claude Code sessions on this machine +me import git # a repo's commit history +me import git-hook # keep it current via a post-commit hook + +# Connect other AI tools (Claude Code is covered by `me claude init`) me opencode install me codex install me gemini install -me claude install # MCP-only - -# Or, for the full Claude Code plugin (hooks + slash commands + MCP): -claude plugin marketplace add timescale/memory-engine -claude plugin install memory-engine@memory-engine ``` ## How it works @@ -56,7 +71,7 @@ Memory Engine runs as an MCP server that AI agents connect to over stdio. Each a - **ltree** for hierarchical tree paths - **JSONB + GIN** for metadata filtering - **tstzrange** for temporal queries -- **Row-Level Security** for access control +- **Tree-scoped access grants** evaluated in the search SQL (no RLS) ## Documentation diff --git a/REDESIGN.md b/REDESIGN.md new file mode 100644 index 0000000..7a592e4 --- /dev/null +++ b/REDESIGN.md @@ -0,0 +1,848 @@ +# Memory Engine (Re)Design + +## Migration Strategy + +Build the new implementation in parallel. When we're happy with it, stand up the database, stop the old server, port/copy/migrate the data from the old prod to the new database, and start the new server. We can practice this migration strategy if needed. This approach also lets us switch embedding models as we migrate; don’t move the embeddings and reembed once moved. + +## V1 Scope and Non-Goals + +This section captures what the first shippable version of the redesign covers and what it deliberately does not. + +### Out of Scope for V1 + +The following are intentionally deferred. They may be revisited in a later version, but they are not built in v1 and code/design should not assume them: + +- Standalone non-OAuth users. User principals are created through OAuth login. Shared service accounts, integrations, and non-human first-class accounts that authenticate only via API keys are valuable but deferred. +- Hosted MCP. The hosted API is JSON-RPC over HTTPS. MCP support exists only in the local stdio proxy. +- Magic/private-path authorization semantics. Private areas, if any, are modeled with explicit tree layout and grants, not with reserved path patterns or implicit deny rules. +- WebSocket and other streaming transports. Bulk import/export uses HTTPS in v1 and may grow chunked endpoints or signed object storage later if needed. +- Actual sharding. The schema and authorization boundary are designed to allow it, but v1 runs all spaces in a single database. +- Billing. The `core` schema may eventually host it, but v1 does not implement billing tables or flows. + +### Non-Goals + +The following are intentionally rejected. Do not build them, and stop and write down the requirement instead if a use case appears to need them: + +- Deny rules and negative access. The access model is monotonic. +- Nested groups. A group may contain users and agents only. +- Agent space-admin or group-admin authority. Agents cannot hold or inherit administrative authority, even via membership in an admin-flagged group. + +## Core + +The `core` schema is a singular, global set of tables. The core manages authentication and authorization. Eventually, it will also handle billing. + +Core SQL does not need to be templated because there is only one core schema. The schema name and table names are stable, and all references should still be schema-qualified for safety. + +### `core.version` + +`core.version` records the current schema version of the global core schema. It is a singleton table: the core schema has exactly one version row. + +The version row lets the server determine whether the core schema is current, needs migration, or is newer than the running server can safely handle. Unlike space schemas, the core schema is singular and global, so core migration state is not scoped by space. + +The version table is intentionally separate from the migration table. The migration table records which steps were applied, but the version row gives the server a cheap compatibility check before doing any real work. If an old server connects to a newer database, it can reject the connection immediately. If the server version matches the database version, it can skip migration checks altogether. + +### `core.migration` + +`core.migration` records which incremental migrations have been applied to the core schema. Each applied migration is recorded once with the target version and timestamp at which it was applied. + +Core migrations use the same incremental/idempotent approach as space schemas. Incremental migrations create or transform durable tables and data exactly once. Idempotent SQL can be re-run safely to refresh functions, triggers, views, policies, and other replaceable database objects. + +Keeping core migration history in `core.migration` makes bootstrap and upgrade behavior explicit and idempotent while keeping it separate from each space's own migration history. + +### `core.space` + +`core.space` enumerates the memory containers in the system. A space is an isolated repository of memories with its own groups and tree access rules. + +Spaces are the user-facing boundary between contexts. A person may have a personal space, belong to an employer's space, and participate in other shared spaces without those memories or access rules accidentally mixing. Memory-oriented commands run in the context of one selected space. Spaces should feel "air gapped." + +Each space has a stable slug used to identify it in URLs, CLI configuration, and the physical schema that stores its large operational tables. The space record also tracks placement information, such as the shard where the space's memory tables live, so the global core schema can route operations to the correct database location. + +### `core.principal` + +`core.principal` stores every identity-like thing that can receive privileges, authenticate, appear in audit fields, or participate in group membership. + +Principals have three kinds: + +- `user`: a first-class principal that is not owned by another principal. This is usually a human OAuth user, but may also be a standalone non-human account such as a shared service account, app, or integration. +- `group`: a collection or capability principal. Groups receive privileges, and users/agents inherit those privileges through `core.group_member`. +- `agent`: a user-owned non-human principal, such as an agent, script, local app, bot, or scheduled job. Agents are used when a human wants a tool to act with attributable, usually narrower, access. + +Agents exist to make agent/script access self-service. A user can create ~~zero or more~~ agents without being a space admin. The owning user can manage the agent's lifecycle and can grant it access up to the access the owner already has. Agents are normal principals for authorization purposes: they can receive direct tree access, belong to groups, authenticate with API keys, and appear in audit fields. + +Principal names follow the scope where people naturally expect them to be unique. User names are global because users are global identities that can participate in many spaces. Group names are space-specific because groups like `engineering`, `admin`, or `design` should be meaningful inside a space without conflicting with similarly named groups in other spaces. Agent names are scoped under their owning user, so multiple users can each have an agent named `claude`, `opencode`, or `importer` without conflict. + +Groups are the only principals that are intrinsically scoped to a single space. A group principal records the space where it is defined, which allows group names to be unique per space. Users and agents are global principals; their relationship to spaces is represented separately through `core.principal_space`. + +With the exception of the initial bootstrap user, principals start with a blank slate: no group membership, no tree access, and no space administrative privileges. Access must be granted directly, inherited through group membership, configured through agent ownership rules, or assigned by setting the space admin flag. + +Agents do not start out with access equal to the owning user. If the access was intended to be equal, there's not much benefit to creating an agent (just use the user itself) (other than attribution in the case that we ever build audit logs). The major feature of having agents is the ability to give them more restricted access. Agents start with blank-slate access. + +### `core.principal_space` + +`core.principal_space` records which principals belong to which spaces. A principal may exist globally without being admitted to every space. To operate in a space, a user, group, or agent must have a `principal_space` row for that space. + +This table is the boundary between global identity and space-local authorization. Users are global and may participate in many spaces. Agents belong to their owning user globally, but must still be admitted to a space before receiving access there. Groups are space-specific; a group principal belongs to exactly one space. + +`principal_space` does not grant memory access on its own. It establishes that the principal is known in the space and records space-local state, including whether the principal is a space admin. Actual memory access comes from `core.tree_access`, either granted directly to the principal or inherited through `core.group_member`. + +Groups still have `principal_space` rows even though their owning space is also recorded on the principal. This deliberate duplication keeps one rule for authorization: any principal that participates in a space has a `core.principal_space` row. It also gives groups the same space-local state as users and agents, including active/disabled state and the space admin flag. + +The `admin` flag on `core.principal_space` is the space-wide administration capability. A principal with `admin = true` can administer users, agents, groups, group membership, invitations, and tree access in that space. If a group has `admin = true`, members of that group inherit space admin authority through `core.group_member`. The admin flag does not itself grant memory visibility; memory visibility and write authority still come from `core.tree_access`. However, an admin can grant tree access to any principal, including themselves. + +The initial user for a space receives `admin = true` and explicit `owner` access to the root tree path. This makes initial single-player use straightforward while keeping space administration and memory visibility represented separately. The system should prevent removing or demoting the last admin principal in a space. + +### `core.space_invitation` + +`core.space_invitation` stores pending invitations for humans to join a space. An invitation is not a principal and does not grant access by itself. It is a pending offer to admit a user principal into a space. + +Invitations are usually addressed to an email address. The invited human may already have a user principal in the system, or they may be entirely new. The system should not create a new principal merely because an invitation was sent. A principal is created or resolved only when the invited person authenticates and accepts the invitation. + +Accepting an invitation creates a `core.principal_space` row for the accepting user in the invited space. If the invitation includes initial group membership, accepting also creates the corresponding `core.group_member` rows. Actual memory access still comes from group membership and `core.tree_access`; the invitation itself never grants direct memory access. + +If invitations are email-based, acceptance should require the authenticated OAuth identity to have a verified email address matching the invitation. Possession of an invite link alone should not be sufficient, because links can be forwarded. + +Invitations should have a lifecycle: pending, accepted, revoked, or expired. They should record who created them, when they were created, when they expire, who accepted them, and who revoked them if revoked. Revocation and expiration prevent future acceptance but do not affect already accepted memberships. + +Creating an invitation requires space admin authority. If the invitation includes initial group membership, the inviter must also be allowed to administer those groups, either through the membership admin flag or through space admin authority. + +Invitations avoid unwanted forced membership. A user is not added to a space until they accept, so unwanted invitations can be ignored, declined, revoked, or allowed to expire. + +### `core.group_member` + +`core.group_member` assigns users and agents to groups within a specific space. Every row is scoped by `space_id`, a group principal, and a member principal. A membership row means the member inherits privileges granted to the group in that space, including tree access and space admin authority. + +Groups are intentionally not nestable. A group may contain users and agents, but it may not contain another group. This avoids recursive membership graphs and keeps the authorization model easier to explain: a user or agent either belongs to a group directly or does not. + +Each membership can carry an `admin` flag. A member with `admin = true` can add and remove members for that group in that space, and can decide whether new memberships also receive the group membership admin flag. + +The `admin` flag on a group membership controls administration of that group only. It lets the member add and remove members for the group, but it does not imply ownership of memories, visibility into all memories, or space-wide administrative authority. + +Groups are the natural way to delegate access for teams. For example, a space admin can create a `project-x` group, add all project team members to that group, and grant the group `owner` access on `projects.x`. Members of `project-x` then inherit ownership of that tree branch and can manage access below `projects.x` without becoming space admins. + +### `core.tree_access` + +`core.tree_access` grants memory access to principals within a specific space. Every row is scoped by `space_id`, a principal, a tree path, and an access level. + +Access applies to the named tree path and all descendants. Granting access on `projects.x` also grants access to `projects.x.design`, `projects.x.budget`, and future children under `projects.x` in the same space. + +Access is a simple ladder: + +- `read`: can search, list, and get memories. +- `write`: includes `read`; can create, update, delete, move, and copy memories. +- `owner`: includes `write`; can grant and revoke access below the owned path. + +The model is monotonic. Grants add access; there is no deny table and no negative access rule. Removing a grant removes that exact grant, but does not create an exception below a broader inherited grant. + +There is no concept of "revoking" tree access. The only mutation primitives are "add grant" and "remove grant" (also called "delete grant"). Both operate on a specific `(space_id, principal_id, tree_path, access)` row. To remove a grant, the caller must specify the grant row that exists. If the requested grant row does not exist, the operation reports an error rather than silently succeeding. + +This is intentional. "Revoke access to `projects.x`" is ambiguous: does it mean "delete the matching grant row," "delete any grant that would imply access to `projects.x` (including ancestor grants)," or "make sure the principal can no longer access `projects.x` by some means"? Forcing the caller to name the exact grant row keeps the semantics explicit. If the caller's expected grant does not exist, the error surfaces the mismatch instead of hiding it behind a no-op. To reduce a principal's effective access on a subtree, remove the specific grant rows that produce that access; if access is inherited from a group, edit the group's grants or the principal's group membership instead. + +Tree access can be granted directly to users, agents, or groups. Group grants are inherited by users and agents through `core.group_member`. Agents receive normal tree access like any other principal, but their owner can self-service grants up to the owner's own access. + +Space admins can administer tree access anywhere in a space, but admin status does not itself imply memory visibility. Visibility and write authority come from `core.tree_access` rows. + +The `owner` access level is the scoped administration mechanism for tree paths. A principal with `owner` access on `private.mat` can grant and revoke access below `private.mat` without involving a space admin. This is the intended mechanism for private user areas: if Mat owns `private.mat`, Mat can decide which users, agents, or groups can access that subtree. + +The same pattern applies to collaborative project areas. Members of a team can be granted `owner` access on `projects.x`, allowing them to manage access for that branch of the tree without becoming space admins. This lets a space delegate administration of specific subtrees while keeping space-wide administration reserved for principals with `core.principal_space.admin = true`. + +### Agent Access + +There is a strong product argument that a user-owned agent should never have more access than its owning user. Agents exist so a user can give a tool attributable and usually narrower access than the user has. If an agent should have fully independent access, it may be better modeled as a first-class `user` principal rather than as a user-owned `agent`. + +There are two possible interpretations of this rule. + +The weaker interpretation is grant-time enforcement. Under this model, the system prevents an owner from granting an agent access the owner does not currently have. This is simple, but it does not preserve the invariant over time. If Alice grants `alice/agent` access to `projects.x` and later loses her own access to `projects.x`, the agent may retain stale access unless the system also finds and revokes it. Furthermore, tree access or group membership that Alice does not have might be granted directly to `alice/agent` by someone other than Alice. + +The stronger interpretation is runtime capping. Under this model, an agent's effective access is always capped by the owner's current effective access. The agent can still have direct tree access and group-derived access, but those grants are masked by the agent owner's grants. The actual access used at runtime is the intersection of the agent's configured access and the owner's current access. + +One implementation option is grant-time enforcement only. It is easy to implement and explain, but it is fragile. Maintaining the invariant would require cascading cleanup whenever an owner loses tree access, is removed from a group, loses space admin status, or otherwise has effective access reduced. + +Another implementation option is runtime access intersection. Conceptually, compute the agent's configured effective access, compute the owner's effective access, and intersect them. For tree access, intersection is tractable because grants are path-prefix rules: overlapping paths produce the more specific path and the lower access level. For example, if the owner has `write` on `projects` and the agent has `read` on `projects.x`, the agent effectively has `read` on `projects.x`. + +Runtime capping is more complex, but it actually enforces the invariant. It also handles later access changes automatically: if the owner loses access, the agent loses effective access without deleting or rewriting the agent's configured grants. + +Space admin status needs special care. The simplest v1 rule is that agents cannot be space admins. This must include inherited space admin authority from groups: if a group has `core.principal_space.admin = true`, an agent's membership in that group should not by itself make the agent an effective space admin. If agents are allowed to carry or inherit the space admin flag, then an agent's effective admin authority should also be capped by the owner, meaning the agent is effectively a space admin only when both the agent and the owner have space admin authority. + +Group membership has the same issue. If agents can belong to groups, group-derived access should still be capped by owner access at runtime. If that proves too complex for v1, a simpler initial version could allow agents to receive only direct tree access masks and defer agent group membership. + +The preferred long-term model is runtime capping: user-owned agents are constrained by the owner's current effective access, while standalone non-human actors that need independent access are modeled as first-class `user` principals. + +**V1** + +- Agents cannot be space admins. +- Agents cannot be group admins. +- Agents may be group members. Group-derived tree access for an agent is intersected with the owning user's current effective access at runtime, so the "agent never exceeds its owner" invariant holds whether access comes from direct grants or group membership. +- Membership in an admin-flagged group does not make an agent an effective admin. The space-admin and group-admin restrictions apply to inherited authority as well. +- Goal: an agent's tree access (direct and group-derived) is capped by the owner's tree access and enforced at runtime. +- Agent group membership should only be removed from v1 if it proves exceedingly difficult to implement correctly. The fallback in that case is agents-with-direct-grants-only, with agent group membership deferred to a later version. + +#### Agent Access Masking Implementation Sketch + +Here is a vibe-coded sketch of what runtime masking might look like. I'm not at all confident in its correctness. It does illustrate that while the masking approach is conceptually simple at face value, it is not straightforward to implement. + +The core masking operation takes two effective access sets: + +- the owner's effective access: `(tree_path, access)` +- the agent's configured effective access: `(tree_path, access)` + +The intersection rule is: + +- Two paths overlap when either path contains the other. +- The effective path is the more specific path. +- The effective access level is the lower of the two access levels. + +In SQL, the masking operation could look like this: + +```sql +with owner_access(tree_path, access) as +( + values + ('projects'::ltree, 1) + , ('projects.x'::ltree, 2) +) +, agent_access(tree_path, access) as +( + values + ('projects.x'::ltree, 1) + , ('projects.y'::ltree, 2) +) +, raw_intersection as +( + select + case + when o.tree_path @> a.tree_path then a.tree_path + when a.tree_path @> o.tree_path then o.tree_path + end as tree_path + , least(o.access, a.access) as access + from owner_access o + inner join agent_access a + on o.tree_path @> a.tree_path + or a.tree_path @> o.tree_path +) +, merged as +( + select + tree_path + , max(access) as access + from raw_intersection + group by tree_path +) +, reduced as +( + select m.* + from merged m + where not exists + ( + select 1 + from merged x + where x.tree_path @> m.tree_path + and x.tree_path <> m.tree_path + and x.access >= m.access + ) +) +select * +from reduced +order by tree_path; +``` + +The `reduced` step removes redundant descendant rows when an ancestor already grants equal or greater access. For example, `projects read` makes `projects.x read` redundant, but `projects.x write` is not redundant because it is stronger than the ancestor grant. + +For future access pushdown, the same operation can consume rendered JSONB access sets: + +```sql +with owner_access as +( + select tree::ltree as tree_path, access::int4 + from jsonb_to_recordset($1) as x(tree text, access int) +) +, agent_access as +( + select tree::ltree as tree_path, access::int4 + from jsonb_to_recordset($2) as x(tree text, access int) +) +-- apply the same intersection, merge, and reduction steps +``` + +Memory operations would use the capped effective access set: + +```sql +where exists +( + select 1 + from effective_access e + where e.access >= 1 + and e.tree_path @> m.tree +) +``` + +Write checks use `e.access >= 2`; scoped administration checks use `e.access >= 3`. + +The important implementation rule is that every authorization check should flow through one effective-access function. User access is direct access plus group-derived access. Agent access is the agent's configured access, including group-derived access, intersected with the owning user's current effective access. + +### Private Areas + +Multiplayer spaces create an immediate product need for private areas. Teams often want a broad shared context that most members can read or write, while still giving each human a place for notes, experiments, drafts, or agent context that should not be visible to everyone else. The desired user experience is something like “give the team write access to everything shared, but not to each person's private area.” + +Two approaches have been proposed. + +One approach is a special carve-out rule: reserve a path pattern such as `private.` or `~` and define root grants to exclude those paths automatically. This would make a broad root grant behave like “everything except private areas.” + +Another approach is to provision spaces with conventional top-level areas, such as `shared` and `private`. Broad team grants would apply to `shared`, while each user would receive owner access to their own subtree under `private`, such as `private.alice` or `private.bob`. + +The motivation is valid: users should not need to design an access model from scratch just to get a normal shared/private collaboration pattern. The open question is whether private areas should be implemented as special authorization semantics or as a recommended tree layout and provisioning convention. + +Magic private paths and implicit deny rules are problematic because they make grants harder to reason about. A grant on root would no longer mean root access; it would mean root access except for paths the system treats specially. That creates surprising behavior for users and makes it harder to explain why a principal can or cannot see a memory. + +They also complicate the access evaluator. The core tree access rule is currently simple: a principal can access a memory when an allowed tree path contains the memory's tree path. Special carve-outs mean every access check must also know about reserved path patterns and subtract them from otherwise valid grants. This pushes the model toward deny semantics even if there is no explicit deny table. + +Deny-like rules become especially awkward with inherited group access. If one group grants broad access and another rule implicitly denies a subtree, the system needs a conflict-resolution policy. Usually denies win, but that means adding a user to a group can unexpectedly remove access, and removing a rule can unexpectedly reveal data. Those interactions are difficult to present clearly in the product. + +Magic paths also constrain future tree organization. If `private.` or another pattern has special meaning, spaces cannot freely use that part of the tree for ordinary memories. The tree becomes partly user-defined and partly reserved by the authorization system, which is exactly the kind of hidden convention the design is trying to avoid. + +Finally, private-path carve-outs make efficient search harder. Memory search needs to combine BM25, HNSW, ltree, metadata, temporal filters, and authorization filtering while continuing to scan until enough authorized results are found. Keeping authorization as a positive set of grant paths maps cleanly to `ltree` containment checks. Subtracting special private paths adds another dimension to every search and makes future access pushdown/sharding more fragile. + +The existing primitives can already model the desired shared/private pattern without special authorization semantics. A space can place shared memories under a known branch such as `shared` or `public`, grant broad team access to that branch, and place per-user private memories under branches such as `private.alice` and `private.bob` with owner access granted only to the corresponding users. + +Under this model, “grant write to everything except private areas” becomes “grant write to `shared`.” The private areas are not exceptions to a root grant; they are simply outside the broadly granted subtree. + +We should defer magic private paths, implicit deny rules, and automatic private area behavior until real usage shows that the explicit tree-layout convention is insufficient. This keeps the v1 access model monotonic, efficient, and explainable. + +### `core.api_key` + +`core.api_key` stores API credentials for non-interactive authentication. A `user` or `agent` principal can have zero or more API keys. Groups cannot have API keys. + +API keys are global credentials for a principal, not credentials for a specific space. After authenticating with an API key, the principal may operate only in spaces where it has been admitted through `core.principal_space` and only with the access granted through `core.tree_access` or inherited through `core.group_member`. + +This keeps key management attached to the principal rather than duplicating credentials per space. A user's agent can use the same key across multiple spaces if it has been admitted to those spaces, while still receiving different access in each space. + +API keys should support independent lifecycle management. Keys can be created, listed, revoked, and rotated without deleting the principal. A user can manage keys for their own agents, and space admins can create/delete keys for any user or agent when as required. + +An API key should be split into a lookup component and a secret component. The lookup component is stored in plaintext for efficient key lookup. The secret component is shown once to the caller and stored only as a strong hash. Authentication succeeds only when both identify the same active key. + +### `core.oauth_identity` + +`core.oauth_identity` stores durable links between OAuth provider identities and user principals. It answers the question: when Google, GitHub, or another OAuth provider says this is user X, which `core.principal` should that authenticate as? + +An OAuth identity belongs to a `user` principal. Agents authenticate with API keys, and groups do not authenticate directly. + +The durable identity key is the OAuth provider plus the provider's stable subject identifier. Email addresses, display names, and avatars are useful profile metadata, but they are not the primary identity because emails can change and may not be verified. + +A user may have multiple OAuth identities linked over time. For example, the same user principal may be linked to both a Google identity and a GitHub identity. A single provider identity should map to only one user principal. + +`core.oauth_identity` should not store transient OAuth state. It is the long-lived account link used after an OAuth flow has completed and the provider identity has been verified. + +### `core.oauth_flow` + +`core.oauth_flow` stores short-lived state for OAuth login flows. This includes the temporary values needed to complete browser-based, CLI, or device-code authentication safely. + +OAuth flows are not durable account links and are not login sessions. They exist only while authentication is in progress. After the flow succeeds, the system links or resolves a `core.oauth_identity` and creates a `core.session`. After the flow fails, expires, or is consumed, the flow record can be removed. + +The flow record should contain enough information to validate the callback or polling request, protect against CSRF/replay, and resume the intended login operation. Depending on the OAuth mode, this may include provider, state, PKCE verifier/challenge data, device code metadata, redirect target, expiration time, and consumption status. + +OAuth flow records should be treated as temporary credentials. They should expire quickly, be single-use where possible, and never grant space access by themselves. + +### `core.session` + +`core.session` stores interactive login sessions created after a user authenticates through OAuth. A user can have one or more active sessions, such as sessions from different machines, browsers, or CLI installations. + +Sessions are global credentials for a user, not credentials for a specific space. After authenticating with a session, the user may operate only in spaces where the user has been admitted through `core.principal_space` and only with access granted through `core.tree_access` or inherited through `core.group_member`. + +Sessions support normal login lifecycle management. They can be created at login, refreshed or extended according to policy, listed for account security, and revoked during logout or credential cleanup. Revoking a session invalidates that session without affecting the user, their other sessions, or their API keys. + +Sessions are for user principals authenticated by OAuth. Agents and standalone non-interactive clients should use `core.api_key` instead. + +## Space + +Each space has a corresponding PostgreSQL schema that holds the space-local operational tables. Space schemas are created on demand when a space is created or first provisioned. + +The DDL for a space schema is rendered from templates. The most important template variable is the schema name itself, because each space has its own schema. All table, index, trigger, and function references in the rendered SQL should be schema-qualified for safety and to avoid accidental dependence on `search_path`. + +Space provisioning also needs per-space configuration. Different spaces may use different embedding models, and different embedding models may have different vector dimensions. The embedding dimension is therefore a template variable used when creating the `embedding` column and vector indexes. The chosen embedding model, embedding dimension, and other space-local database tuning options should be recorded in `core.space` so the server can route embedding work and future migrations correctly. + +This design lets small installations keep all spaces in one database while preserving an operational boundary for future scaling. A space schema can later be placed on a different shard without changing the logical authorization model in `core`. + +## `.memory` + +`.memory` is the primary per-space table. Each row is one memory in the space. This table stays in the space-specific schema because it is the large, search-heavy operational data that will eventually need to scale and shard independently from global authorization metadata (if we are successful). + +Each memory has a UUIDv7 primary key, textual `content`, arbitrary object-shaped JSON metadata in `meta`, a hierarchical `tree` path, optional temporal range information, an optional embedding vector, and timestamps. The `tree` path is the basis for both organization and authorization. The `meta` column supports flexible user-defined structure without creating additional tables for every memory type. + +The table supports three main search dimensions: + +- BM25 full-text search over `content`. +- HNSW vector similarity search over `embedding`. +- Structured filtering over `tree`, `meta`, and `temporal`. + +The `embedding_version` column tracks whether the stored embedding corresponds to the current memory content. When content changes, the embedding is cleared and the version advances so the embedding worker can regenerate the correct vector and ignore stale queue items. + +Temporal values follow one convention. Point-in-time memories use an inclusive single-point range. Period memories use an inclusive-exclusive range. This keeps temporal filtering predictable and avoids ambiguous range boundary behavior. + +## `.embedding_queue` + +`.embedding_queue` is the per-space work queue for embedding generation. Queue rows point to memories in the same space and record the `embedding_version` that should be generated. + +The queue is version-aware. Multiple queue rows may exist for the same memory over time, but workers claim only unresolved rows and can ignore work for older embedding versions when a newer version exists. This prevents stale embedding work from overwriting newer memory content. + +Queue visibility is controlled by `vt`, the visibility time. Workers claim rows whose `vt` is due and whose `outcome` is still null. Attempts and `last_error` record retry history. Finished rows are marked with an outcome such as `completed`, `failed`, or `cancelled` and can later be pruned. + +The queue is space-local for the same reason as `memory`: embedding work is tightly coupled to space-local memory rows and should scale with the memory shard. + +## `.version` + +`.version` records the current schema version of the space-local database objects. It is a singleton table: each space schema has exactly one version row. + +The version row lets the server determine whether the space schema is current, needs migration, or is newer than the running server can safely handle. This check is space-local because spaces may live on different shards and may be migrated independently. + +The version table is intentionally separate from the migration table. The migration table records which steps were applied, but the version row gives the server a cheap compatibility check before operating on the space. If an old server connects to a newer space schema, it can reject the operation immediately. If the server version matches the space schema version, it can skip migration checks for that space altogether. + +The table should also record when the version was last updated so migrations and operational tooling can inspect the state of a space without relying only on global metadata. + +## `.migration` + +`.migration` records which incremental migrations have been applied to a space schema. Each applied migration is recorded once with the target version and timestamp at which it was applied. + +The migration table makes space provisioning and upgrades idempotent. When a migration runs, the migrator can skip incremental migrations that have already been recorded and apply only the missing ones. After incremental migrations complete, idempotent SQL can be re-run safely to refresh functions, triggers, and other replaceable database objects. + +Keeping migration history inside the space schema makes each space self-describing. This is useful when spaces are created on demand, upgraded independently, or eventually moved across shards. + +## Authorization Boundary + +Authorization metadata lives in the global `core` schema. This includes spaces, principals, group membership, tree access, OAuth identities, sessions, and API keys. The per-space schemas hold the large operational data: `memory` and `embedding_queue`. + +Keeping authorization metadata in `core` is important because effective access depends on several related facts: principal kind, space membership, space admin state, group membership, group administration, direct tree access, group-derived tree access, agent ownership, and the owner's own effective access. Resolving that graph should happen in one transactional context over tables with real foreign key constraints. + +The boundary between authorization and memory operations should be an effective access set. Memory operations should not know how to interpret principals, groups, agents, or space admin state. They should consume rows shaped like: + +```sql +(tree_path ltree, access int4) +``` + +The core schema should expose a function similar to: + +```sql +core.effective_tree_access +( _space_id uuid +, _principal_id uuid +) +returns table +( tree_path ltree +, access int4 +) +``` + +This function is responsible for resolving direct access, group-derived access, and agent access masking. For a user, effective access is direct tree access plus group-derived tree access. For an agent, effective access is the agent's configured access, including group-derived access, intersected with the owning user's current effective access. + +Initially, space-specific memory functions can call `core.effective_tree_access(...)` directly in a materialized CTE. This preserves referential integrity for principals, groups, and tree access while keeping access evaluation inside SQL where BM25, HNSW, and tree indexes can be used correctly. + +```sql +with effective_access as materialized +( + select * + from core.effective_tree_access($1, $2) +) +select m.* +from space_slug.memory m +where exists +( + select 1 + from effective_access a + where a.access >= 1 + and a.tree_path @> m.tree +); +``` + +In a future sharded implementation, the coordinator can call the same core function, serialize the result, and pass it to the shard-local memory function. + +```sql +select jsonb_agg +( + jsonb_build_object + ( 'tree', tree_path::text + , 'access', access + ) +) +from core.effective_tree_access($space_id, $principal_id); +``` + +A pushed-down access set would be a small list of tree paths and access levels, for example: + +```json +[ + { "tree": "projects.x", "access": 2 }, + { "tree": "shared.docs", "access": 1 } +] +``` + +The shard-local function would parse the JSONB as a recordset, materialize it, and join memories against it. + +```sql +with effective_access as materialized +( + select tree::ltree as tree_path, access::int4 + from jsonb_to_recordset($access_jsonb) + as x(tree text, access int) +) +select m.* +from space_slug.memory m +where exists +( + select 1 + from effective_access a + where a.access >= 1 + and a.tree_path @> m.tree +); +``` + +Clients and agents must never provide this access set directly. It is produced only by trusted server-side code after authentication and authorization. In the sharded version, the rendered access set is a snapshot for a trusted operation. If authorization changes after rendering but before shard execution, the shard executes against the rendered snapshot. + +This gives us the simple first implementation while preserving a migration path for vertical scaling limits and eventual sharding. The memory layer always consumes effective access; only the source of that effective access changes. + +## Deletion and Cascading + +### No Soft Deletes + +One cardinal rule: no soft deletes. Anywhere. Ever. + +Soft deletes (`deleted_at`, `archived_at`, `is_deleted`, `active = false`, or any other "tombstone in the live table" pattern) cause problems that compound over time: + +- They break unique constraints. A `name` column that should be unique now has to be unique-among-non-deleted, which means partial indexes and conditional uniqueness logic everywhere. +- They break foreign key constraints. Downstream rows can keep pointing to "deleted" parents, so every join has to filter on the soft-delete flag or risk leaking removed data. +- They bloat tables with rows nobody is supposed to see. Production tables can end up 90% dead rows, with the database wading through garbage to satisfy every query. +- They make application code ambiguous. Every query has to remember to exclude soft-deleted rows. Forgetting once is a bug; forgetting in a search query is a security bug. + +The rule is simple: tables represent live state. If a row is no longer live, hard delete it. + +### When to Hard Delete + +Prefer hard deletes by default. Specifically: + +- `core.session`: hard delete on logout, revoke, or expiry cleanup. +- `core.oauth_flow`: hard delete after consumed, failed, or expired. +- `core.api_key`: hard delete on revoke. +- `core.space_invitation`: hard delete on revoke, expiry, or after the invitation has been accepted and no longer needs to be visible. +- `core.group_member`: hard delete when membership is removed. +- `core.tree_access`: hard delete when a grant is revoked. +- `core.principal_space`: hard delete when a principal is removed from a space. +- `core.principal` (groups, agents, users): hard delete when removed, subject to cascade rules below. +- `.embedding_queue`: hard delete completed, failed, and cancelled rows via periodic cleanup. +- `.memory`: hard delete on memory delete. + +### Expiry Cleanup + +Some rows are inherently time-bounded: sessions, OAuth flows, invitations, embedding queue outcomes. These should have a periodic cleanup process that hard deletes rows past their expiry or retention window. Expired rows are not soft-deleted to "remember they existed"; they are removed. + +If a particular table needs longer retention for operational debugging, the retention window should be configurable and enforced by the cleanup process, not by leaving expired rows in the live table indefinitely. + +### Audit / Dead Tables + +If we ever do need to keep evidence of a deleted row, the row must move out of the live table into a separate audit or dead table. The live table only holds live state; history goes elsewhere. + +This pattern is opt-in per table. V1 does not require audit tables anywhere. If a future feature needs them (for example, compliance, billing reconciliation, or security forensics), we add a dedicated table such as `core.audit_event` or `core.dead_api_key` and write to it from the same transaction that performs the hard delete. + +### Cascading Deletes + +Forcing administrators to hand-revoke every grant and membership before deleting a principal is bad UX. Commands that delete parent objects should expose cascade behavior for their expected dependents rather than failing with a wall of FK errors. + +The conventions are: + +- `--cascade`: also delete dependent rows that would otherwise block the operation. Refers to expected, documented dependent rows for that command. +- `--force`: skip confirmation prompts. `--force` does not mean "ignore integrity"; it means "I already know what this will do." + +Commands may require `--cascade`, `--force`, or both for destructive operations. The default behavior without flags should be safe and explain what is blocking the delete. + +#### Deleting a Group + +`me group delete [--cascade]` should cascade to: + +- `core.group_member` rows where the group is the group. +- `core.tree_access` rows granted to the group. +- `core.principal_space` row for the group. +- The group's `core.principal` row. + +This matches intent: the group no longer exists, so its memberships and grants no longer exist either. + +#### Deleting an Agent + +`me agent delete [--cascade]` should cascade to: + +- API keys owned by the agent. +- Group memberships for the agent in every space. +- Direct `core.tree_access` grants to the agent in every space. +- `core.principal_space` rows for the agent. +- The agent's `core.principal` row. + +#### Removing a Principal from a Space + +`me space member remove [--cascade]` removes that principal from the named space and cascades to: + +- The principal's group memberships in that space. +- Direct tree access grants to that principal in that space. +- The principal's `core.principal_space` row for that space. + +If the principal is a user with owned agents, the cascade also removes those agents from the same space (their `principal_space` row and any space-local memberships and grants). The global agent principal and its API keys remain, because API keys and agents are global, not space-scoped. + +The user, their global agents, and their API keys are not deleted by this command. To delete the user globally, use a separate command. + +#### Deleting a Space + +`me space delete [--force]` is the most destructive operation. It removes: + +- All `core.principal_space` rows for the space. +- All `core.group_member` rows in the space. +- All `core.tree_access` rows in the space. +- All group principals scoped to the space. +- All `core.space_invitation` rows for the space. +- The space's per-schema operational data: `.memory`, `.embedding_queue`, `.version`, `.migration`, and the schema itself. +- The `core.space` row. + +This command should always require `--force` or an explicit confirmation prompt. + +### Last-Admin Safeguard + +The cascade rules above do not override the invariant that a space must always have at least one admin principal. Any cascade that would remove the last `core.principal_space.admin = true` principal in a space must fail rather than leave the space adminless. The error should name the conflicting principal so the operator can promote a replacement before retrying. + +## The API Server + +JSON-RPC over HTTPS. + +## API, Client, and MCP Boundary + +The hosted API server exposes JSON-RPC over HTTPS. JSON-RPC gives us a simple, stable, application-owned protocol that we can shape around Memory Engine's product and authorization model without inheriting MCP-specific compatibility constraints. + +Request and response schemas live in a shared Zod model package. This package is the source of truth for the JSON-RPC method contract and is shared by the API server and TypeScript client. The API server uses the schemas to validate requests and shape responses. The client package depends on the schema package and provides a thin typed wrapper around calling the hosted JSON-RPC API. + +The CLI and local MCP server both use the client package. This keeps transport and tool-specific concerns out of the API server and avoids duplicating API call logic. + +Both the schema package and client package should be published to npm. This lets external TypeScript consumers call the hosted API directly without going through the CLI or MCP layer. The client package makes it easy to use Memory Engine from TypeScript scripts with full typing and minimal boilerplate. + +JSON-RPC over HTTPS is also much easier to integrate with scripts and services in other languages than a hosted MCP server would be. Any language that can make an HTTPS request and parse JSON can call the hosted API directly. A hosted MCP server, by contrast, would require MCP transport, framing, tool registration semantics, and per-client/per-model compatibility handling, which is heavy for ad-hoc scripts and integrations in languages that do not have a Memory Engine client library. + +HTTPS does have one tradeoff worth noting. The previous version of Memory Engine used JSON-RPC over WebSockets, which allowed streaming for bulk imports, exports, and unusually large memories. JSON-RPC over HTTPS imposes request and response payload size limits, which is more constraining for those operations. We accepted that tradeoff because the operational and client complexity of WebSockets outweighed the streaming benefit for the common case. If bulk and streaming workloads become important, we can revisit by adding chunked endpoints, signed object storage uploads/downloads, or a streaming transport for specific operations rather than reverting the entire API. + +We explicitly are not making the hosted API server an MCP server in the initial design. MCP is valuable for integrating with AI agents, but it brings extra protocol overhead and model/client compatibility constraints. Different MCP clients and model providers handle optional, nullable, tuple, record, and JSON Schema details differently. For example, some clients omit optional fields, while others send explicit `null`; some model/tooling paths render nullable unions poorly or reject certain schema shapes. + +The local MCP server is therefore a stdio proxy. It exposes MCP tools to agents, handles MCP-specific schema compatibility, normalizes model/client quirks, and forwards calls to the hosted JSON-RPC API through the client package. This isolates MCP complexity in one local integration layer while keeping the hosted API clean and flexible. + +The intended dependency flow is: + +``` +shared zod models + -> API server + -> client package + -> CLI + -> local MCP server +``` + +The hosted API remains JSON-RPC. The local MCP server adapts MCP tool calls into JSON-RPC client calls. A hosted MCP server is not ruled out, but it is not part of the current implementation plan. If we add hosted MCP later, it should be an adapter over the same JSON-RPC/client boundary rather than replacing the application API. + +## Environment Vars and Global CLI Options + +- `ME_SERVER` or `--server` \- the URL of the API (dev/prod/self-host) +- `ME_API_KEY` or `--api-key` \- an api key pointing to a specific user|agent +- `ME_SPACE` or `--space` \- the slug of a space to scope commands to + +## Config Files + +We need to store session tokens somewhere. We need to store the currently selected space somewhere. + +Use keychain so long as we keep proper scoping between servers + +## Authentication Commands + +### `me login [space_id|slug|name]` + +Authenticates with the system via OAuth and creates a session token. Session token should be saved in a known file. + +After authentication, `me login` selects a current space using the following rules: + +- If the user belongs to exactly one space, auto-select it. +- If the user belongs to multiple spaces, use the space specified by the positional argument, `--space`, or `ME_SPACE`. +- If the user belongs to multiple spaces and no space was specified, show an interactive picker when stdin/stdout is a TTY. Outside a TTY, exit with an error indicating that `--space` or `ME_SPACE` is required. +- If the user belongs to no space, select none. Subsequent commands that require a space error out with guidance to create a space or accept an invitation. + +The selected space is stored alongside the session credentials and scoped per server, so future invocations resume the same space without re-prompting. + +### `me logout` + +Expires the current session token. Removes it from the file. + +### `me whoami` + +Displays info about the principal and possibly OAuth stuff. + +## Space Commands + +### `me space use ` + +### `me space create` + +Creates a new space. + +V1 does not provision any out-of-the-box tree organization. A newly created space starts with an empty tree. Space admins set up whatever layout they want. For single-player mode, most users will just start writing memories at whatever paths make sense to them. + +The creating user receives: + +- `core.principal_space.admin = true` for the new space. +- `owner` access on the root tree path, so they can grant and revoke access anywhere below it. + +Stretch goal (only if time permits): a `--template ` or `--multiuser` flag that provisions some out-of-the-box structure for collaborative spaces (for example, a `shared` branch with team grants and `private.` branches owned by individual users). This is explicitly optional for v1 and should not block shipping. + +### `me space delete` + +### `me space alter` + +### `me space list` + +### `me space invite` + +### `me space invite list` + +### `me space invite revoke` + +## User Commands + +User principals are created via OAuth login and managed primarily through identity, invitation, and space membership commands. The `me user` command surface is therefore mostly unneeded for the immediate next version. + +Standalone non-OAuth users (for example, shared service accounts, integrations, or non-human first-class accounts that authenticate only via API keys) are valuable in the longer term, but we are deferring them until after the initial release. When we add them, this section will define commands for creating, renaming, deleting, and inspecting standalone user principals. + +For now, the only intentionally supported user-management surface is: + +### `me user group list ` + +Lists the groups the named user belongs to in the current space. + +## Agent Commands + +### `me agent create ` + +Creates an agent principal owned by the current user. + +### `me agent delete ` + +Deletes an agent principal owned by the current user. + +### `me agent rename ` + +Creates an agent principal owned by the current user. + +### `me agent group list ` + +probably more commands here + +## Group Commands + +Group commands must either have ME\_SPACE or \--space specified, or they use the currently `use`d space. + +### `me group create ` + +You must be a space admin to create new groups. + +### `me group delete ` + +You must be a space admin to delete groups. + +### `me group rename ` + +You must be a space admin to alter groups. + +### `me group member add ` + +You must be a space admin or be a member of the group with the `admin` flag on the membership in order to add members. Member must be in the `core.principal_space` table for this space. Member cannot be another group. + +### `me group member remove ` + +You must be a space admin or be a member of the group with the `admin` flag on the membership in order to remove members. Member must be in the `core.principal_space` table for this space. + +### `me group member list ` + +Any user|agent in `core.principal_space` for the space may list members of a group in the space. + +## Access Commands + +There is no `me access revoke`. The only mutation verbs are `grant` and `rm-grant`, matching the `core.tree_access` semantics: a grant is a specific `(principal, tree_path, access)` row, and removing one removes that exact row. + +### `me access grant ` + +Creates a `core.tree_access` row for the principal at the given tree path with the given access level. If an equivalent grant already exists, the command reports that fact rather than silently succeeding. + +### `me access rm-grant ` + +Deletes the specific grant row identified by `(principal, tree_path, access)`. If no such row exists, the command errors out. This is intentional: callers must name the exact grant they intend to remove, so unexpected "missing" grants surface as errors instead of silent no-ops. + +`rm-grant` does not cascade to ancestor or descendant grants and does not affect access inherited through group membership. To reduce inherited access, remove the relevant group's grant or change the principal's group membership. + +### `me access list ` + +Lists all `core.tree_access` rows for the principal in the current space, including both direct grants and group-derived grants (clearly labelled). + +### `me access list ` + +Lists all `core.tree_access` rows in the current space whose path is an ancestor of, equal to, or a descendant of the given path, so admins can see who has access to a given subtree. + +## API Key Commands + +### `me apikey create ` + +If not specified, lists api keys for self. Otherwise, must be an agent owned by the user. + +### `me apikey list ` + +If not specified, lists api keys for self. Otherwise, must be an agent owned by the user. + +### `me apikey revoke ` + +API key must belong to the user or an agent owned by the user. + +## Memory Commands + +Can we make `memory` optional? Can the memory commands be top-level? + +### `me [memory] create` + +### `me [memory] get ` + +### `me [memory] edit ` + +### `me [memory] patch|update` + +### `me [memory] delete|rm ` + +### `me [memory] delete|rm --tree ` + +### `me [memory] tree --tree --levels` + +### `me [memory] move|mv ` + +### `me [memory] copy|cp ` + +### `me [memory] search` + +### `me [memory] import` + +### `me [memory] export` + +## MCP Server + +### Local MCP `me mcp` + +Runs a stdio MCP server locally scoped to a space and user. The MCP server proxies to the hosted API. Uses an API key via either ME\_API\_KEY or \--api-key. + +The benefit of a local MCP server is that it can import/export to/from files without reading the contents through the context window (although, this is something of a security hole). + +- me\_memory\_get +- me\_memory\_search +- me\_memory\_update|patch +- me\_memory\_delete +- me\_memory\_tree +- me\_memory\_import +- me\_memory\_export + +### Hosted MCP + +No file-related tools. More thought needs to go here. + +- me\_memory\_get +- me\_memory\_search +- me\_memory\_update|patch +- me\_memory\_delete +- me\_memory\_tree diff --git a/REDESIGN_DIFFERENCES.md b/REDESIGN_DIFFERENCES.md new file mode 100644 index 0000000..ca29e74 --- /dev/null +++ b/REDESIGN_DIFFERENCES.md @@ -0,0 +1,405 @@ +# REDESIGN.md vs. Current Implementation — Differences + +This document compares `REDESIGN.md` against the code as it actually exists on +the `multiplayer` branch. It is organized as: + +1. TL;DR of the substantive divergences +2. Architectural divergences (design intent changed) +3. Naming / shape divergences (same idea, different surface) +4. Not yet implemented (gaps vs. the redesign) +5. Implemented beyond the redesign (exists in code, absent from the doc) +6. Confirmed matches (the redesign describes reality) +7. Doc-hygiene notes for REDESIGN.md itself + +File references are `path:line` against the repo root. + +--- + +## 1. TL;DR + +### 1a. Differences + +Where the doc and the code diverge but both build the feature. The **Decision** +column records items we've adjudicated (which side to keep); `—` = not yet +decided. Items the redesign lists but the code does **not** build live in §1b. + +| # | Topic | Redesign says | Implementation does | Severity | Decision | +|---|-------|---------------|---------------------|----------|----------| +| A | Auth tables | `core.session`, `core.oauth_identity`, `core.oauth_flow` live in `core` | Separate `auth` schema (better-auth shaped): `auth.users/sessions/accounts/device_authorization/verifications`; `auth.users.id == core.principal.id` | **Major** | **Keep current** (see §2.A) | +| B | Tree provisioning / private areas | V1 provisions **no** structure; magic private paths **deferred**; creator gets `owner@root` | Reserved roots `home.` (`~` sugar) + `share` (`SHARE_NAMESPACE`); creator gets `admin` + `owner@home` + `owner@share` (**not** `owner@root`); bare create defaults to `share` | **Major** | **Keep current** (UX; see §2.B) | +| D | Access function | `core.effective_tree_access(_space_id, _principal_id)` → `returns table(tree_path, access)` | `core.build_tree_access(_member_id, _space_id)` → `returns jsonb` | Naming | **Keep current** (see §3.D) | +| E | API endpoints | A single JSON-RPC API (implied) | **Two** endpoints: `/api/v1/memory/rpc` + `/api/v1/user/rpc`, plus REST `/api/v1/auth/*` | Naming/shape | **Keep current** (see §3.E) | +| H | User group listing | `me user group list ` (the one `me user` command) | `me group mine` lists your own groups (`whoami` + `group.listForMember`); no `me user` CLI for another user (the RPC already allows it for a space admin) | Partial | **Keep current** — `me group mine` serves the common (self) case; the `me user` surface stays deferred (admins can use the RPC for others) | + +### 1b. Not implemented (gaps vs. the redesign) + +Listed in the redesign but not built. Detail in §4. + +| # | Topic | Redesign wants | Status | +|---|-------|----------------|--------| +| C | Embedding config | Per-space model/dimension, recorded in `core.space` | Not implemented (hardcoded uniform; templated DDL only) | +| G | `me memory copy`/`cp` | Listed in §"Memory Commands" | Not implemented | + +(Item **F**, the last-admin safeguard, was previously listed here — now +**implemented**; see §6. Item **H**, user group listing, moved to §1a — it's +partly built via `me group mine`.) + +The agent access-masking model (the part the doc was least confident about) **is** +implemented as designed — see §6. + +--- + +## 2. Architectural divergences + +### A. Auth lives in a separate `auth` schema, not in `core` — decision: **keep the current implementation** + +The redesign places all authentication state in `core`: `core.session`, +`core.oauth_identity`, `core.oauth_flow`. The implementation instead uses a +dedicated **`auth` schema** shaped like better-auth, and `core` contains no auth +tables at all. + +Actual `auth` schema (`packages/database/auth/migrate/incremental/`): + +- `auth.users` (`001_users.sql:5`) — `id, name, email, email_verified, image, created_at, updated_at`. **`auth.users.id == core.principal.id`** for user principals. +- `auth.accounts` (`002_accounts.sql:8`) — OAuth provider links (`provider_id` ∈ {google, github}, tokens, scope). This is the redesign's `oauth_identity`. +- `auth.sessions` (`003_sessions.sql:10`) — `token_hash` (sha256), `expires_at`, `ip_address`, `user_agent`. This is the redesign's `session`. +- `auth.device_authorization` (`004_device_authorization.sql:7`) — device-code flow state. This is the redesign's `oauth_flow`. +- `auth.verifications` (`005_verifications.sql:7`) — present for better-auth shape parity. + +Net effect: the "authorization boundary lives entirely in `core`" framing in the +redesign (§"Authorization Boundary") is true for *authorization* (principals, +grants, groups) but **not** for *authentication* — authentication is its own +schema. The redesign never mentions an `auth` schema or better-auth. + +After evaluating the two, **keep the separate `auth` schema** and update +REDESIGN.md to describe it (per §7). Reasoning: + +- **Separation of concerns is real.** Authn (sessions, OAuth secrets, device + codes, PKCE, verification, expiry sweeps) and authz (the grant graph) have + different security surfaces and change cadences. Intermingling token/secret + tables with the grant graph — which every group/membership migration touches — + is exactly what you want to avoid, and it's consistent with the design's own + clean authz boundary (the `_tree_access` seam). +- **An established auth *shape* beats greenfield SQL auth.** The redesign's + `core.oauth_flow`/`oauth_identity`/`session` are underspecified, hand-rolled + auth. Mirroring better-auth's vetted model (account linking, multi-provider, + verification, session lifecycle) is lower-risk and leaves a credible path to + adopt the library or a managed service later. +- **The shared-id pattern neutralizes the redesign's main edge.** + `auth.users.id == core.principal.id` is the standard "identity table ↔ domain + entity share a PK" pattern: no mapping table, no meaningful duplication — two + concern-specific rows under one id, written atomically by `provisionUser` + (`packages/server/provision.ts:80,89`). You keep ~all the simplicity of "one + identity" while keeping the boundary. +- **Preserves optionality.** The deliberate absence of a cross-schema FK keeps + `auth` splittable onto its own DB/service later (`packages/database/index.ts` + notes it could be "distributed across databases again"). + +Caveats (cost of this choice): + +- It follows better-auth's **shape**, not the library — `packages/auth` depends + only on `@memory.build/database` + `postgres`, with deliberate divergences + (sha256 `token_hash`; a bespoke `device_authorization`). So the win is a vetted + schema + upgrade path, not free battle-tested code. +- `auth.verifications` is a **dead table** carried for shape parity (never + written). +- The `auth.users` ⇄ `core.principal` invariant is **app-enforced only** (no DB + FK). Whether to add one is its own decision — see + `DECISIONS_FOR_REVIEW.md` → "No cross-schema FK between `core.principal` and + `auth.users`" (current call: don't, defer to user-deletion / standalone-users). + +### B. Reserved tree paths and provisioning are built, not deferred — decision: **keep the current implementation** + +This is the largest behavioral divergence. The redesign's V1 scope says (§"Private +Areas", §"me space create"): + +- "We should **defer** magic private paths, implicit deny rules, and automatic + private area behavior." +- "V1 does not provision any out-of-the-box tree organization. A newly created + space starts with an empty tree." +- "The creating user receives… `owner` access on the **root** tree path." + +The implementation instead bakes in two reserved roots and provisions them: + +- `home.` with `~` as input sugar — `HOME_NAMESPACE` and `homePrefix()` + in `packages/database/space/path.ts:29,60`. `add_principal_to_space(...)` + (`core/migrate/idempotent/006_membership.sql`) auto-grants a joining **user** + `owner` on `home.`. +- `share` — `SHARE_NAMESPACE` in `packages/database/space/path.ts:38`; a bare + `memory.create` with no `tree` defaults here. +- A space **creator** gets `admin` + `owner@home.` + `owner@share`, and + **explicitly not `owner@root`** — `packages/server/provision.ts:55` (`addSpaceCreator`). + This is the opposite of the redesign's "creator gets owner@root." + +Functionally these are still ordinary positive ltree grants (no deny rules, no +implicit subtraction), so the redesign's *non-goal* of "no negative access" is +respected. But the **convention layer the redesign deferred is shipped**, and the +creator's grant is `home`+`share` rather than `root`. Any reader of REDESIGN.md +would expect a fresh space to be empty and root-owned; it is neither. + +**Decision: keep current — implemented deliberately, for UX.** Multiplayer spaces +need a usable shared/private layout out of the box; making every new space's admin +design an access model from scratch before writing a single memory is poor +onboarding. The redesign itself calls the motivation valid and lists the +shared/private provisioning as a `me space create` **stretch goal** — we chose to +ship it. Importantly it's built the way the redesign *preferred*: ordinary +positive ltree grants over conventional `home`/`share` roots, **not** magic +private-path patterns or implicit deny rules — so the monotonic, no-deny non-goals +still hold (the access evaluator stays a plain ltree-containment check). The one +substantive thing to fold into REDESIGN.md is that the creator gets +`owner@home` + `owner@share` rather than `owner@root` (so a creator doesn't see +other members' homes; as an admin it can self-grant `owner@root` if it wants the +whole tree). + +--- + +## 3. Naming / shape divergences (same concept, different surface) + +### D. Access resolution function — decision: **keep the current implementation** + +- Redesign: `core.effective_tree_access(_space_id uuid, _principal_id uuid) + returns table(tree_path ltree, access int4)`. +- Actual: `core.build_tree_access(_member_id uuid, _space_id uuid) returns jsonb` + (`core/migrate/idempotent/003_tree_access.sql:131`). Differences: **name**, + **argument order**, takes **`_member_id`** (not `principal_id`), and returns a + **JSONB array** of `{tree_path, access}` objects rather than a SQL table. + +After evaluating the two, the current implementation is **better or equal on +every axis** — so we keep the code and (per §7) treat REDESIGN.md's signature as +the weaker spec to be updated. Reasoning per axis: + +- **Parameter `_member_id` (current) > `_principal_id` (redesign) — correctness.** + Effective access is only ever computed for an *authenticating actor* (a user or + agent). A group never authenticates and isn't owner-maskable, and + `build_tree_access` only dispatches `'u'`/`'a'`. `member_id` (the u|a-only + generated column) encodes that constraint in the signature; the redesign's + `_principal_id` is looser and wrongly implies a group could be passed. +- **Argument order `(member_id, space_id)` (current) — consistency.** The whole + helper family is subject-first: `user_tree_access(_user_id, _space_id)`, + `agent_tree_access(_agent_id, _space_id)`, `member_tree_access(…, _space_id)` + (`003_tree_access.sql`). The redesign's space-first order would make the public + entry the lone exception. +- **Return type `jsonb` (current) vs `table` (redesign) — `jsonb` fits this + architecture; the table form's edge is unrealized.** The set always + round-trips through the application layer: `build_tree_access` → TS array + (`packages/engine/core/db.ts:429`), where the app uses it for the **auth gate** + (`treeAccess.length === 0` → 403, `authenticate-space.ts`) and **owner checks** + (`rpc/memory/support.ts`), then passes it **back into every space function as a + jsonb argument** (`sql.json(treeAccess)::jsonb`, + `packages/engine/space/db.ts:101`; consumed via `jsonb_to_recordset` in + `space/migrate/idempotent/001_memory.sql:33`). The only advantage of a + table-returning function — joining it directly in SQL — is never used, because + nothing computes access purely in-SQL; the app is always in the loop. And it + would save nothing: postgres.js parses either return shape into the same + `[{tree_path, access}]` JS array, and the app re-serializes with `sql.json(…)` + on the way back down regardless. Meanwhile `jsonb` matches the future sharded + pushdown with **zero refactor**, and the typed/composable layer the redesign + wanted **already exists internally** as the `*_tree_access` table functions — + `build_tree_access` is explicitly just the jsonb *bridge* on top of them. +- **Name `build_` vs `effective_` — cosmetic, the only place the redesign reads + nicer.** "Effective access" is the precise term for net/resolved permissions; + "build" describes the bridge role (its doc comment: "the bridge … returns … + the jsonb array shape"). Not worth a rename across SQL + `CLAUDE.md` + TS. If a + more semantic public name is ever wanted, `effective_tree_access` (returning + jsonb) is the one thing worth lifting from the redesign. + +Per CLAUDE.md the auth gate is "non-empty `build_tree_access`." + +### E. Two RPC endpoints, plus REST auth — decision: **keep the current implementation** + +The redesign describes "the hosted API server exposes JSON-RPC over HTTPS" as a +single surface. The implementation splits it (`packages/server/router.ts:252`): + +- `/api/v1/memory/rpc` — session **or** api-key + required `X-Me-Space`. Hosts + `memory.*`, `principal.*`, `group.*`, `grant.*`, `invite.*`. +- `/api/v1/user/rpc` — **session only** (api keys rejected here). Hosts `whoami`, + `agent.*`, `apiKey.*`, `space.*`. +- `/api/v1/auth/*` — REST device-flow endpoints (`device/code`, `device/token`, + `device/verify`, `device/approve`, `callback/:provider`). + +**Decision: keep current.** The split isn't arbitrary — it encodes two orthogonal +policies as endpoint-level invariants: + +- **Credential.** `/user/rpc` is session-only — api keys are *rejected*, not just + unprivileged (`authenticate-user.ts`). So "agents can't manage agents / keys / + spaces" is **impossible by construction**: an agent can't even reach + `agent.*` / `apiKey.*` / `space.*`. On a single endpoint this would be a + per-method "session-only" flag — a privilege-escalation bug waiting for the + first forgotten flag. +- **Scope.** `/memory/rpc` is space-scoped (required `X-Me-Space`); the + `/user/rpc` methods are inherently global/pre-space (`space.list` / `space.create` + can't sit behind a space header). + +The REST `/auth/*` routes aren't really a divergence: OAuth device flow + provider +callbacks are redirect/poll/browser-shaped and can't be JSON-RPC methods — the +redesign's "JSON-RPC over HTTPS" simply omitted that auth must be REST. The cost +is two client classes (`createMemoryClient` / `createUserClient`) sharing the same +`protocol` + transport packages — minor. The single-endpoint alternative is +simpler to *document* but weaker: it demotes a structural security boundary to a +per-method flag. So this reads as the doc under-specifying, not a considered +alternative; "JSON-RPC over HTTPS" still describes the data/control plane. + +### CLI verb renames (vs. the redesign's command list) + +All same-intent, different spelling: + +- `me space alter` → **`me space rename`** (`commands/space.ts:212`). +- `me agent group list` → **`me agent groups`** (`commands/agent.ts:161`). +- `me apikey revoke` → **`me apikey delete`** (aliases `rm`, `revoke`) (+ a + `me apikey get`) (`commands/apikey.ts`). Canonical verb is `delete` (aligning + with the "no soft delete / hard delete" stance — revoke ≡ delete); `revoke` is + kept as an alias for familiarity. +- `me group member add/remove/list` → **`me group add` / `remove`(`rm-member`) / + `members`** (`commands/group.ts`). + +--- + +## 4. Not yet implemented (gaps vs. the redesign) + +- **C. Per-space embedding config + placement metadata.** The redesign (§"Space") + wants the embedding model/dimension per-space, templated into the DDL, and + recorded in `core.space`; the space record should also track placement (the + shard). Built only partway: the DDL **is** templated + (`embedding halfvec({{embedding_dimensions}})`, + `space/migrate/incremental/001_memory.sql:10`), but the value is **hardcoded to + 1536 / `text-embedding-3-small` for every space** (`packages/server/config.ts:8`, + `packages/server/index.ts:212`). `core.space` records neither model/dimension + (it carries a TODO: `-- we likely need columns for embedding provider, model, + dimensions`, `core/migrate/incremental/001_space.sql:9`) nor a shard/placement + column. Consistent with the "no sharding in v1" non-goal, but the per-space + hooks the redesign called for aren't there yet. +- **G. `me memory copy` / `cp`.** Listed in §"Memory Commands"; `move`/`mv` + exists, `copy`/`cp` does not (`commands/memory.ts`). The MCP server likewise has + `me_memory_mv` but no copy tool. +- **Verified-email enforcement on invite acceptance.** The redesign (§`core.space_invitation`) + wants acceptance to require an OAuth-verified email matching the invitation + ("possession of an invite link alone should not be sufficient"). Invitations + are implemented (`invite create/list/revoke`, `redeem_space_invitations`), but + the verified-email match requirement should be confirmed against + `009_invitation.sql` / redemption before relying on it. + +--- + +## 5. Implemented beyond the redesign (in code, not in the doc) + +- **`me agent add `** — adds one of your global agents to the active space + (`commands/agent.ts:134`). The redesign treats agent→space admission as implied + by `principal_space` but never gives it a command. (Agents are global; they must + be admitted to a space before they can hold a key/grants there.) +- **Client/version gating.** `X-Client-Version` header check returns HTTP 426 + "Upgrade Required" below `MIN_CLIENT_VERSION` (`server/middleware/client-version.ts`), + and the migrator rejects an app older than the DB version + (`database/migrate/kit.ts:334`). The redesign mentions version *tables* for + compatibility but not a client-version handshake. +- **`core.space.language`** column for the BM25 text-search config + (`001_space.sql:8`) — per-space text language, not mentioned in the redesign. +- **Extra env vars:** `ME_SESSION_TOKEN` and `ME_NO_KEYCHAIN` (`packages/cli/credentials.ts`, + `packages/cli/keychain.ts`) beyond the doc's `ME_SERVER`/`ME_API_KEY`/`ME_SPACE`. +- **CLI config split + keychain.** Implemented as `~/.config/me/config.yaml` + (non-secret, per-server `active_space`) + `credentials.yaml` (0600 secret + fallback) + OS keychain (macOS `security`, Linux `secret-tool`). The redesign + only says "store session tokens somewhere… use keychain"; the concrete split is + an elaboration. +- **Extra CLI surface:** `me serve` (web UI), `me pack`, `me claude` / `me codex` + / `me gemini` / `me opencode` (install + import), `me completions`, `me version`, + `me upgrade`. The doc only asks about a few of these. +- **MCP tool set is broader than the doc's list.** Actual tools + (`packages/cli/mcp/server.ts`): `me_memory_create`, `me_memory_get`, + `me_memory_search`, `me_memory_update`, `me_memory_delete`, + `me_memory_delete_tree`, `me_memory_mv`, `me_memory_tree`, `me_memory_import`, + `me_memory_export`. The redesign's local list omits `create`, `mv`, and + `delete_tree`, and writes `update|patch` (there is a single `update`, no + `patch`). +- **Transitive group admin (Model 2)** is implemented (`is_principal_space_admin`, + `member_groups` in `003_tree_access.sql` / `001_principal_space.sql`): a user in + an admin-flagged group inherits space admin; agents are excluded. The redesign + discusses this but the concrete enforcement is in code. + +--- + +## 6. Confirmed matches (the redesign describes reality) + +These are worth recording because they are the parts the redesign was least sure +about or most opinionated on, and the code honors them: + +- **Agent access runtime capping.** The "stronger interpretation" the redesign + preferred — an agent's effective access is its configured access (direct + + group-derived) **intersected with the owner's current effective access at + runtime** — is implemented in `agent_tree_access` within + `core/migrate/idempotent/003_tree_access.sql:54` (overlap → more-specific path, + `least(access)`, then reduce redundant descendants). The doc's "vibe-coded + sketch" became real SQL. The V1 rules hold: agents can't be space admins + (`is_principal_space_admin` excludes `kind='a'`), can't be group admins + (`member_groups` zeroes the admin flag for agents), may be group members, and + inherited admin from an admin-flagged group does not make an agent an admin. +- **No soft deletes.** No `deleted_at`/`archived_at`/`is_deleted`/`active` + anywhere; hard deletes via FK `on delete cascade` plus explicit cascade + functions (`remove_principal_from_space`, etc.). Matches §"Deletion and + Cascading". +- **Last-admin safeguard** (was item F — now implemented, and stronger than the + redesign's wording). A space can't be left without an **effective** admin — a + *user* who is a direct admin **or** a member of an admin-flagged group. The + `enforce_last_admin` trigger fn (`core/migrate/idempotent/001_principal_space.sql`) + fires on `core.principal_space` (admin removed/demoted) **and** `core.group_member` + (member removed from an admin group), and rejects any change leaving zero + effective admins, raising SQLSTATE `ME001` → `LAST_ADMIN` (`rpc/core-error.ts`). + It covers every path uniformly — `principal.remove`, the `add_principal_to_space` + demote, removing the last member of the sole admin group, and FK cascades from + `delete_principal` (deleting an admin user/group) — and exempts whole-space + teardown (`delete_space` drops the `space` row first, so the trigger sees it gone + and skips; the `select … for update` also serializes concurrent removals). + Checking the *effective* set (not just the `principal_space.admin` flag) closes + the brick where a space's sole admin is an **empty admin group** — an + unrecoverable, ungoverned state the flag-only check would have allowed. Matches + (exceeds) §"Last-Admin Safeguard". +- **Principal model.** `kind ∈ {u,a,g}`, generated `member_id` (u|a), generated + `user_id`/`agent_id`/`group_id`, agent `owner_id → principal(user_id)`, name + scoping (users global, agents per-owner, groups per-space). Matches §`core.principal`. +- **`tree_access` ladder** read=1 / write=2 / owner=3, applies to path + all + descendants, monotonic, no deny table. Matches §`core.tree_access`. CLI verbs + `grant` / `rm-grant` (no `revoke`) match the doc exactly (`commands/access.ts`). +- **Space schema tables** `.memory`, `.embedding_queue`, + `.version`, `.migration`; `embedding_version` on memory; version-aware + queue with `vt` / `outcome` / `attempts` / `last_error`; temporal `[a,a]` vs + `[start,end)` convention enforced by check constraint. Matches §"Space" / §`.*`. +- **Transport & boundaries.** JSON-RPC over HTTP (not WebSockets), no hosted MCP + server, local **stdio** MCP proxy that forwards through the client package using + `ME_API_KEY`; `@memory.build/protocol` (Zod) is the contract source of truth and + both `protocol` and `client` are published (`private: false`). Dependency flow + protocol → server → client → CLI → MCP holds. Matches §"API, Client, and MCP + Boundary". +- **`me memory` is optional / top-level.** Both `me memory ` and `me ` + (e.g. `me search`, `me create`) work — answers the doc's open question "Can the + memory commands be top-level?" with "yes." +- **Auth specifics** the doc implied: GitHub/Google OAuth, device-code flow, + session + api-key secrets are **sha256** (not argon2), shared `extractBearerToken` + helper. Matches. + +--- + +## 7. Doc-hygiene notes for REDESIGN.md + +If REDESIGN.md is meant to track reality, these lines are now stale: + +- Move `core.session` / `core.oauth_identity` / `core.oauth_flow` out of the + `core` section and document the separate `auth` schema (better-auth shape) — + decided to keep the separate schema, see §2.A. +- §"Private Areas" / §"me space create": the `home`/`share`/`~` convention and the + creator's `owner@home`+`owner@share` (not `owner@root`) grant are shipped, not + deferred — decided to keep (implemented for UX, see §2.B); promote the + shared/private provisioning from stretch goal to V1 scope. +- §"Space": `core.space` does not yet carry embedding or placement columns + (there's a TODO); embedding is hardcoded uniform. Either implement or downgrade + the prose to "future." +- §"Authorization Boundary": update `effective_tree_access(_space_id, + _principal_id) returns table` to the real `build_tree_access(_member_id, + _space_id) returns jsonb` — the current signature is the preferred one (see + §3.D for the per-axis reasoning); optionally adopt the name `effective_…` while + keeping the `_member_id` argument and jsonb return. +- Command list: `space alter`→`rename`, `apikey revoke`→`delete`, `agent group + list`→`agent groups`; add `me agent add`; `me memory copy` and `me user group + list` are unbuilt; the single-API framing should mention the two RPC endpoints + + `/api/v1/auth/*` (decided to keep the split — see §3.E). +- The **last-admin safeguard** is now implemented (SQLSTATE `ME001` / + `LAST_ADMIN`, the `enforce_last_admin` trigger) — keep §"Last-Admin Safeguard" + and note the trigger-based enforcement + the admin-group-with-no-members edge. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..72dbba0 --- /dev/null +++ b/TODO.md @@ -0,0 +1,250 @@ +# TODO + +Tracked follow-up work. For the in-progress Bun.SQL → postgres.js driver swap, +see `CLAUDE.md` → "Database driver migration" (status + per-file recipe). + +## CLI: `me apikey create ` when the agent isn't in the space + +`apiKey.create` requires the agent already be a member of the active space +(`requireOwnedAgent` → NOT_FOUND otherwise). `me apikey create ` surfaces +that raw NOT_FOUND, so the user has to know to run `me agent add ` first. + +- [x] Done (2026-06-05) — `me apikey create` now maps the server `NOT_FOUND` to + an actionable message ("Agent '' isn't in this space yet — run + 'me agent add ' first"). Added a reusable `isAppErrorCode` helper to + util.ts. (Auto-adding the agent was considered but skipped — silently + changing space membership as a side effect of minting a key is surprising.) +- [x] Superseded (2026-06-05) — by the "API keys are global" change. `apiKey.*` + moved to the user RPC and minting now needs only agent ownership (no active + space, no `me agent add` prerequisite), so the NOT_FOUND mapping above was + removed from `me apikey create`. (`isAppErrorCode` stays in util.ts.) + +## Space invitations: email delivery + expiry (deferred from v1) + +Space invitations are built (INV-1..5): an email-keyed `core.space_invitation`, +redeemed at verified login (or applied immediately for an already-registered +user), with `me space invite [--admin] [--share …]` / `invite list` / +`invite revoke`. Two pieces were intentionally deferred from v1: + +- [ ] Email/link delivery — v1 sends no notification; a pending invite is only + acted on when the invitee next signs in (auto-redeemed) or is told out of + band. Send an invitation email with a sign-in link. +- [ ] Invite expiry — `space_invitation` has no expiry; a pending invite lives + until redeemed or revoked. Add an expiry column (+ a sweep) if wanted. + +## Worker: call space SQL functions instead of raw queries + +The embedding worker's write-back in `packages/worker/process.ts` still issues +raw `UPDATE embedding_queue …` / `UPDATE memory SET embedding …` statements, +against the "logic in DB functions, TS calls functions" principle the rest of +the cutover follows. + +- [x] Done (2026-06-05) — added `complete_embedding(queue_id, memory_id, + embedding_version, embedding)` (version-guarded memory write + atomic + `completed`/`cancelled` queue finalization, returns the outcome), + `fail_embedding(queue_id, error)` (record transient error, leave outcome + NULL), and `release_embedding(queue_id)` (attempt-undo for rate limits) to + `space/migrate/idempotent/003_embedding_queue.sql`. `process.ts` calls them + via `tx.unsafe` (like the existing claim/prune); it now holds zero inline + DML. Existing process integration tests regression-guard the behavior; new + tests cover the functions directly (incl. the write-back-time version + mismatch → `cancelled`, and fail/release no-op once terminal). + +## Worker: batch the embedding write-back (fewer DB round-trips) + +`processBatch`'s write-back loops over each claimed row in its own `sql.begin` +transaction, calling `complete_embedding` / `fail_embedding` one row at a time — +so a batch of N claimed rows costs ~N transactions / round-trips on the +write-back side (the claim is already a single call). Over a remote DB that +per-row latency dominates a batch. + +- [ ] Make the write-back set-based: a batch SQL function (e.g. + `complete_embeddings(_rows jsonb)` taking + `[{queue_id, memory_id, embedding_version, embedding}]`, doing the + version-guarded memory updates + queue finalization in one statement-pair + and returning per-row outcomes), called once per batch instead of per row. + Do the same for the transient-fail and rate-limit `release` paths (one + call covering the whole batch). Keep the version-guard + `completed`/`cancelled` semantics (data-driven, not errors). +- [ ] Decide error isolation: one transaction for the batch is simplest but a + single poison row (e.g. a malformed vector) would fail the whole batch. + Consider a per-row fallback when the batched call errors, so one bad row + doesn't block its siblings (which today each commit independently). + +## Decision: `core` and `space` are one package (`@memory.build/database`) + +Resolved (2026-06): merged `packages/core` + `packages/space` into a single +`@memory.build/database`, kept as separate `core/` and `space/` modules. The team +co-locates the control plane and data plane in one database/deployment, and pgdog +sharding/distribution of spaces is off the table for now. The per-slug schema model +and the `set local pgdog.shard` code stay in the `space/` module, so re-splitting +later is cheap if distribution returns. + +- [x] Done (2026-06-05) — Biome `noRestrictedImports` overrides in `biome.json` + forbid `packages/database/space/**` from importing core (`**/core`, + `**/core/**`, or the package root `@memory.build/database` which re-exports + it) and symmetrically forbid core from importing space, each with an + explanatory message. Verified it fires on a cross-import in both + directions. + +## Consolidate duplicated test-utils + +- [x] Done — the generic, driver-level helpers (`resolveTestDatabaseUrl`, `connect`, + `expectReject`, and schema introspection: `schemaExists`, `tableExists`, + `listTables`, `listFunctions`, `listTriggers`, `extensionInstalled`, + `columnType`, `listIndexes`, `getIndexDef`, `getIndexReloptions`, + `appliedMigrations`, `getSchemaVersion`) now live once in + `packages/database/migrate/test-utils.ts`. `core/migrate/test-utils.ts` and + `space/migrate/test-utils.ts` `export *` from it and add only their + provisioning (`TestCore`/`withTestCore`/`randomCoreSchema`; + `TestSpace`/`withTestSpace`/`randomSlug`/`testSchema`), so the test files are + unchanged. Verified: typecheck/lint clean, unit + ghost db suites pass. +- [ ] `engine`/`accounts` still carry their own `tableExists`/`schemaExists` copies. + They're separate packages, so sharing with them needs a dev-only package + (or fold them in during the postgres.js rollout). + +## Harden `search_path` on SQL functions (+ maybe move extensions off `public`) + +All the schema SQL functions currently set +`search_path to pg_catalog, {{schema}}, public, pg_temp`. They can be tightened +(every object reference is already schema-qualified, and none create temp +objects): + +- [ ] Auth (and likely core/space) data functions → `pg_catalog, public`: drop + `{{schema}}` (nothing unqualified) and `pg_temp` (so a temp object can never + shadow). The SECURITY DEFINER `update_updated_at` trigger fn can go all the + way to `search_path = ''` — it only uses `pg_catalog.now()` + the NEW record. +- [ ] `public` only has to stay because of `citext` (the `users.email` column + + the `_email::citext` compare; its `=` operator can't be cleanly + schema-qualified in `a = b`). Consider installing extensions + (`citext`, and engine's `ltree`/`vector`/`pg_textsearch`) into a dedicated + `extensions` schema instead of `public`; then the path becomes + `pg_catalog, extensions` and `public` drops out entirely. This touches the + migrate bootstrap (`ensureExtension` installs `with schema public`) — decide + holistically before changing the function `search_path`s. + +## User-facing tree-path convention (lenient input → canonical ltree) + +Tree paths are stored as ltree (dot-separated; the root is the empty path, +exported as `core.ROOT_PATH`). Internally everything stays ltree-native (the +store layer, the SQL functions, `provisionUser`). At the **user-facing boundary** +(RPC handlers, CLI, MCP) we want lenient input normalized once to that canonical +form — the right convention is what's natural for users, not what ltree accepts. + +- [x] Done — `packages/database/space/path.ts` exports `normalizeTreePath` + (strict, concrete paths), `normalizeTreeFilter` (lenient, lquery/ltxtquery + passes through), `homePrefix`, and `denormalizeTreePath`. Wired **server-side** + in the space RPC handlers (`rpc/memory/memory.ts` + `grant.ts` via + `inputTreePath`/`inputTreeFilter`/`displayTreePath` in `support.ts`), which + is the single chokepoint for CLI + MCP + web (they send raw input; the + server normalizes). Includes `~` home directories: a leading `~` expands to + `home.` (the authenticated caller), reverse- + mapped to `~.…` on output for the caller's own home. Labels allow + `[A-Za-z0-9_-]` (PG16+ hyphens). Malformed input → `VALIDATION_ERROR`. +- [x] Output form decided (2026-06-05): **dot is the canonical separator + everywhere** (`work.projects`, and home as `~.blah`); slashes are accepted + on input but never emitted. Docs already use dots, so no doc change needed. +- [ ] Reverse-mapping only covers the **caller's own** home (other principals' + homes show the raw `home..…`). Fine for now; revisit if listing + other members' home paths becomes common (would need a uuid→`~user` or + handle lookup). + +## Consolidate the migration runner logic + +- [x] Done — the shared machinery lives in `packages/database/migrate/kit.ts`: + advisory locking (`advisoryLockKey`, `acquireAdvisoryLock`), session timeouts + (`applySessionTimeouts`), extension / Postgres-version preconditions + (`ensurePostgresVersion`, `ensureExtension`, `ensureRequiredExtensions`, + `REQUIRED_EXTENSIONS`), schema checks (`doesSchemaExist`, + `assertSchemaOwnership`, `isValidSchemaName`), `{{…}}` `template` rendering, + SQL-file execution with error-location logging (`executeSqlFile`), and the + incremental-once / idempotent-always `runSchemaMigrations` runner — all + parameterized by a `label` (drives span/attribute/log names) + `dir`. + `migrateCore` / `migrateSpace` / `bootstrapSpaceDatabase` are now thin + orchestrators holding only their schema-specific bits (options, SQL lists, + slug/shard handling, template vars). Verified: typecheck/lint clean, + unit + ghost db suites pass. (bootstrap's lock moved from a hardcoded + single-key id to the shared two-key derived lock.) + +## Refresh `docs/` for the principal / space model + +The `docs/` pages (getting-started, concepts, access-control, `cli/*`, `mcp/*`) +still describe the retired engine / org / role / accounts model. `CLAUDE.md` is +now the authoritative summary of the current design. + +- [ ] Rewrite the docs to the new model: principals (user | agent | group), + spaces (immutable slug / renamable name, `X-Me-Space`), 3-level + tree-access grants, session-vs-api-key auth, and the + `me space/group/access/agent/apikey/memory` command surface. Update + `docs/cli/*` (drop engine/org/invitation/user/owner/role; add + space/group/access/agent) and `docs/mcp/*`. The docs-site renders these. +- [ ] **API keys are global** (changed 2026-06-05) — document the model when the + docs are rewritten: + - Format is `me..` (no space slug). One key works in + **any** space the owning agent has been admitted to; the space is chosen + per-request via `X-Me-Space`. + - `apiKey.*` lives on the **user RPC** (session-only) — TS consumers call + `createUserClient().apiKey.*`, not the memory client. `me apikey` needs + no active space. + - Minting a key requires only **agent ownership** — not space membership. + A freshly-minted key is inert until the agent is added to a space + (`me agent add`) and granted access there. + - **Owner masking caveat**: an agent's effective access is capped by its + owner's current access, so a key authenticates into a space only if the + *owner* also has access there (document this, it surprises people). + - `me install`, `me mcp`, and the Claude plugin now require an + explicit space (`--space` / `ME_SPACE` / active space; the plugin has a + new required `space` config) since the key no longer carries it. + - Migration note for any existing setups: old 4-part `me.....` keys + are invalid and must be re-minted. + +## Deploy: env rename coordination (Phase 5) + +Phase 5 renamed the server's required DB env var and removed the accounts DB. +The server throws at boot if `DATABASE_URL` is unset (no back-compat fallback, +by design). + +- [ ] With the `multiplayer` → `main` deploy: set `DATABASE_URL` (was + `ENGINE_DATABASE_URL`) in every environment; `ACCOUNTS_DATABASE_URL` is no + longer read; pool tunables renamed `ENGINE_POOL_*` → `DB_POOL_*` and + `WORKER_ENGINE_*` → `WORKER_*` / `WORKER_DB_*`. The old `accounts` schema + (and any old RLS `me_` engine schemas) are now orphaned — no + migration drops them; remove manually if a non-fresh DB ever runs this. + +## `me serve` web UI: finish + verify + +`packages/web` is the React UI for `me serve`. Its bundled assets +(`packages/cli/serve/web-assets.generated.ts`) are an empty placeholder, +`packages/web` is excluded from the root typecheck (and has pre-existing Monaco +typecheck errors), and there is no serve → `/api/v1/memory/rpc` integration +test. The `/rpc` proxy + web client are migrated (session token + `X-Me-Space`) +but unproven at runtime. + +- [ ] Build/bundle the web UI (`scripts/bundle-web-assets.ts`), fix the Monaco + typecheck errors, and add an end-to-end check that the `me serve` `/rpc` + proxy reaches the memory endpoint. Decide whether `packages/web` should be + in CI / the root typecheck. + +## Test coverage: every search mode + parameter actually takes effect + +`memory.search`'s `orderBy` was silently ignored for ~the whole pre-release — the +param and the `me search --order-by` flag parsed fine but never reached the SQL +(fixed in `e9a6eec`). Nothing caught it because the search tests only assert +"ranked search returns a match," not "each parameter changes the result." Other +params could be quietly broken the same way. + +- [ ] Add behavior tests (space-store integration level, where the SQL actually + runs) asserting each search **parameter changes the output**, not just that a + query returns rows. Cover the matrix: + - **modes**: bm25-only, vector-only, hybrid (RRF), unranked filter-only. + - **params**: `orderBy` asc/desc (incl. the default), `limit`, + `candidateLimit`, `semanticThreshold`/`maxVecDist`, `weights` + (fulltext/semantic), `tree` (ltree/lquery/ltxtquery), `meta` contains, + `grep`, temporal filters (within/overlaps/contains → before/after). + Each test should construct inputs where the param demonstrably + reorders/filters results (desc vs asc returns the reverse; a tighter + threshold drops a known row; etc.). +- [ ] Add a thin handler/wire-level check (`call("memory.search", …)`) that the + protocol params map onto the store options — so the wire→handler→store + plumbing can't silently drop a field again (the exact gap that hid the + `orderBy` bug: the handler discarded the param before the store ever saw it). diff --git a/biome.json b/biome.json index d5b5d9b..dcbe9df 100644 --- a/biome.json +++ b/biome.json @@ -56,6 +56,54 @@ } } } + }, + { + "includes": ["packages/database/space/**"], + "linter": { + "rules": { + "style": { + "noRestrictedImports": { + "level": "error", + "options": { + "patterns": [ + { + "group": [ + "**/core", + "**/core/**", + "@memory.build/database" + ], + "message": "packages/database/space must not import core/ (nor the package root, which re-exports it). Keep the data plane free of the control plane so the space/core split stays cheap to undo." + } + ] + } + } + } + } + } + }, + { + "includes": ["packages/database/core/**"], + "linter": { + "rules": { + "style": { + "noRestrictedImports": { + "level": "error", + "options": { + "patterns": [ + { + "group": [ + "**/space", + "**/space/**", + "@memory.build/database" + ], + "message": "packages/database/core must not import space/ (nor the package root, which re-exports it). Keep the control plane free of the data plane so the space/core split stays cheap to undo." + } + ] + } + } + } + } + } } ], "css": { diff --git a/bun.lock b/bun.lock index 10ba56f..ac7afbf 100644 --- a/bun.lock +++ b/bun.lock @@ -12,16 +12,27 @@ "typescript": "^6.0.2", }, }, - "packages/accounts": { - "name": "@memory.build/accounts", - "version": "0.2.0", + "e2e": { + "name": "@memory.build/e2e", + "version": "0.2.5", "dependencies": { - "@pydantic/logfire-node": "^0.13.1", + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", + "@memory.build/embedding": "workspace:*", + "postgres": "^3.4.9", + }, + }, + "packages/auth": { + "name": "@memory.build/auth", + "version": "0.0.0", + "dependencies": { + "@memory.build/database": "workspace:*", + "postgres": "^3.4.9", }, }, "packages/cli": { "name": "@memory.build/cli", - "version": "0.2.5", + "version": "0.2.6", "bin": { "me": "./index.ts", }, @@ -38,7 +49,7 @@ }, "packages/client": { "name": "@memory.build/client", - "version": "0.2.5", + "version": "0.2.6", "dependencies": { "@memory.build/protocol": "workspace:*", }, @@ -49,6 +60,15 @@ "typescript", ], }, + "packages/database": { + "name": "@memory.build/database", + "version": "0.2.5", + "dependencies": { + "@memory.build/protocol": "workspace:*", + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", + }, + }, "packages/docs-site": { "name": "@memory.build/docs-site", "version": "0.0.0", @@ -83,7 +103,7 @@ }, "packages/embedding": { "name": "@memory.build/embedding", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@ai-sdk/openai": "^3.0.0", "@pydantic/logfire-node": "^0.13.1", @@ -92,14 +112,16 @@ }, "packages/engine": { "name": "@memory.build/engine", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { + "@memory.build/database": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", }, }, "packages/protocol": { "name": "@memory.build/protocol", - "version": "0.2.5", + "version": "0.2.6", "dependencies": { "zod": "^4.0.0", }, @@ -112,14 +134,16 @@ }, "packages/server": { "name": "memory-engine-server", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { - "@memory.build/accounts": "workspace:*", + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", "@memory.build/engine": "workspace:*", "@memory.build/protocol": "workspace:*", "@memory.build/worker": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", "zod": "^4.0.0", }, }, @@ -153,17 +177,20 @@ }, "packages/worker": { "name": "@memory.build/worker", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", - "@memory.build/engine": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", }, }, "scripts": { "name": "scripts", "dependencies": { + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", + "postgres": "^3.4.9", "yaml": "^2.7.0", }, }, @@ -363,14 +390,18 @@ "@js-sdsl/ordered-map": ["@js-sdsl/ordered-map@4.4.2", "", {}, "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="], - "@memory.build/accounts": ["@memory.build/accounts@workspace:packages/accounts"], + "@memory.build/auth": ["@memory.build/auth@workspace:packages/auth"], "@memory.build/cli": ["@memory.build/cli@workspace:packages/cli"], "@memory.build/client": ["@memory.build/client@workspace:packages/client"], + "@memory.build/database": ["@memory.build/database@workspace:packages/database"], + "@memory.build/docs-site": ["@memory.build/docs-site@workspace:packages/docs-site"], + "@memory.build/e2e": ["@memory.build/e2e@workspace:e2e"], + "@memory.build/embedding": ["@memory.build/embedding@workspace:packages/embedding"], "@memory.build/engine": ["@memory.build/engine@workspace:packages/engine"], @@ -1205,6 +1236,8 @@ "postcss": ["postcss@8.5.10", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ=="], + "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="], + "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="], diff --git a/docs/access-control.md b/docs/access-control.md index 93f40ee..3dbaa02 100644 --- a/docs/access-control.md +++ b/docs/access-control.md @@ -1,142 +1,116 @@ # Access Control -Memory Engine uses tree-grant RBAC (Role-Based Access Control) enforced at the database level with PostgreSQL Row-Level Security. +Memory Engine organizes knowledge into **spaces**. Access within a space is granted on **tree paths**, not by role. There is no Row-Level Security — the server computes a caller's effective access and passes it into every database call. -## Users +## Principals -A user is a principal within an engine. Users can: +A **principal** is anything that can be granted access. There are three kinds: -- Own memories -- Receive grants to access tree paths -- Authenticate via API keys -- Belong to roles +| Kind | What it is | +|------|------------| +| **user** (`u`) | A human, authenticated by a session token (OAuth via GitHub or Google). | +| **agent** (`a`) | A service account owned by a user, authenticated by an API key. | +| **group** (`g`) | A named bundle of users and agents. | -Create a user: +A **member** is the user/agent sense only — the things that can be put into a group or hold an API key. Group membership is transitive: a member of a group gains the group's space membership, its admin (if the group is an admin), and all of its tree-access grants. -```bash -me user create alice -``` +## Spaces -Users with the `--superuser` flag bypass all access checks. Users with `--createrole` can create other users and roles. +A **space** is an isolated collection of memories with its own roster, groups, and access grants. Each space has: -Users created with `--no-login` are roles -- they cannot authenticate directly but can be granted access that members inherit. +- An immutable 12-character **slug** — also the `X-Me-Space` header value and the `me_` database schema name. +- A renamable display **name** (`me space rename` changes only this). -## Roles +A user can belong to many spaces; each memory lives in exactly one space. There are no organization, engine, or shard concepts above a space. -Roles group users together. When a grant is given to a role, all members of that role inherit the access. +## Two axes of authority -```bash -# Create a role -me role create engineering +Access splits into two independent axes: -# Add members -me role add-member engineering alice -me role add-member engineering bob +- **Structural authority** — `me space invite`, the roster (`me agent add`, `me group ...`), and invitations. This is the space **admin** flag. Admin transfers transitively through an admin group. Agents are never admins. +- **Data authority** — who can read/write/own memories at a given tree path. This is a **tree-access grant**. -# Grant access to the role (all members inherit it) -me grant create engineering work.projects read create update -``` +A space must always keep at least one *effective* admin (a user who is a direct admin or a member of an admin group). The last-admin safeguard rejects any removal or demotion that would drop it (error code `LAST_ADMIN`). -Roles are implemented as users with `canLogin: false`. This means grants work the same way for users and roles. +## Tree-access grants -## Grants +A grant attaches an access **level** to a principal at a **tree path**. Levels are additive: -Grants control what actions a user (or role) can perform on a tree path. A grant specifies: +| Level | Name | Capabilities | +|-------|------|--------------| +| 1 | **read** | Search and retrieve memories at or below the path. | +| 2 | **write** | Read + create, update, move, and delete memories. | +| 3 | **owner** | Write + manage access (grant/revoke) within the subtree. | -- **user** -- who receives the access -- **path** -- which tree path (and all descendants) -- **actions** -- what they can do: `read`, `create`, `update`, `delete` +Grants are **hierarchical**: a grant at `share.work` also covers `share.work.projects`, `share.work.projects.api`, and so on. An `owner` grant at a path delegates access-management for that whole subtree; `owner@root` (the empty path) owns the entire space. ```bash -# Grant read and create access to a tree branch -me grant create alice work.projects read create - -# Grant full access -me grant create bob work read create update delete +# Grant read access to a subtree +me access grant alice@example.com share.work r -# Check access -me grant check alice work.projects.api read -``` +# Grant write access +me access grant bob@example.com share.work.backend w -Grants are hierarchical -- a grant on `work` covers `work.projects`, `work.projects.api`, etc. +# Grant ownership of a subtree (lets the grantee manage access below it) +me access grant team-leads share.work o -### Actions +# List grants in the active space (optionally scope to one principal or path) +me access list +me access list alice@example.com +me access list --path share.work -| Action | Description | -|--------|-------------| -| `read` | Search and retrieve memories | -| `create` | Create new memories | -| `update` | Update existing memories | -| `delete` | Delete memories | - -Grant management and ownership are controlled separately: grants with `--with-grant-option` let a grantee re-grant their access, and ownership (`me owner set`) gives a user full admin access to a tree path. Superuser bootstrap is handled via the `superuser` flag on the user row, not via a grant action. - -### Grant option - -When creating a grant with `--with-grant-option`, the grantee can re-grant that same access to others: - -```bash -me grant create alice work.projects read create --with-grant-option +# Remove a grant +me access rm-grant bob@example.com share.work.backend ``` -Alice can now grant `read` and `create` on `work.projects` to other users. +The level argument accepts `r` (read), `w` (write), or `o` (owner). -## Ownership +## Reserved tree roots -Each tree path can have at most one owner. The owner has implicit admin access to that path and all descendants. +Every space has two conventional roots: -```bash -# Set owner -me owner set work.projects.api alice - -# Check owner -me owner get work.projects.api - -# List all ownership records -me owner list -``` - -Ownership is distinct from grants: +- **`share`** — the shared root. Memories everyone in the space should see go here. This is where the file importers default a tree-less record, and where `me memory create` / `me_memory_create` callers usually place memories. +- **`home.`** — a per-member private root. The input shortcut **`~`** expands to your own home, so `~.notes` means `home..notes` and displays back as `~.notes`. -- **Grants** are explicit, cumulative, and can be given to multiple users. -- **Ownership** is unique per path and provides automatic admin access. +`.` is the canonical path separator (`/` is also accepted on input and normalized). Labels must match `[A-Za-z0-9_-]`. -## How it works +### Default grants -Access control is enforced by PostgreSQL Row-Level Security (RLS) policies on the `me.memory` table. When a user authenticates with an API key, the database session is configured with their identity. Every query automatically checks whether the user has the required grant for the memory's tree path. +- A space **creator** gets `admin` + `owner@home` + `owner@share` — **not** `owner@root`. So the creator sees `share` and their own `~`, but not other members' homes. Because they're an admin, they can self-grant `owner@root` if they need the whole space. +- A **user** who joins a space is granted `owner@home` (their own private root). An admin then grants whatever shared access is appropriate (often via `me space invite --share`). -This means access control cannot be bypassed by application bugs -- it's enforced by the database itself. +## How it's enforced -:::warning[The invisible wall] -When RLS denies access, you get **empty results, not errors**. A search returns fewer results silently. A `memory.get` returns "not found" even if the memory exists. This is by design (PostgreSQL RLS behavior), but it can be confusing when debugging. +There is no Row-Level Security. For each request, the server calls `build_tree_access(principalId, spaceId)`, which collapses the principal's own grants and any inherited via group membership into a single set of `(tree_path, access)` rows. That set is passed as an argument into the space's SQL functions (`search_memory`, `get_memory`, …), which filter to the paths the caller may see. -If you're seeing missing results: +The authorization gate to use a space at all is holding **at least one** grant — every member has one (`owner@home` at minimum). -1. Check the user's access with `me grant check read` -2. Verify the memory exists by checking as a superuser -3. Check that the user has grants covering the memory's tree path -4. Remember that grants are hierarchical -- a grant on `work` covers `work.projects.*` +:::warning[Quiet filtering] +Access filtering happens inside the query. If you lack `read` on a memory's tree path, a search simply returns fewer rows and `me memory get` reports "not found" — you get no error distinguishing "doesn't exist" from "not visible to you." If you're missing results you expect, check your grants with `me access list `. ::: ## Example: team setup ```bash -# Create users -me user create alice -me user create bob -me user create carol - -# Create a shared role -me role create team -me role add-member team alice -me role add-member team bob -me role add-member team carol - -# Grant the team read access to everything -me grant create team "" read - -# Grant write access to specific branches -me grant create alice work.frontend read create update -me grant create bob work.backend read create update -me grant create carol work.infra read create update delete +# Create and enter a space (you become admin + owner@home + owner@share) +me space create "Acme Engineering" + +# Invite teammates by email; --share sets their access to the shared root +me space invite alice@example.com --share write +me space invite bob@example.com --share read +me space invite lead@example.com --admin --share owner + +# Group people for shared grants +me group create backend +me group add backend alice@example.com +me group add backend bob@example.com + +# Grant the group write access to a subtree (members inherit it) +me access grant backend share.work.backend w + +# Add one of your agents to the space and give it write access to share +me agent add ci-bot +me access grant ci-bot share w ``` + +See [`me access`](cli/me-access.md), [`me space`](cli/me-space.md), [`me group`](cli/me-group.md), and [`me agent`](cli/me-agent.md) for full command references. diff --git a/docs/agents.txt b/docs/agents.txt index f95e6e0..1ab9b7e 100644 --- a/docs/agents.txt +++ b/docs/agents.txt @@ -14,9 +14,9 @@ mcp: opencode: me opencode install codex_cli: me codex install gemini_cli: me gemini install - claude_code: me claude install - claude_code_plugin: claude plugin marketplace add timescale/memory-engine && claude plugin install memory-engine@memory-engine - description: Memory engine ships as an MCP server. OpenCode, Codex CLI, Gemini CLI, and Claude Code all support MCP-only install via `me install`. Claude Code additionally supports a full plugin (hooks + slash commands + MCP) via Claude Code's plugin marketplace. + claude_code: me claude install --mcp-only + claude_code_plugin: me claude install + description: Memory engine ships as an MCP server. OpenCode, Codex CLI, Gemini CLI, and Claude Code all support MCP-only install via `me install` (Claude Code: `me claude install --mcp-only`). For Claude Code, the default `me claude install` installs the full plugin (hooks + slash commands + MCP) via Claude Code's plugin marketplace. compatible_clients: - Claude Code - Codex CLI diff --git a/docs/cli/agent-session-imports.md b/docs/cli/agent-session-imports.md index 3da6250..bb46646 100644 --- a/docs/cli/agent-session-imports.md +++ b/docs/cli/agent-session-imports.md @@ -1,16 +1,16 @@ # Agent session imports -Shared reference for the per-agent `import` subcommands: +Shared reference for the agent-session import subcommands: -- [`me claude import`](me-claude.md#me-claude-import) -- [`me codex import`](me-codex.md#me-codex-import) -- [`me opencode import`](me-opencode.md#me-opencode-import) +- `me import claude` ([`me claude import`](me-claude.md#me-claude-import) is its alias) +- `me import codex` ([`me codex import`](me-codex.md#me-codex-import) is its alias) +- `me import opencode` ([`me opencode import`](me-opencode.md#me-opencode-import) is its alias) Each source-native message becomes one memory. Re-running the same command only inserts newly-seen messages (deterministic UUIDs make re-imports idempotent). ## Shared options -All three subcommands accept the same flags (with one extra flag on `me claude import`). +All three subcommands accept the same flags (with one extra flag on the Claude importer). | Option | Description | |--------|-------------| @@ -18,7 +18,7 @@ All three subcommands accept the same flags (with one extra flag on `me claude i | `--project ` | Only import sessions whose cwd equals or is below this path. | | `--since ` | Only import sessions started at or after this ISO 8601 timestamp. | | `--until ` | Only import sessions started at or before this ISO 8601 timestamp. | -| `--tree-root ` | Tree root under which `.` nodes are placed. Default: `projects`. Must match `[a-z0-9_]+(\.[a-z0-9_]+)*`. | +| `--tree-root ` | Tree root under which `.` nodes are placed. Default: `share.projects`. Accepts ltree labels (`[A-Za-z0-9_-]`) separated by `.` or `/`, with an optional leading `~` for your home (e.g. `~.projects`). | | `--sessions-node-name ` | Per-project node name for imported agent sessions. Default: `agent_sessions`. Must match `[a-z0-9_]+`. | | `--full-transcript` | Also store reasoning, tool calls, and tool results as their own message memories (default: user + assistant text only). | | `--include-temp-cwd` | Include sessions whose cwd is a system temp directory (`/tmp`, `/private/var/folders/...`). Off by default. | @@ -40,19 +40,17 @@ Each imported message is stored under: .. ``` -For example, a Claude message from a session run in `/Users/me/dev/memory-engine` ends up under `projects.memory_engine.agent_sessions` by default. Every message from every session in a project shares that same tree node; individual sessions are distinguished by `meta.source_session_id`. +For example, a Claude message from a session run in `/Users/me/dev/memory-engine` ends up under `share.projects.memory_engine.agent_sessions` by default. Every message from every session in a project shares that same tree node; individual sessions are distinguished by `meta.source_session_id`. Project slugs come from the git repo root directory name when the cwd is inside a repo, or from `basename(cwd)` otherwise. Slug collisions (two different cwds that normalize to the same label) are resolved automatically by appending a 4-char hash suffix -- the first cwd seen gets the plain slug, subsequent ones get `slug_`. The full cwd is always preserved in `meta.source_cwd`. ## Idempotency -Each imported message gets a deterministic UUIDv7 derived from `(tool, session_id, message_id, timestamp)`. On re-import: +Each imported message gets a deterministic UUIDv7 derived from `(tool, session_id, message_id, timestamp)`. Re-imports reconcile **server-side**: every planned message is submitted through the engine's conditional upsert, which inserts new ids, rewrites in place any row whose stored `meta.importer_version` differs from the current importer's (so a version bump re-renders previously-imported messages in the same batched pass), and skips rows that are already current. There is no per-session lookup and no session-size limit — a session with tens of thousands of imported messages reconciles exactly like a small one. -1. The importer looks up each message by that id. -2. If the memory already exists and `meta.importer_version` matches, it is skipped. -3. Otherwise the memory is (re)written. +Source files are append-only for all three tools, so re-importing an in-progress session simply inserts its newly-appended messages on the next run. The live-capture hook additionally narrows each submission to the messages after the newest already-imported one (a single `limit 1` search) — purely a bandwidth optimization; correctness never depends on it. -Source files are append-only for all three tools, so re-importing an in-progress session simply inserts its newly-appended messages on the next run. +`--dry-run` reports every parsed message as a would-be insert: without submitting, there is no server classification into inserted/updated/skipped. ## Content shape diff --git a/docs/cli/me-access.md b/docs/cli/me-access.md new file mode 100644 index 0000000..e8794cc --- /dev/null +++ b/docs/cli/me-access.md @@ -0,0 +1,81 @@ +# me access + +Manage tree-access grants in the active space. + +A grant attaches an access **level** to a principal (user, agent, or group) at a **tree path**. Levels are additive and hierarchical — a grant at `share.work` also covers everything below it: + +| Level | Flag | Capabilities | +|-------|------|--------------| +| read | `r` | Search and retrieve memories at or below the path. | +| write | `w` | Read + create, update, move, and delete memories. | +| owner | `o` | Write + manage access (grant/revoke) within the subtree. | + +`owner` at the empty (root) path owns the whole space. Granting access requires `owner` on the path in question (an admin can self-grant `owner@root`). See [Access Control](../access-control.md). + +These commands authenticate with your **session** and operate on the active space. + +## Commands + +- [me access grant](#me-access-grant) -- grant or update access at a path +- [me access rm-grant](#me-access-rm-grant) -- remove a grant +- [me access list](#me-access-list) -- list grants + +--- + +## me access grant + +Grant or update a principal's access at a tree path. + +``` +me access grant +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `principal` | yes | Principal id or name (user email, agent name, or group name). | +| `path` | yes | Tree path; use an empty string `""` for the space root. | +| `level` | yes | Access level: `r` (read), `w` (write), or `o` (owner). | + +```bash +me access grant alice@example.com share.work r +me access grant backend share.work.api w +me access grant lead@example.com "" o # owner@root — whole space +``` + +--- + +## me access rm-grant + +Remove a principal's grant at a tree path. + +``` +me access rm-grant +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `principal` | yes | Principal id or name. | +| `path` | yes | Tree path of the grant to remove. | + +--- + +## me access list + +List grants in the active space, optionally scoped to one principal and/or a path subtree. Alias: `me access ls`. + +``` +me access list [principal] [--path ] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `principal` | no | Filter to a single principal (id or name). | + +| Option | Description | +|--------|-------------| +| `--path ` | Only grants at or below this tree path. | + +## See also + +- [`me group`](me-group.md) -- grant to a group so all members inherit access. +- [`me space invite`](me-space.md#me-space-invite) -- set a new member's shared-root access at invite time. diff --git a/docs/cli/me-agent.md b/docs/cli/me-agent.md new file mode 100644 index 0000000..5dfc029 --- /dev/null +++ b/docs/cli/me-agent.md @@ -0,0 +1,103 @@ +# me agent + +Manage agents. + +An **agent** is a service account you own — a non-human principal that authenticates with an API key. Agents are **global** (owned by you, names unique per user), independent of any space. Create an agent, add it to the spaces it should work in, then mint it an API key with [`me apikey`](me-apikey.md). + +These commands authenticate with your **session** (`me login`). Lifecycle commands (`create`/`list`/`rename`/`delete`) are global; `add` and `groups` operate on the active space. + +## Commands + +- [me agent list](#me-agent-list) -- list your agents +- [me agent create](#me-agent-create) -- create an agent +- [me agent rename](#me-agent-rename) -- rename an agent +- [me agent delete](#me-agent-delete) -- delete an agent +- [me agent add](#me-agent-add) -- add an agent to the active space +- [me agent groups](#me-agent-groups) -- list an agent's groups in the space + +--- + +## me agent list + +List your agents. Alias: `me agent ls`. + +``` +me agent list +``` + +--- + +## me agent create + +Create an agent (a global service account you own). + +``` +me agent create +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `name` | yes | Agent name (unique among your agents). | + +--- + +## me agent rename + +Rename an agent. + +``` +me agent rename +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | +| `new-name` | yes | New name. | + +--- + +## me agent delete + +Delete an agent. Its API keys are deleted with it. Alias: `me agent rm`. + +``` +me agent delete +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | + +--- + +## me agent add + +Add one of your agents to the active space's roster. It joins with `owner@home`; grant it shared access with [`me access`](me-access.md). + +``` +me agent add +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | + +--- + +## me agent groups + +List the groups an agent belongs to in the active space. + +``` +me agent groups +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | + +## See also + +- [`me apikey`](me-apikey.md) -- mint, list, and revoke an agent's API keys. +- [`me access`](me-access.md) -- grant the agent access to tree paths. +- [MCP Integration](../mcp-integration.md) -- run an agent against a space over MCP. diff --git a/docs/cli/me-apikey.md b/docs/cli/me-apikey.md index b49f3a3..3e9fc7d 100644 --- a/docs/cli/me-apikey.md +++ b/docs/cli/me-apikey.md @@ -2,101 +2,76 @@ Manage API keys. -API keys authenticate users to an engine. Each key is scoped to a single user and can be used for MCP server connections, CLI authentication, and direct API access. +API keys are how **agents** authenticate. Each key belongs to one of your agents and is **global** — not bound to a space. The same key works in any space the agent has been admitted to; the space comes from the `X-Me-Space` header (`--space` / `ME_SPACE`). Keys are formatted `me..`. -## Commands - -- [me apikey list](#me-apikey-list) -- list API keys for a user -- [me apikey create](#me-apikey-create) -- create a new API key -- [me apikey show](#me-apikey-show) -- show a stored API key from credentials.yaml -- [me apikey revoke](#me-apikey-revoke) -- revoke an API key -- [me apikey delete](#me-apikey-delete) -- permanently delete an API key - ---- - -## me apikey list +Humans authenticate with a session (`me login`), not an API key. These commands authenticate with your **session**. -List API keys for a user. +The CLI never persists API keys. A created key is printed **once** for you to place where the agent runs (typically via the `ME_API_KEY` environment variable). The alias `me apikey revoke` is equivalent to `me apikey delete`. -``` -me apikey list -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | +## Commands -Displays a table of API keys with ID, name, last used date, and status. +- [me apikey create](#me-apikey-create) -- mint a key for an agent +- [me apikey list](#me-apikey-list) -- list an agent's keys +- [me apikey get](#me-apikey-get) -- show key metadata +- [me apikey delete](#me-apikey-delete) -- delete (revoke) a key --- ## me apikey create -Create a new API key. +Mint a new API key for one of your agents. The raw key is shown only once — store it securely. ``` -me apikey create [name] [options] +me apikey create [name] [--expires ] ``` | Argument | Required | Description | |----------|----------|-------------| -| `user` | yes | User name or ID. | +| `agent` | yes | Agent id or name. | | `name` | no | Key name (auto-generated if omitted). | | Option | Description | |--------|-------------| | `--expires ` | Expiration timestamp (ISO 8601). | -The raw key value is displayed only once at creation time. Store it securely. - --- -## me apikey show +## me apikey list -Show the API key stored locally in `credentials.yaml` for an engine. Reads only — no network call. +List an agent's API keys (metadata only — never the secret). Alias: `me apikey ls`. ``` -me apikey show [options] +me apikey list ``` -| Option | Description | -|--------|-------------| -| `--engine ` | Engine slug to look up. Defaults to the active engine for the resolved server. | - -The active server is resolved in the usual order (`--server` flag > `ME_SERVER` env > `default_server` in `credentials.yaml`). The active engine comes from that server's `active_engine` entry; switch it with `me engine use ` or override per-call with `--engine`. - -Errors when no engine can be resolved or when the named engine has no stored API key. - -Useful for scripting: +| Argument | Required | Description | +|----------|----------|-------------| +| `agent` | yes | Agent id or name. | -```sh -export ME_API_KEY=$(me apikey show --json | jq -r .apiKey) -``` +Displays a table of keys with ID, name, last-used date, and expiry. --- -## me apikey revoke +## me apikey get -Revoke an API key. +Show metadata for a single API key. ``` -me apikey revoke +me apikey get ``` | Argument | Required | Description | |----------|----------|-------------| | `id` | yes | API key ID. | -Revokes the key (makes it inactive). The key record is retained but can no longer be used for authentication. - --- ## me apikey delete -Permanently delete an API key. +Permanently delete (revoke) an API key. There is no soft-revoke state — delete is the only way to invalidate a key. Irreversible. Aliases: `me apikey rm`, `me apikey revoke`. ``` -me apikey delete [options] +me apikey delete [-y] ``` | Argument | Required | Description | @@ -107,4 +82,7 @@ me apikey delete [options] |--------|-------------| | `-y, --yes` | Skip the confirmation prompt. | -This operation is irreversible. +## See also + +- [`me agent`](me-agent.md) -- create the agents that hold these keys and add them to spaces. +- [MCP Integration](../mcp-integration.md) -- supply a key to an MCP-connected agent via `--api-key` or `ME_API_KEY`. diff --git a/docs/cli/me-claude.md b/docs/cli/me-claude.md index cbd5b2a..50bfcf2 100644 --- a/docs/cli/me-claude.md +++ b/docs/cli/me-claude.md @@ -4,7 +4,8 @@ Claude Code integration commands. ## Commands -- [me claude install](#me-claude-install) -- register `me` as an MCP server with Claude Code (MCP-only) +- [me claude install](#me-claude-install) -- install the Memory Engine plugin for Claude Code (full plugin by default, `--mcp-only` for just the MCP server) +- [me claude init](#me-claude-init) -- one-shot setup: backfill sessions, import git history, install the post-commit hook, record the project's memory location in CLAUDE.md - [me claude hook](#me-claude-hook) -- invoked by the Claude Code plugin to capture events as memories - [me claude import](#me-claude-import) -- import Claude Code sessions from `~/.claude/projects` @@ -12,35 +13,71 @@ Claude Code integration commands. ## me claude install -Register `me` as an MCP server with Claude Code. +Install the Memory Engine plugin for Claude Code. -This is the **MCP-only** install path: it adds the `me` tools to Claude Code without installing the full Memory Engine plugin. If you want hooks (auto-capture of Claude Code events) and slash commands, install the plugin instead -- see [me claude hook](#me-claude-hook). +By default this installs the **full plugin** -- hooks (auto-capture of Claude Code events), slash commands, and the MCP tools -- by driving Claude Code's native plugin CLI for you: ``` me claude install [options] ``` +Under the hood it runs the equivalent of: + +```bash +claude plugin marketplace add timescale/memory-engine +claude plugin install memory-engine@memory-engine \ + --config server= [--config space=] [--config api_key=] +``` + +The marketplace step is idempotent (skipped if already configured), and the resolved `server` / `space` / `api_key` are passed through `--config` -- the same path as the interactive `/plugin` configure flow. After install, restart Claude Code (or run `/plugin`) to load the hooks and slash commands. + +Pass `--mcp-only` to skip the plugin and register just the `me` MCP server (no hooks, no slash commands -- the previous default behavior). + | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | -| `--server ` | Server URL to embed in the MCP config. | +| `--mcp-only` | Register only the `me` MCP server (no hooks or slash commands). | +| `--api-key ` | API key for a headless agent. Default: the plugin/MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | +| `--server ` | Server URL to embed in the config. | | `-s, --scope ` | Claude Code config scope: `local`, `user`, or `project`. Default: `user`. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +Credential handling is the same for both modes: with no `--api-key`, the plugin (and the MCP server) uses your `me login` session, resolved from the OS keychain / `~/.config/me` at runtime (so it survives re-login), and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that requires a pinned `--space`. -The `--scope` flag mirrors `claude mcp add --scope`: +The `--scope` flag mirrors `claude plugin install --scope` / `claude mcp add --scope`: -- `local` -- registration scoped to the current project on this machine only. -- `user` -- registration available to all projects for your user (default). -- `project` -- registration committed to the current project (e.g. checked into `.claude/`). +- `local` -- scoped to the current project on this machine only. +- `user` -- available to all projects for your user (default). +- `project` -- committed to the current project (e.g. checked into `.claude/`). For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). --- +## me claude init + +One-shot setup of Claude Code memory integration for the current project. + +``` +me claude init [options] +``` + +Setup is a list of independent steps. In an interactive terminal `init` presents a multiselect of all steps (each pre-checked) so you can deselect any; non-interactively it runs every step except those turned off by a `--skip-` flag. + +| Step | Skip flag | What it does | +|------|-----------|--------------| +| Install the Claude Code plugin | `--skip-plugin-install` | Runs the same install as [`me claude install`](#me-claude-install) (full plugin, `user` scope, login-session auth). Hidden when the `claude` binary isn't on PATH; when `claude plugin list` already shows the plugin, the step is replaced by a ✓ "already installed" line above the picker. | +| Import this project's Claude Code sessions | `--skip-transcript-import` | Backfills sessions recorded in this project (cwd at/under the repo root, temp-dir projects included) from `~/.claude/projects`. For a machine-wide backfill across all projects, run [`me import claude`](me-import.md#me-import-claude--codex--opencode). | +| Import git commit history | `--skip-git-import` | Imports the repo's full commit history — the same import as [`me import git`](me-import.md#me-import-git). Skipped automatically when the current directory is not inside a git repo. | +| Install a git post-commit hook | `--skip-git-hook` | Installs the managed hook from [`me import git-hook`](me-import.md#me-import-git-hook) so each commit triggers a background incremental import. Hidden outside a git repo or when a `core.hooksPath` manager owns the hook path; when the hook is already installed, the step is replaced by a ✓ "already installed" line above the picker. | +| Add a memory pointer to CLAUDE.md | `--skip-claude-md` | Upserts a managed block into the project's CLAUDE.md naming the project tree (`share.projects.`), its `agent_sessions` and `git_history` nodes, and how to search them. Idempotent — re-runs replace the block in place. | + +Re-running `init` is safe: both imports are incremental/idempotent and the CLAUDE.md block is replaced, not duplicated. + +--- + ## me claude hook -Invoked by the Claude Code plugin. Reads the event JSON from stdin, resolves config from `CLAUDE_PLUGIN_OPTION_*` env vars, and captures the event as a memory. +Invoked by the Claude Code plugin on `Stop` (each turn) and `SessionEnd`. Reads the `transcript_path` from the event JSON on stdin, resolves config from `CLAUDE_PLUGIN_OPTION_*` env vars (falling back to your `me login` session), and imports the session transcript — the same parse + write as [`me … import`](agent-session-imports.md), incremental so each call only writes messages new since the last. ``` me claude hook --event @@ -50,16 +87,18 @@ me claude hook --event |--------|-------------| | `--event ` | Hook event name (required). | -This command is not run directly -- the Claude Code plugin calls it. The plugin (which includes hooks, slash commands, and MCP) is installed via Claude Code's native flow: +This command is not run directly -- the Claude Code plugin calls it. The plugin (which includes hooks, slash commands, and MCP) is installed by [me claude install](#me-claude-install), which drives Claude Code's native plugin flow for you. You can also run that flow by hand: ```bash claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] # then, in a Claude Code session: -/plugin # select memory-engine, Configure, fill api_key/server/tree_prefix +/plugin # select memory-engine, Configure (all values optional if logged in) ``` -If you only want the MCP tools (no hooks, no slash commands), use [me claude install](#me-claude-install) instead. +Both `api_key` and `space` are optional: blank `api_key` uses your `me login` session (set it to attribute captures to a dedicated agent), and blank `space` uses your active space (`me space use`; pin it for project/shared installs). + +If you only want the MCP tools (no hooks, no slash commands), run [me claude install --mcp-only](#me-claude-install) instead. Best-effort: logs failures to stderr but always exits 0 so that a hook failure never blocks a Claude Code session. @@ -67,7 +106,7 @@ Best-effort: logs failures to stderr but always exits 0 so that a hook failure n ## me claude import -Import Claude Code sessions from `~/.claude/projects//.jsonl`. +Import Claude Code sessions from `~/.claude/projects//.jsonl`. This is an alias of [`me import claude`](me-import.md#me-import-claude--codex--opencode). ``` me claude import [options] diff --git a/docs/cli/me-codex.md b/docs/cli/me-codex.md index febc4b1..ea6eff4 100644 --- a/docs/cli/me-codex.md +++ b/docs/cli/me-codex.md @@ -19,10 +19,11 @@ me codex install [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | +| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | | `--server ` | Server URL to embed in the MCP config. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). @@ -30,7 +31,7 @@ For manual MCP client configuration, see [MCP Integration](../mcp-integration.md ## me codex import -Import Codex sessions from `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl` and `~/.codex/archived_sessions/*.jsonl`. +Import Codex sessions from `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl` and `~/.codex/archived_sessions/*.jsonl`. This is an alias of [`me import codex`](me-import.md#me-import-claude--codex--opencode). ``` me codex import [options] diff --git a/docs/cli/me-engine.md b/docs/cli/me-engine.md deleted file mode 100644 index e8db199..0000000 --- a/docs/cli/me-engine.md +++ /dev/null @@ -1,101 +0,0 @@ -# me engine - -Manage engines. - -An engine is an isolated memory database. Each engine has its own memories, users, roles, grants, and API keys. - -## Commands - -- [me engine list](#me-engine-list) -- list engines across all your organizations -- [me engine use](#me-engine-use) -- select the active engine -- [me engine create](#me-engine-create) -- create a new engine -- [me engine rename](#me-engine-rename) -- rename an engine -- [me engine delete](#me-engine-delete) -- permanently delete an engine - ---- - -## me engine list - -List engines across all your organizations. - -``` -me engine list -``` - -Displays a table of all engines you have access to, showing ID, name, slug, organization, and status. The active engine is marked. - ---- - -## me engine use - -Select the active engine. - -``` -me engine use [id-or-name] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | no | Engine ID or name. If omitted, an interactive picker is shown. | - -Switches the active engine. If no API key exists for the engine, one is created automatically. The active engine is used by all subsequent commands that interact with memories. - ---- - -## me engine create - -Create a new engine in an organization. - -``` -me engine create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | Engine name. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization ID. If omitted, an interactive picker is shown. | -| `--language ` | Text search language (default: `english`). | - ---- - -## me engine rename - -Rename an engine. - -``` -me engine rename -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | Engine ID or name. | -| `new-name` | yes | New engine name. | - -Renaming changes only the human-readable name. The engine slug -- which backs the underlying database schema (`me_`) and any stored API keys -- is randomly generated at creation time and never changes. Renaming does not invalidate the active engine selection or any API keys. - -Engine names must be unique within an organization. Renaming to a name already used by another engine in the same org fails with a `CONFLICT` error. - -Requires the `owner` or `admin` role on the organization that owns the engine. - ---- - -## me engine delete - -Permanently delete an engine and all its data. - -``` -me engine delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | Engine ID or name. | - -| Option | Description | -|--------|-------------| -| `--force` | Skip the confirmation prompt. | - -You will be asked to type the engine name to confirm unless `--force` is used. This operation is irreversible. diff --git a/docs/cli/me-gemini.md b/docs/cli/me-gemini.md index bd6a267..361990a 100644 --- a/docs/cli/me-gemini.md +++ b/docs/cli/me-gemini.md @@ -18,10 +18,11 @@ me gemini install [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | +| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | | `--server ` | Server URL to embed in the MCP config. | | `-s, --scope ` | Gemini CLI config scope: `user` or `project`. Default: `user`. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). diff --git a/docs/cli/me-grant.md b/docs/cli/me-grant.md deleted file mode 100644 index 5005b1e..0000000 --- a/docs/cli/me-grant.md +++ /dev/null @@ -1,87 +0,0 @@ -# me grant - -Manage tree grants. - -Grants control access to memories by tree path. A grant gives a user specific actions (read, create, update, delete) on a tree path and all its descendants. - -## Commands - -- [me grant create](#me-grant-create) -- grant tree access to a user -- [me grant revoke](#me-grant-revoke) -- revoke tree access -- [me grant list](#me-grant-list) -- list grants -- [me grant check](#me-grant-check) -- check if a user has access - ---- - -## me grant create - -Grant tree access to a user. - -``` -me grant create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | -| `path` | yes | Tree path to grant access to. | -| `actions...` | yes | One or more actions: `read`, `create`, `update`, `delete`. | - -| Option | Description | -|--------|-------------| -| `--with-grant-option` | Allow the grantee to re-grant this access to others. | - -### Example - -```bash -me grant create alice work.projects read create update -``` - ---- - -## me grant revoke - -Revoke tree access from a user. - -``` -me grant revoke -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | -| `path` | yes | Tree path to revoke access from. | - ---- - -## me grant list - -List grants. - -``` -me grant list [user] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | no | Filter by user name or ID. | - -Displays a table of grants with user, tree path, actions, and grant option. - ---- - -## me grant check - -Check if a user has access to a tree path. - -``` -me grant check -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User name or ID. | -| `path` | yes | Tree path. | -| `action` | yes | Action to check: `read`, `create`, `update`, `delete`. | - -Reports whether access is allowed or denied. diff --git a/docs/cli/me-group.md b/docs/cli/me-group.md new file mode 100644 index 0000000..fb23313 --- /dev/null +++ b/docs/cli/me-group.md @@ -0,0 +1,134 @@ +# me group + +Manage groups in the active space. + +A **group** is a named bundle of members (users and agents). Membership is **transitive**: a group member inherits the group's space membership, its admin flag (if the group is an admin), and all of its tree-access grants. Grant access to a group once and every member gets it. + +These commands authenticate with your **session** and operate on the active space. + +## Commands + +- [me group list](#me-group-list) -- list groups in the space +- [me group mine](#me-group-mine) -- list the groups you're in +- [me group create](#me-group-create) -- create a group +- [me group rename](#me-group-rename) -- rename a group +- [me group delete](#me-group-delete) -- delete a group +- [me group add](#me-group-add) -- add a member +- [me group remove](#me-group-remove) -- remove a member +- [me group members](#me-group-members) -- list a group's members + +--- + +## me group list + +List groups in the active space. Alias: `me group ls`. + +``` +me group list +``` + +--- + +## me group mine + +List the groups you are a member of in the active space. + +``` +me group mine +``` + +--- + +## me group create + +Create a group. + +``` +me group create +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `name` | yes | Group name. | + +--- + +## me group rename + +Rename a group. + +``` +me group rename +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | +| `new-name` | yes | New name. | + +--- + +## me group delete + +Delete a group. Alias: `me group rm`. + +``` +me group delete +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | + +--- + +## me group add + +Add a member (user or agent) to a group. + +``` +me group add [--admin] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | +| `member` | yes | User or agent id or name. | + +| Option | Description | +|--------|-------------| +| `--admin` | Make them a group admin (can manage the group's membership). | + +--- + +## me group remove + +Remove a member from a group. Alias: `me group rm-member`. + +``` +me group remove +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | +| `member` | yes | User or agent id or name. | + +--- + +## me group members + +List a group's members. + +``` +me group members +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `group` | yes | Group id or name. | + +## See also + +- [`me access`](me-access.md) -- grant a group access to a tree path. +- [Access Control](../access-control.md) -- transitive membership and the authority model. diff --git a/docs/cli/me-import.md b/docs/cli/me-import.md new file mode 100644 index 0000000..f92d60b --- /dev/null +++ b/docs/cli/me-import.md @@ -0,0 +1,164 @@ +# me import + +Get data into Memory Engine — one subcommand per source. + +## Commands + +- [me import memories](#me-import-memories) -- import memory records from files or stdin (md/yaml/json/ndjson) +- [me import claude](#me-import-claude--codex--opencode) -- import Claude Code sessions +- [me import codex](#me-import-claude--codex--opencode) -- import Codex sessions +- [me import opencode](#me-import-claude--codex--opencode) -- import OpenCode sessions +- [me import git](#me-import-git) -- import a repo's git commit history +- [me import git-hook](#me-import-git-hook) -- install a post-commit hook that keeps git history memories current + +There is no bare default: `me import ` does not parse — use `me import memories `. + +--- + +## me import memories + +Import memory records from files or stdin. `me memory import` is an alias of this command. + +``` +me import memories [files...] [options] +``` + +See [me memory import](me-memory.md#me-memory-import) for the full option reference, format detection, skip semantics, and chunking behavior, and [File Formats](../formats.md) for the record schemas. + +--- + +## me import claude / codex / opencode + +Import agent sessions from each tool's native storage. The per-agent spellings (`me claude import`, `me codex import`, `me opencode import`) are aliases of these commands. + +``` +me import claude [options] +me import codex [options] +me import opencode [options] +``` + +See [agent session imports](agent-session-imports.md) for the shared option reference, tree layout, idempotency rules, content shape, and metadata schema. + +--- + +## me import git + +Import a repo's git commit history as memories — one memory per commit, holding the commit message plus a capped changed-file list. Commit intent ("why did we do X") and touched paths become searchable agent context. + +``` +me import git [repo] [options] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `repo` | no | Path inside the repo to import. Default: the current directory. | + +| Option | Description | +|--------|-------------| +| `--branch ` | Branch, tag, or rev to walk. Default: `HEAD`. | +| `--since ` | Only commits at/after this date (any format git accepts). | +| `--until ` | Only commits at/before this date. | +| `--max-count ` | Import at most this many recent commits. | +| `--full` | Walk the full history (skip the incremental high-water lookup). | +| `--no-merges` | Drop all merge commits. | +| `--no-file-list` | Omit the changed-file list from commit memories. | +| `--tree-root ` | Tree root under which `.git_history` is placed. Default: `share.projects`. | +| `--dry-run` | Parse and report what would be imported without writing. | +| `-v, --verbose` | Per-commit progress output. | + +### Tree layout + +Commits are stored under: + +``` +..git_history +``` + +The project slug is derived exactly as for [agent session imports](agent-session-imports.md#tree-layout) (git remote repo name, else repo root directory name), so a project's commit history sits next to its `agent_sessions` node — e.g. `share.projects.memory_engine.git_history`. + +### Content shape + +Each memory's content is the commit subject, the body (truncated past 64 KiB), and a `Files:` block listing up to 50 changed paths with `(+added -deleted)` line counts (`(binary)` for binary files). `--no-file-list` omits the block. + +Merge commits with no message body (`Merge branch 'x'` boilerplate) are skipped by default; merges that carry a body — GitHub PR merge commits put the PR title there — are imported. `--no-merges` drops all merges. + +### Idempotency and incremental re-runs + +Each commit gets a deterministic UUIDv7 keyed by `(tree, sha)` with the commit date as its timestamp half. Re-imports are server-side no-ops: an already-imported commit is skipped, never duplicated. + +Re-runs are also incremental: the newest already-imported commit is looked up server-side, and when it is an ancestor of the target rev only `..` is walked. After a force-push (or when importing a different branch) the walk falls back to the full log — still safe, because the deterministic ids dedupe the overlap. Explicit bounds (`--since`, `--until`, `--max-count`, `--full`) always walk exactly what they say. + +### Metadata + +| Key | Description | +|-----|-------------| +| `type` | Always `"git_commit"`. | +| `sha` | Full 40-hex commit sha. | +| `source_git_repo` | Git remote URL (when the repo has one). | +| `source_project_slug` | ltree-safe project label (same as the tree subnode). | +| `author_name` / `author_email` | Commit author. | +| `author_date` / `commit_date` | ISO 8601 author and committer dates. | +| `files_changed` / `insertions` / `deletions` | Change stats (binary files excluded from line counts). | +| `is_merge` | `true` on merge commits (absent otherwise). | +| `imported_at` | ISO 8601 timestamp of this import run. | +| `importer_version` | Version tag of the importer schema. | + +Temporal is a point-in-time at the commit date. + +### Example + +Backfill this repo's history, then keep it current with cheap re-runs: + +```bash +me import git --dry-run -v # preview +me import git # full backfill (first run) +me import git # later: walks only commits since the last import +``` + +--- + +## me import git-hook + +Install a managed git `post-commit` hook that re-runs [`me import git`](#me-import-git) in the background after every commit, keeping the repo's git history memories current without manual re-runs. + +``` +me import git-hook [repo] +me import git-hook --remove +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `repo` | no | Path inside the repo. Default: the current directory. | + +| Option | Description | +|--------|-------------| +| `--remove` | Remove the managed block (and the hook file, if nothing else remains). | + +### What gets installed + +A marker-delimited managed block in the repo's effective `post-commit` hook (worktree-aware, resolved via `git rev-parse --git-path hooks`): + +```sh +# >>> memory-engine (managed by `me import git-hook`) >>> +# Best-effort and asynchronous: never blocks or fails the commit. +("/path/to/me" import git >/dev/null 2>&1 &) +# <<< memory-engine <<< +``` + +The embedded `me` path is absolute, so commits from GUI git clients (no shell PATH) still trigger the import. If a `post-commit` hook already exists, the block is appended once and the existing script is preserved; re-running `git-hook` replaces the block in place (idempotent, refreshes the embedded path). A foreign hook that exits early never reaches the appended block — move the block up manually in that case. + +Because [`me import git`](#me-import-git) is high-water incremental, **any** hook fire catches up the entire backlog — including commits that arrived via pull, merge, or rebase since the last fire. A single `post-commit` hook therefore suffices; there is no post-merge/post-rewrite matrix to install. + +### Hooks managers (core.hooksPath) + +When the repo routes hooks through `core.hooksPath` (husky, lefthook, and similar committed hooks managers), `git-hook` refuses rather than write into committed files. Add this line to the manager's `post-commit` hook instead: + +```sh +me import git >/dev/null 2>&1 & +``` + +### Scope and failure mode + +The hook lives in `.git/hooks` — per clone, never committed, never pushed. CI checkouts and teammates' clones are unaffected; each clone opts in by running `me import git-hook` itself ([`me claude init`](me-claude.md#me-claude-init) offers it as a setup step). + +The import is deliberately silent and best-effort: it never blocks or fails a commit, which also means auth or connectivity problems won't surface at commit time. If history seems stale, run `me import git` manually to see the error — the next successful fire catches everything up. diff --git a/docs/cli/me-invitation.md b/docs/cli/me-invitation.md deleted file mode 100644 index 4a3fc69..0000000 --- a/docs/cli/me-invitation.md +++ /dev/null @@ -1,84 +0,0 @@ -# me invitation - -Manage invitations. - -Invitations allow you to add people to an organization before they have an account. The invitee receives a token they can use to accept the invitation after signing up. - -## Commands - -- [me invitation create](#me-invitation-create) -- invite someone to an organization -- [me invitation list](#me-invitation-list) -- list pending invitations -- [me invitation accept](#me-invitation-accept) -- accept an invitation -- [me invitation revoke](#me-invitation-revoke) -- revoke a pending invitation - ---- - -## me invitation create - -Invite someone to an organization. - -``` -me invitation create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `email` | yes | Email address to invite. | -| `role` | yes | Role: `owner`, `admin`, or `member`. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID. | -| `--expires ` | Expiration in days (1-30, default: 7). | - -Displays the invitation ID, role, expiry, and the invitation token to share with the invitee. - ---- - -## me invitation list - -List pending invitations. - -``` -me invitation list [org] [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `org` | no | Organization name, slug, or ID. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID (alternative to positional argument). | - -Displays a table of pending invitations with ID, email, role, and expiry. - ---- - -## me invitation accept - -Accept an invitation. - -``` -me invitation accept -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `token` | yes | Invitation token received from the inviter. | - -You must be logged in to accept an invitation. - ---- - -## me invitation revoke - -Revoke a pending invitation. - -``` -me invitation revoke -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id` | yes | Invitation ID. | diff --git a/docs/cli/me-login.md b/docs/cli/me-login.md index dce715c..9592e13 100644 --- a/docs/cli/me-login.md +++ b/docs/cli/me-login.md @@ -5,14 +5,20 @@ Authenticate with Memory Engine via OAuth. ## Usage ``` -me login +me login [space] ``` +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | no | Space slug or name to make active after login. | + ## Description -Starts an OAuth device flow. You choose a provider (Google or GitHub), then the CLI displays a device code and opens your browser for authorization. Once you approve, the CLI stores your session token. +Starts an OAuth device flow. You choose a provider (Google or GitHub), the CLI displays a device code and opens your browser, and once you approve it stores your **session token**. Sessions are rolling: valid for 7 days and refreshed as you keep using the CLI. + +If you pass a `space` argument, it becomes the active space. Otherwise, if you belong to exactly one space it's selected automatically; if you belong to several, run `me space use` to pick one. The active space is carried as the `X-Me-Space` header on subsequent commands. -If your account has exactly one engine, it is automatically selected as the active engine and an API key is stored. +Login also runs the same version compatibility check as `me version` before opening the browser, so an out-of-date CLI gets a clean upgrade prompt instead of failing mid-flow. ## Global Options @@ -22,6 +28,12 @@ If your account has exactly one engine, it is automatically selected as the acti ## Notes -- Credentials are stored in `~/.config/me/credentials.yaml`. -- After login, use `me engine use` to select an engine if you have more than one. -- Use `me logout` to clear stored credentials. +- The session token is stored in your OS keychain when one is available (macOS `security`, Linux `secret-tool`); otherwise it falls back to `~/.config/me/credentials.yaml` (mode 0600). Set `ME_NO_KEYCHAIN=1` to force the file fallback. +- Non-secret settings (default server and per-server active space) live in `~/.config/me/config.yaml`. +- **API keys are for agents, not humans** — `me login` never creates one. Mint agent keys with [`me apikey create`](me-apikey.md#me-apikey-create). +- Use [`me logout`](me-logout.md) to clear the session; the non-secret config is kept so re-login resumes. + +## See also + +- [`me space`](me-space.md) -- list and switch the active space. +- [`me whoami`](me-whoami.md) -- show your identity and active space. diff --git a/docs/cli/me-logout.md b/docs/cli/me-logout.md index 44e7fb2..9dbb516 100644 --- a/docs/cli/me-logout.md +++ b/docs/cli/me-logout.md @@ -10,7 +10,9 @@ me logout ## Description -Removes all stored credentials (session token, API keys) for the active server from the credentials file. +Clears the stored **session token** for the active server — from the OS keychain when one is in use, and from `~/.config/me/credentials.yaml` otherwise. The non-secret config (default server and active space in `~/.config/me/config.yaml`) is kept, so a later `me login` resumes where you left off. + +Agent API keys are never persisted by the CLI (they only ever come from `ME_API_KEY`), so there is nothing to clear for agents. ## Global Options diff --git a/docs/cli/me-mcp.md b/docs/cli/me-mcp.md index 2b84416..8753189 100644 --- a/docs/cli/me-mcp.md +++ b/docs/cli/me-mcp.md @@ -20,9 +20,16 @@ me mcp [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key for engine authentication. Can also be set via `ME_API_KEY` env var. | +| `--api-key ` | Agent API key. If omitted, the server uses your stored `me login` session. | +| `--space ` | Space to operate in (the `X-Me-Space`). | -The server URL is resolved from `--server` (global option) > `ME_SERVER` env > `https://api.memory.build`. +Resolution order: + +- **Auth token**: `--api-key` > `ME_API_KEY` > stored session token. +- **Space**: `--space` > `ME_SPACE` > stored active space. +- **Server URL**: `--server` (global option) > `ME_SERVER` > `https://api.memory.build`. + +A logged-in developer needs no key or space — the active session and active space are used automatically. For an unattended/headless agent, pass `--api-key` and `--space` (or set `ME_API_KEY` / `ME_SPACE`). This command is typically not run directly -- it is invoked by AI tools based on their MCP configuration. @@ -46,4 +53,4 @@ claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] ``` -Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `api_key`, `server`, and `tree_prefix`. +Then start Claude Code, run `/plugin`, select `memory-engine`, and configure the options (all optional except `server`): leave `api_key` blank to use your `me login` session, leave `space` blank to use your active space, and `tree_root` defaults to `share.projects`. diff --git a/docs/cli/me-memory.md b/docs/cli/me-memory.md index 26790b5..c469813 100644 --- a/docs/cli/me-memory.md +++ b/docs/cli/me-memory.md @@ -34,11 +34,11 @@ me memory create [content] [options] | Option | Description | |--------|-------------| | `--content ` | Memory content (alternative to positional argument). | -| `--tree ` | Tree path (e.g., `work.projects.me`). | +| `--tree ` | **Required.** Tree path where the memory is stored (e.g., `share.work.projects`). Use `share` for memories the rest of the space should see, or `~` (your private home, e.g. `~.notes`) for memories that must stay private to you. | | `--meta ` | Metadata as a JSON string. | | `--temporal ` | Temporal range as `start[,end]` (ISO 8601). | -Content can come from the positional argument, the `--content` flag, or piped via stdin. +Content can come from the positional argument, the `--content` flag, or piped via stdin. A `--tree` path is required. --- @@ -213,7 +213,7 @@ Moves all memories under the source prefix to the destination, preserving subtre ## me memory import -Import memories from files or stdin. +Import memories from files or stdin. This is an alias of [`me import memories`](me-import.md#me-import-memories) (unlike the other memory subcommands, `import` has no bare top-level alias — the top-level `me import` is the [import group](me-import.md)). ``` me memory import [files...] [options] @@ -235,11 +235,11 @@ Supports Markdown (with YAML frontmatter), YAML, JSON, and NDJSON. Format is aut ### Skipped memories -Memories with an explicit `id` that already exists in the engine are silently skipped server-side (via `ON CONFLICT DO NOTHING`) rather than failing the whole batch. The command surfaces these as `skipped` so re-imports of unchanged data and id collisions with unrelated memories are observable. Memories without an `id` get a server-generated UUIDv7 and never collide. +Memories with an explicit `id` that already exists in the space are silently skipped server-side (a conflict skip in `create_memory`) rather than failing the whole batch. The command surfaces these as `skipped` so re-imports of unchanged data and id collisions with unrelated memories are observable. Memories without an `id` get a server-generated UUIDv7 and never collide. JSON output adds `skipped` (count) and `skippedIds` (array of conflicting ids). Text output appends `(K skipped — id already exists)` to the summary, or prints `Imported 0 memories (N already exist, no changes)` when everything was a re-import. Run with `--verbose` to see each skipped id inline. -Skipped memories do not contribute to the exit code; only parse and engine errors do. +Skipped memories do not contribute to the exit code; only parse and server errors do. `--dry-run` validates parsing only; it does not predict id collisions with already-imported memories. Run with `--verbose` after a real import to see the skipped ids. diff --git a/docs/cli/me-opencode.md b/docs/cli/me-opencode.md index 12919e3..2d283a5 100644 --- a/docs/cli/me-opencode.md +++ b/docs/cli/me-opencode.md @@ -19,10 +19,11 @@ me opencode install [options] | Option | Description | |--------|-------------| -| `--api-key ` | API key to embed in the MCP config. | +| `--api-key ` | API key for a headless agent. Default: the MCP server uses your `me login` session, resolved at runtime. | +| `--space ` | Pin a space. Default: resolve `ME_SPACE` / active space at runtime. | | `--server ` | Server URL to embed in the MCP config. | -If no `--api-key` or `--server` is provided, values are resolved from `~/.config/me/credentials.yaml` (set by `me login` and `me engine use`). +By default only the server URL is baked into the config: at runtime `me mcp` uses your `me login` session (resolved from the OS keychain / `~/.config/me` each run, so it survives re-login) and your active space (set by `me space use` / `ME_SPACE`). Pass `--api-key` (mint one with `me apikey create `) for a headless agent that cannot reach your keychain; that bakes the key and requires a pinned `--space`. For manual MCP client configuration, see [MCP Integration](../mcp-integration.md). @@ -30,7 +31,7 @@ For manual MCP client configuration, see [MCP Integration](../mcp-integration.md ## me opencode import -Import OpenCode sessions from `~/.local/share/opencode/storage/`. +Import OpenCode sessions from `~/.local/share/opencode/storage/`. This is an alias of [`me import opencode`](me-import.md#me-import-claude--codex--opencode). ``` me opencode import [options] diff --git a/docs/cli/me-org.md b/docs/cli/me-org.md deleted file mode 100644 index f5098aa..0000000 --- a/docs/cli/me-org.md +++ /dev/null @@ -1,138 +0,0 @@ -# me org - -Manage organizations. - -Organizations group engines and members. Each engine belongs to exactly one organization. - -## Commands - -- [me org list](#me-org-list) -- list your organizations -- [me org create](#me-org-create) -- create an organization -- [me org rename](#me-org-rename) -- rename an organization -- [me org delete](#me-org-delete) -- delete an organization -- [me org member list](#me-org-member-list) -- list organization members -- [me org member add](#me-org-member-add) -- add a member -- [me org member remove](#me-org-member-remove) -- remove a member - ---- - -## me org list - -List your organizations. - -``` -me org list -``` - -Displays a table of organizations you belong to, showing ID, name, and slug. - ---- - -## me org create - -Create an organization. - -``` -me org create -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | Organization name. | - ---- - -## me org rename - -Rename an organization. - -``` -me org rename -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name-or-id` | yes | Organization name, slug, or ID. | -| `new-name` | yes | New organization name. | - -Renaming changes only the human-readable name. The org slug is randomly generated at creation time and never changes, so any references to the org by slug or ID continue to work. - -Requires the `owner` or `admin` role on the organization. - ---- - -## me org delete - -Delete an organization. - -``` -me org delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name-or-id` | yes | Organization name, slug, or ID. | - -| Option | Description | -|--------|-------------| -| `-y, --yes` | Skip the confirmation prompt. | - -This operation is irreversible. - ---- - -## me org member list - -List organization members. - -``` -me org member list [org] [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `org` | no | Organization name, slug, or ID. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID (alternative to positional argument). | - -Displays a table of members with name, email, role, and join date. - ---- - -## me org member add - -Add a member to an organization. - -``` -me org member add [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `email-or-id` | yes | Email address or identity ID of the person to add. | -| `role` | yes | Role: `owner`, `admin`, or `member`. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID. | - ---- - -## me org member remove - -Remove a member from an organization. - -``` -me org member remove [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name-email-or-id` | yes | Member name, email, or identity ID. | - -| Option | Description | -|--------|-------------| -| `--org ` | Organization name, slug, or ID. | -| `-y, --yes` | Skip the confirmation prompt. | diff --git a/docs/cli/me-owner.md b/docs/cli/me-owner.md deleted file mode 100644 index 5df4f4f..0000000 --- a/docs/cli/me-owner.md +++ /dev/null @@ -1,73 +0,0 @@ -# me owner - -Manage tree ownership. - -Ownership gives a user implicit admin access to a tree path and all its descendants. Unlike grants, ownership is unique per path -- each path has at most one owner. - -## Commands - -- [me owner set](#me-owner-set) -- set tree path owner -- [me owner remove](#me-owner-remove) -- remove tree path owner -- [me owner get](#me-owner-get) -- get tree path owner -- [me owner list](#me-owner-list) -- list ownership records - ---- - -## me owner set - -Set tree path owner. - -``` -me owner set -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `path` | yes | Tree path. | -| `user` | yes | User name or ID. | - ---- - -## me owner remove - -Remove tree path owner. - -``` -me owner remove -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `path` | yes | Tree path. | - ---- - -## me owner get - -Get tree path owner. - -``` -me owner get -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `path` | yes | Tree path. | - -Displays the path, owner name, set-by, and creation date. - ---- - -## me owner list - -List ownership records. - -``` -me owner list [user] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | no | Filter by user name or ID. | - -Displays a table of ownership records with tree path and owner. diff --git a/docs/cli/me-pack.md b/docs/cli/me-pack.md index ca4a42f..3b6fe65 100644 --- a/docs/cli/me-pack.md +++ b/docs/cli/me-pack.md @@ -2,12 +2,12 @@ Manage memory packs. -Memory packs are YAML files containing pre-built collections of memories. They provide structured knowledge that can be installed into any engine -- things like framework documentation, best practices, or domain-specific reference material. +Memory packs are YAML files containing pre-built collections of memories. They provide structured knowledge that can be installed into any space -- things like framework documentation, best practices, or domain-specific reference material. ## Commands - [me pack validate](#me-pack-validate) -- validate a pack file -- [me pack install](#me-pack-install) -- install a pack into the active engine +- [me pack install](#me-pack-install) -- install a pack into the active space - [me pack list](#me-pack-list) -- list installed packs --- @@ -30,7 +30,7 @@ Parses the YAML file and runs pack-specific constraint validation. Reports wheth ## me pack install -Install a memory pack into the active engine. +Install a memory pack into the active space. ``` me pack install [options] @@ -48,7 +48,7 @@ me pack install [options] The install process: 1. Validates the pack file. -2. Connects to the active engine. +2. Connects to the active space. 3. Finds existing memories from the same pack (by metadata). 4. Deletes stale memories from previous versions (with confirmation). 5. Creates all memories from the pack with `pack.*` tree prefixes and pack metadata. @@ -114,7 +114,7 @@ Dry-run output includes `wouldSkipIdempotent` (predicted from rows already at th ## me pack list -List installed packs in the active engine. +List installed packs in the active space. ``` me pack list diff --git a/docs/cli/me-role.md b/docs/cli/me-role.md deleted file mode 100644 index b173c16..0000000 --- a/docs/cli/me-role.md +++ /dev/null @@ -1,131 +0,0 @@ -# me role - -Manage roles. - -Roles are groups of users within an engine. Grant access to a role and all its members inherit that access. Roles cannot authenticate directly -- they are used purely for grouping. - -## Commands - -- [me role create](#me-role-create) -- create a role -- [me role delete](#me-role-delete) -- delete a role -- [me role list](#me-role-list) -- list all roles -- [me role add-member](#me-role-add-member) -- add a user to a role -- [me role remove-member](#me-role-remove-member) -- remove a user from a role -- [me role members](#me-role-members) -- list members of a role -- [me role list-for](#me-role-list-for) -- list roles a user belongs to - ---- - -## me role create - -Create a role. - -``` -me role create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | Role name. | - -| Option | Description | -|--------|-------------| -| `--identity-id ` | Link to an accounts identity. | - ---- - -## me role delete - -Delete a role. Alias: `me role rm`. - -``` -me role delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | Role ID or name. | - -| Option | Description | -|--------|-------------| -| `-y, --yes` | Skip confirmation prompt. | - -Deleting a role removes all grants and membership associations. This is irreversible. - ---- - -## me role list - -List all roles. - -``` -me role list -``` - -Displays a table of roles with ID and name. - ---- - -## me role add-member - -Add a user to a role. - -``` -me role add-member [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `role` | yes | Role ID or name. | -| `member` | yes | User ID or name. | - -| Option | Description | -|--------|-------------| -| `--with-admin-option` | Allow the member to manage this role. | - ---- - -## me role remove-member - -Remove a user from a role. - -``` -me role remove-member -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `role` | yes | Role ID or name. | -| `member` | yes | User ID or name. | - ---- - -## me role members - -List members of a role. - -``` -me role members -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `role` | yes | Role ID or name. | - -Displays a table of members with ID, name, and admin status. - ---- - -## me role list-for - -List roles a user belongs to. - -``` -me role list-for -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `user` | yes | User ID or name. | - -Displays a table of roles with ID, name, and admin status. diff --git a/docs/cli/me-serve.md b/docs/cli/me-serve.md index a5bcd56..7f84c9a 100644 --- a/docs/cli/me-serve.md +++ b/docs/cli/me-serve.md @@ -13,13 +13,13 @@ me serve [--port ] [--host ] [--no-open] Starts a local HTTP server that: - Serves a React-based UI for browsing, searching, viewing, editing, and deleting memories. -- Proxies JSON-RPC calls from the browser to the configured engine, injecting your stored API key so the key never leaves the machine. +- Proxies JSON-RPC calls from the browser to the server, injecting your stored session token so it never leaves the machine. By default the server binds to `127.0.0.1:3000`; if 3000 is busy it tries 3001, 3002, … up to 3019 before giving up. Passing `--port` explicitly is strict — it does not auto-increment. The browser opens automatically on startup unless `--no-open` is passed. Press `Ctrl+C` to stop. -The UI talks to whichever engine is active for the current server — same resolution as every other `me` command (`--server` flag > `ME_SERVER` env > stored `default_server`; within the server, the active engine is picked via `me engine use`). Run `me whoami` to confirm. +The UI talks to whichever space is active for the current server — same resolution as every other `me` command (`--server` flag > `ME_SERVER` env > stored default server; within the server, the active space is picked via `me space use`). Run `me whoami` to confirm. ## Options @@ -60,7 +60,7 @@ The UI talks to whichever engine is active for the current server — same resol ## Security notes -- The server binds to `127.0.0.1` only — no LAN exposure. The browser never sees your API key or session token; `me serve` injects them into RPC calls on the way out. +- The server binds to `127.0.0.1` only — no LAN exposure. The browser never sees your session token; `me serve` injects it into RPC calls on the way out. - No authentication is required on the local server. Do not `--host 0.0.0.0` or tunnel the port unless you understand the implications. ## Examples @@ -72,11 +72,11 @@ me serve # Use a specific port and skip auto-open (handy when iterating in dev). me serve --port 8080 --no-open -# Point at a specific engine server. +# Point at a specific server. me serve --server https://api.memory.build ``` ## See also -- [`me engine use`](me-engine.md) — pick the active engine that `me serve` will connect to. +- [`me space use`](me-space.md#me-space-use) — pick the active space that `me serve` will connect to. - [`me memory search`](me-memory.md#me-memory-search) — the CLI equivalent of the UI's search bar. diff --git a/docs/cli/me-space.md b/docs/cli/me-space.md new file mode 100644 index 0000000..864c781 --- /dev/null +++ b/docs/cli/me-space.md @@ -0,0 +1,135 @@ +# me space + +Manage spaces. + +A **space** is an isolated collection of memories with its own roster, groups, and access grants. It is identified by an immutable 12-character **slug** (also the `X-Me-Space` header value and the `me_` database schema) and a renamable display **name**. Your *active* space is the one carried on every memory command; set it with `me space use` (or `me login `). + +These commands authenticate with your **session** (humans only — `me login`). Invitations operate on the active space. + +## Commands + +- [me space list](#me-space-list) -- list the spaces you belong to +- [me space use](#me-space-use) -- set the active space +- [me space create](#me-space-create) -- create a space +- [me space rename](#me-space-rename) -- rename a space +- [me space delete](#me-space-delete) -- delete a space +- [me space invite](#me-space-invite) -- invite a user (and manage invitations) + +--- + +## me space list + +List the spaces you belong to. The active space is marked. Alias: `me space ls`. + +``` +me space list +``` + +--- + +## me space use + +Set the active space (the `X-Me-Space` context used by other commands). Stored in `~/.config/me/config.yaml` per server. + +``` +me space use [space] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | no | Space slug or name. Prompts interactively if omitted. | + +--- + +## me space create + +Create a new space and make it active. As the creator you become a space **admin** and receive `owner@home` and `owner@share` (not `owner@root`). + +``` +me space create +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `name` | yes | Display name for the space. | + +--- + +## me space rename + +Rename a space's display name. The slug is immutable (it's the schema name and routing key), so this changes only the label. + +``` +me space rename +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | yes | Space slug or name. | +| `new-name` | yes | New display name. | + +--- + +## me space delete + +Permanently delete a space and all its data — memories, grants, groups, invitations. Irreversible. Alias: `me space rm`. + +``` +me space delete [--force] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `space` | yes | Space slug or name. | + +| Option | Description | +|--------|-------------| +| `--force` | Skip the confirmation prompt. | + +--- + +## me space invite + +Invite a user to the active space by email. If the email already belongs to a registered user, they're added immediately; otherwise a pending invitation is recorded and redeemed at their next verified login. **Admin only.** + +``` +me space invite [--admin] [--share ] +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `email` | yes | The invitee's email address. | + +| Option | Description | +|--------|-------------| +| `--admin` | Make the user a space admin (structural authority). | +| `--share ` | Access to grant at the shared root: `none`, `read`, `write`, or `owner` (default: `read`). | + +A joining user always receives `owner@home` (their private root); `--share` controls their access to `share`. + +### me space invite list + +List pending invitations for the active space. Alias: `me space invite ls`. + +``` +me space invite list +``` + +### me space invite revoke + +Revoke a pending invitation by email. + +``` +me space invite revoke +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `email` | yes | The invited email to revoke. | + +## See also + +- [Access Control](../access-control.md) -- principals, the two axes of authority, and tree-access grants. +- [`me access`](me-access.md) -- grant read/write/owner access on tree paths. +- [`me group`](me-group.md) -- bundle members for shared grants. +- [`me agent`](me-agent.md) -- add your agents to a space. diff --git a/docs/cli/me-user.md b/docs/cli/me-user.md deleted file mode 100644 index cd221cd..0000000 --- a/docs/cli/me-user.md +++ /dev/null @@ -1,101 +0,0 @@ -# me user - -Manage engine users. - -Users are principals within an engine that can own memories, receive grants, and authenticate via API keys. Each user belongs to a single engine. - -## Commands - -- [me user list](#me-user-list) -- list users -- [me user create](#me-user-create) -- create a user -- [me user get](#me-user-get) -- get a user by ID or name -- [me user delete](#me-user-delete) -- delete a user -- [me user rename](#me-user-rename) -- rename a user - ---- - -## me user list - -List users in the active engine. - -``` -me user list [options] -``` - -| Option | Description | -|--------|-------------| -| `--login-only` | Only show users that can log in (excludes roles). | - -Displays a table of users with ID, name, and flags (superuser, createrole, role). - ---- - -## me user create - -Create an engine user. - -``` -me user create [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `name` | yes | User name. | - -| Option | Description | -|--------|-------------| -| `--superuser` | Grant superuser privileges. | -| `--createrole` | Allow this user to create other users and roles. | -| `--no-login` | Create as a role (cannot authenticate directly). | -| `--identity-id ` | Link to an accounts identity. | - ---- - -## me user get - -Get a user by ID or name. - -``` -me user get -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | User ID (UUIDv7) or name. | - -Displays full user details: name, ID, superuser, createrole, canLogin, identity, and creation date. - ---- - -## me user delete - -Delete a user. - -``` -me user delete [options] -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | User ID or name. | - -| Option | Description | -|--------|-------------| -| `-y, --yes` | Skip the confirmation prompt. | - -This operation is irreversible. - ---- - -## me user rename - -Rename a user. - -``` -me user rename -``` - -| Argument | Required | Description | -|----------|----------|-------------| -| `id-or-name` | yes | User ID or current name. | -| `new-name` | yes | New name. | diff --git a/docs/cli/me-whoami.md b/docs/cli/me-whoami.md index 3381e86..de6425f 100644 --- a/docs/cli/me-whoami.md +++ b/docs/cli/me-whoami.md @@ -1,6 +1,6 @@ # me whoami -Show current identity and active engine. +Show current identity and active space. ## Usage @@ -10,9 +10,9 @@ me whoami ## Description -Fetches your current identity from the accounts API and displays your name, email, user ID, server URL, and active engine. +Calls the user endpoint (`whoami`) and displays your name, email, principal ID, server URL, and active space. -Returns an error if you are not logged in. +Returns an error if you are not logged in. Set or change the active space with [`me space use`](me-space.md#me-space-use). ## Global Options diff --git a/docs/concepts.md b/docs/concepts.md index 7271c10..31549b7 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -9,7 +9,7 @@ A memory is a single piece of knowledge. Every memory has: - **meta** -- key-value metadata for filtering (e.g., `{"type": "decision", "confidence": "high"}`). - **temporal** -- a time association, either a point-in-time or a date range. -Memories are stored in a single PostgreSQL table. There are no separate tables for different "types" of memory -- the type is a convention in `meta`, not a schema distinction. This keeps queries simple and the data model flexible. +Each space stores its memories in a single PostgreSQL table (the `me_` schema). There are no separate tables for different "types" of memory -- the type is a convention in `meta`, not a schema distinction. This keeps queries simple and the data model flexible. ### Best practices @@ -52,9 +52,18 @@ personal.reading personal.reading.books ``` -Tree paths use PostgreSQL's `ltree` extension. Labels must be **lowercase alphanumeric with underscores** (no spaces, hyphens, or uppercase). +Tree paths use PostgreSQL's `ltree` extension. Labels match `[A-Za-z0-9_-]` (letters, digits, underscores, and hyphens); use `.` as the separator (`/` is also accepted on input and normalized). -Keep paths **2-4 levels deep**. Deeper nesting rarely helps findability. When unsure about the right tree path, omit it -- you can always add one later, and content is still findable via search. +Keep paths **2-4 levels deep**. Deeper nesting rarely helps findability. + +### Reserved roots + +Every space has two conventional roots: + +- **`share`** -- the shared root. Memories the rest of the space should see go here (`share.work.projects`, etc.). The file importers default a tree-less record to `share`. +- **`home.`** -- your private per-member root. The input shortcut **`~`** expands to your own home, so `~.notes` is stored as `home..notes` and displays back as `~.notes`. + +`me memory create` (and the `me_memory_create` MCP tool) **require** an explicit tree -- choose `share` for shared memories or `~` for private ones. See [Access Control](access-control.md) for how grants attach to these paths. ### Tree filter syntax @@ -89,13 +98,13 @@ When filtering by tree (in search, export, or browse), the system auto-detects w ### Conventions -Tree paths are user-defined. There is no mandated hierarchy. Common patterns: +Below the two reserved roots, tree paths are user-defined. There is no mandated hierarchy. Common patterns: ``` -work.projects. # per-project knowledge -me.design. # design decisions -pack. # installed memory packs -notes. # general notes +share.work.projects. # shared per-project knowledge +share.design. # shared design decisions +pack. # installed memory packs (their own root) +~.notes. # private notes ``` ## Metadata @@ -211,13 +220,14 @@ Filters can also be used alone (without semantic or fulltext) to browse memories Search results include a `score` between 0 and 1, where 1 is the best match. For hybrid search, scores are computed via RRF fusion. For filter-only queries, results are sorted by creation time (configurable with `order_by`). -## Engines +## Spaces + +A **space** is an isolated collection of memories with its own roster, groups, and access grants. Each space has: -An engine is an isolated memory database. Each engine has its own: +- Its own memories (the `me_` table) and tree hierarchy. +- A roster of **principals** -- users, agents, and groups. +- Tree-access grants that control who can read/write/own which paths. -- Memories -- Users, roles, and grants -- API keys -- Tree hierarchy +A space is identified by an immutable 12-character **slug** (also the `X-Me-Space` header value) and a renamable display **name**. A user can belong to many spaces; each memory lives in exactly one space. There are no organization, engine, or shard concepts above a space. -Engines belong to organizations. A user can have access to multiple engines across multiple organizations, but each memory lives in exactly one engine. +Manage spaces with [`me space`](cli/me-space.md), and see [Access Control](access-control.md) for principals and grants. diff --git a/docs/formats.md b/docs/formats.md index e91c437..0f29ad5 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -1,6 +1,6 @@ # File Formats -Import and export use the same memory structure across all formats. This page is the canonical reference for the JSON, YAML, Markdown, and NDJSON schemas used by both the CLI (`me memory import` / `me memory export`) and MCP tools (`me_memory_import` / `me_memory_export`). +Import and export use the same memory structure across all formats. This page is the canonical reference for the JSON, YAML, Markdown, and NDJSON schemas used by both the CLI (`me import memories` — alias `me memory import` — / `me memory export`) and MCP tools (`me_memory_import` / `me_memory_export`). ## Memory fields @@ -11,7 +11,7 @@ Every memory has one required field (`content`) and four optional fields: | `id` | `string` | no | UUIDv7. Enables idempotent imports -- re-importing the same ID won't create a duplicate. Must match `^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`. | | `content` | `string` | **yes** | The memory text. Must be non-empty. | | `meta` | `object` | no | Arbitrary key-value metadata. Any valid JSON object. | -| `tree` | `string` | no | Hierarchical path using dot-separated labels (e.g. `work.projects.api`). Labels must be alphanumeric or underscore. Must match `^([A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*)?$`. | +| `tree` | `string` | no | Hierarchical path using dot-separated labels (e.g. `share.work.projects.api`). Labels match `[A-Za-z0-9_-]`; `/` is also accepted as a separator and a leading `~` expands to your private home. When omitted, the file importers (`me import memories`, `me_memory_import`) default the record to the shared root `share`. | | `temporal` | varies | no | Time range for the memory. Accepted shapes depend on format -- see below. | ### Temporal input shapes diff --git a/docs/getting-started.md b/docs/getting-started.md index 8c6c722..4f6ee62 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -16,7 +16,16 @@ This installs the `me` binary to `~/.local/bin`. Make sure it's on your PATH. me login ``` -This starts an OAuth flow via GitHub -- authorize in your browser and the CLI stores your session. +This starts an OAuth device flow via GitHub or Google -- authorize in your browser and the CLI stores your session token (rolling 7-day, refreshed as you use it). On a host with a system keychain the token is stored there; otherwise it falls back to `~/.config/me/credentials.yaml` (mode 0600). + +If you belong to more than one space, pick the active one (it's carried as the `X-Me-Space` on every request): + +```bash +me space list +me space use +``` + +`me login ` selects it in one step, and `me whoami` shows your identity and active space. If your CLI is older than the server (or vice versa), `me login` will tell you and bail out before sending you to the browser. You can run the same check explicitly: @@ -29,10 +38,12 @@ me version ```bash me memory create "PostgreSQL 18 supports native UUIDv7 generation." \ - --tree notes.postgres \ + --tree share.notes.postgres \ --meta '{"topic": "database"}' ``` +A `--tree` is required. Put memories the rest of your space should see under `share.*`, and personal ones under `~.*` (your private home). See [Core Concepts](concepts.md#reserved-roots). + ## Search ```bash @@ -72,14 +83,14 @@ me codex install me gemini install ``` -For Claude Code, install the Memory Engine plugin instead: +For Claude Code, `me claude install` installs the full Memory Engine plugin (hooks + slash commands + MCP): ```bash -claude plugin marketplace add timescale/memory-engine -claude plugin install memory-engine@memory-engine +me claude install # full plugin +me claude install --mcp-only # or just the MCP server ``` -Then start Claude Code, run `/plugin`, select `memory-engine`, and configure `api_key`, `server`, and `tree_prefix`. +This drives Claude Code's native plugin flow for you (`claude plugin marketplace add` + `claude plugin install`), passing your resolved server/space/api_key through `--config`. Afterwards, restart Claude Code (or run `/plugin`) to load the hooks and slash commands; you can re-run `/plugin` → `memory-engine` → Configure to adjust options. All are optional except `server`: leave `api_key` blank to use your `me login` session, leave `space` blank to use your active space, and `tree_root` defaults to `share.projects`. After installation, your AI agent has access to memory tools -- create, search, get, update, delete, and more. @@ -88,7 +99,7 @@ See [MCP Integration](mcp-integration.md) for details. ## What's next - [Core Concepts](concepts.md) -- understand memories, tree paths, metadata, search modes -- [Access Control](access-control.md) -- users, roles, grants, and ownership +- [Access Control](access-control.md) -- spaces, principals, and tree-access grants - [Memory Packs](memory-packs.md) -- install pre-built knowledge collections - [MCP Integration](mcp-integration.md) -- how AI agents use Memory Engine - [CLI Reference](cli/me-memory.md) -- full command reference diff --git a/docs/index.md b/docs/index.md index 5bb6cf4..92fb9ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ Permanent memory for AI agents. Store, search, and organize knowledge across con - [Getting Started](getting-started.md) -- install, login, first memory - [Core Concepts](concepts.md) -- memories, tree paths, metadata, search modes -- [Access Control](access-control.md) -- users, roles, grants, ownership +- [Access Control](access-control.md) -- spaces, principals, tree-access grants - [Memory Packs](memory-packs.md) -- pre-built knowledge collections - [MCP Integration](mcp-integration.md) -- connecting AI agents - [TypeScript Client](typescript-client.md) -- programmatic access from TypeScript/JavaScript diff --git a/docs/mcp-integration.md b/docs/mcp-integration.md index bd37416..cd3fac9 100644 --- a/docs/mcp-integration.md +++ b/docs/mcp-integration.md @@ -12,17 +12,17 @@ When an AI tool launches `me mcp`, it spawns a child process that communicates o └──────────────┘ └──────────┘ └────────────────┘ ``` -Authentication is baked into the command via `--api-key` and `--server` flags. The AI agent never sees or handles credentials — it just calls MCP tools and gets results back. +The AI agent never sees or handles credentials — it just calls MCP tools and gets results back. -Each `me mcp` instance is locked to a single engine via its API key. The MCP server does **not** read the credentials file — the API key must be provided via `--api-key` or the `ME_API_KEY` environment variable. The server URL defaults to `https://api.memory.build` (the hosted engine) but can be overridden with `--server` or `ME_SERVER`. +Each `me mcp` instance is locked to a single **space**, carried as the `X-Me-Space` header. The space is resolved from `--space` > `ME_SPACE` > your stored active space. Authentication is **either** an agent API key (`--api-key` or `ME_API_KEY`) **or**, if no key is given, your stored `me login` session token — so a developer install needs no key at all. The server URL defaults to `https://api.memory.build` but can be overridden with `--server` or `ME_SERVER`. ## Setup ### Prerequisites -You need an API key. Run `me whoami` to see your active engine, or create an API key with `me apikey create`. +Log in with `me login` and select a space — `me whoami` shows your active space and identity. That session is enough to run the MCP server locally. For an unattended or dedicated-agent install, mint an API key with `me apikey create ` and pass it with `--api-key`. -The server defaults to `https://api.memory.build`. Pass `--server ` only if you're running a self-hosted engine. +The server defaults to `https://api.memory.build`. Pass `--server ` only if you're running a self-hosted server. ### Agent-specific installers @@ -32,7 +32,7 @@ me codex install me gemini install ``` -These commands register Memory Engine with the named tool. They read your API key and server URL from the credentials file and bake them into the tool's MCP configuration, so the `me mcp` process can authenticate without the credentials file. +These commands register Memory Engine with the named tool, writing a `me mcp` invocation into the tool's MCP configuration. By default they embed no key — the server uses your `me login` session at runtime. Pass `--api-key` to pin a dedicated agent key instead, `--space ` to pin a space, and `--server ` to pin a non-default server. See the agent-specific command references for details: [`me opencode install`](cli/me-opencode.md#me-opencode-install), [`me codex install`](cli/me-codex.md#me-codex-install), and [`me gemini install`](cli/me-gemini.md#me-gemini-install). @@ -41,16 +41,23 @@ See the agent-specific command references for details: [`me opencode install`](c | OpenCode | `me opencode install` | | Codex CLI | `me codex install` | | Gemini CLI | `me gemini install` | -| Claude Code | Claude Code plugin, described below | +| Claude Code | `me claude install` (full plugin) / `me claude install --mcp-only` | ### Claude Code +```bash +me claude install # full plugin: hooks + slash commands + MCP +me claude install --mcp-only # or just the MCP server +``` + +By default `me claude install` installs the Memory Engine plugin, driving Claude Code's native plugin flow for you (`claude plugin marketplace add` + `claude plugin install`) and passing your resolved `server` / `space` / `api_key` through `--config`. The plugin provides the MCP server and captures Claude Code session events as memories. After installing, restart Claude Code (or run `/plugin`) to load the hooks and slash commands; re-run `/plugin` → `memory-engine` → Configure to adjust options. To run the underlying flow by hand instead: + ```bash claude plugin marketplace add timescale/memory-engine claude plugin install memory-engine@memory-engine [--scope user|project|local] ``` -Claude Code uses the Memory Engine plugin. After installing it, start a Claude Code session, run `/plugin`, select `memory-engine`, and configure `api_key`, `server`, and `tree_prefix`. The plugin provides the MCP server and captures Claude Code session events as memories. +See [`me claude install`](cli/me-claude.md#me-claude-install) for the full option reference. ### Gemini CLI @@ -61,7 +68,7 @@ me gemini install To configure manually: ```bash -gemini mcp add --scope user me me mcp --api-key --server +gemini mcp add --scope user me me mcp --api-key --space --server ``` ### Codex CLI @@ -73,7 +80,7 @@ me codex install To configure manually: ```bash -codex mcp add me -- me mcp --api-key --server +codex mcp add me -- me mcp --api-key --space --server ``` ### OpenCode @@ -85,7 +92,7 @@ codex mcp add me -- me mcp --api-key --server "mcp": { "me": { "type": "local", - "command": ["me", "mcp", "--api-key", "", "--server", ""] + "command": ["me", "mcp", "--api-key", "", "--space", "", "--server", ""] } } } @@ -100,7 +107,7 @@ Add a `.vscode/mcp.json` file to your workspace: "servers": { "me": { "command": "me", - "args": ["mcp", "--api-key", "", "--server", ""] + "args": ["mcp", "--api-key", "", "--space", "", "--server", ""] } } } @@ -119,7 +126,7 @@ Open your Zed settings (`Zed > Settings > Open Settings` or `~/.config/zed/setti "context_servers": { "me": { "command": "me", - "args": ["mcp", "--api-key", "", "--server", ""] + "args": ["mcp", "--api-key", "", "--space", "", "--server", ""] } } } @@ -132,7 +139,7 @@ After saving, check the Agent Panel settings — the indicator next to "me" shou Any tool that supports the MCP stdio transport can use Memory Engine. The server command is: ```bash -me mcp --api-key --server +me mcp --api-key --space --server ``` Point your client at this command with `stdio` as the transport type. @@ -176,9 +183,9 @@ This project uses Memory Engine for persistent knowledge. ## Memory Map -- `design.*` -- architecture decisions and design docs -- `research.*` -- research findings and comparisons -- `bugs.*` -- known issues and workarounds +- `share.design.*` -- architecture decisions and design docs +- `share.research.*` -- research findings and comparisons +- `share.bugs.*` -- known issues and workarounds ## How to Search @@ -199,7 +206,7 @@ me_memory_search({semantic: "how does authentication work"}) me_memory_search({fulltext: "OAuth JWT"}) # Browse a section -me_memory_search({tree: "design.*"}) +me_memory_search({tree: "share.design.*"}) ``` ## Troubleshooting @@ -207,11 +214,11 @@ me_memory_search({tree: "design.*"}) ### MCP server shows "failed" or "disabled" 1. Verify the `me` binary is on your PATH: `which me` -2. Test the server directly: `echo '{}' | me mcp --api-key --server ` +2. Test the server directly: `echo '{}' | me mcp --api-key --space --server ` 3. Re-install with the agent-specific command, for example `me opencode install`, `me codex install`, or `me gemini install`. For Claude Code, open `/plugin` and reconfigure `memory-engine`. ### Agent can't find memories -1. Check that the correct engine is active: `me whoami` +1. Check that the correct space is active: `me whoami` 2. Verify memories exist: `me memory search --fulltext ""` 3. Check that embeddings have been computed: `me memory get ` (look for `hasEmbedding: true`) diff --git a/docs/mcp/me_memory_create.md b/docs/mcp/me_memory_create.md index 3fcd449..dab3e87 100644 --- a/docs/mcp/me_memory_create.md +++ b/docs/mcp/me_memory_create.md @@ -9,7 +9,7 @@ Store a new memory. | `id` | `string \| null` | no | UUIDv7 for idempotent creates. Omit or pass `null` to auto-generate. | | `content` | `string` | yes | The content of the memory. Must be non-empty. | | `meta` | `object \| null` | no | Key-value metadata pairs. Omit or pass `null` to skip. | -| `tree` | `string \| null` | no | Hierarchical path using dot-separated labels (e.g., `work.projects.me`). Omit or pass `null` to store at the root. | +| `tree` | `string` | yes | Hierarchical path where the memory is stored, using dot-separated labels (e.g., `share.work.projects`). Choose deliberately: most memories should go under `share` so the rest of the space can see them; use `~` (your private home, e.g. `~.notes`) only for memories that must stay private to you. | | `temporal` | `object \| null` | no | Time range for the memory. Omit or pass `null` to skip. | ### temporal diff --git a/docs/mcp/me_memory_import.md b/docs/mcp/me_memory_import.md index 89b4ce2..a9bef34 100644 --- a/docs/mcp/me_memory_import.md +++ b/docs/mcp/me_memory_import.md @@ -18,7 +18,7 @@ One of `path` or `content` must be provided. JSON (array or single object), NDJSON, YAML (array or single object), and Markdown (YAML frontmatter + body, one memory per file). -Each memory object supports fields: `id`, `content` (required), `meta`, `tree`, `temporal`. +Each memory object supports fields: `id`, `content` (required), `meta`, `tree`, `temporal`. Unlike `me_memory_create` (which requires an explicit `tree`), a record with no `tree` is imported into the shared root `share`. See [File Formats](../formats.md) for full schema documentation, examples, and format detection rules. @@ -43,13 +43,13 @@ See [File Formats](../formats.md) for full schema documentation, examples, and f | Field | Type | Description | |-------|------|-------------| | `imported` | `number` | Number of memories successfully imported on this call. | -| `skipped` | `number` | Number of memories whose explicit `id` already existed in the engine. Always present (may be `0`). | +| `skipped` | `number` | Number of memories whose explicit `id` already existed in the space. Always present (may be `0`). | | `failed` | `number` | Number of memories in chunks that errored before reaching the server. Always present (may be `0`). | | `ids` | `string[]` | UUIDs of the memories actually inserted on this call. | | `skippedIds` | `string[]` | The explicit ids that were skipped because they already existed. Always present (may be empty). Inspect any of these with `me_memory_get` to see what's there. | | `errors` | `Array<{ chunkIndex, itemCount, ids, error }>` | One entry per failed chunk. Always present (may be empty). | -The tool is idempotent for memories with explicit ids: re-calling with the same arguments leaves the engine in the same state, with all previously-imported ids appearing in `skippedIds` instead of `ids`. Memories submitted without an explicit `id` get a server-generated UUIDv7 and never collide. +The tool is idempotent for memories with explicit ids: re-calling with the same arguments leaves the space in the same state, with all previously-imported ids appearing in `skippedIds` instead of `ids`. Memories submitted without an explicit `id` get a server-generated UUIDv7 and never collide. ### Chunking and partial failures diff --git a/docs/memory-packs.md b/docs/memory-packs.md index d83b723..e752828 100644 --- a/docs/memory-packs.md +++ b/docs/memory-packs.md @@ -8,7 +8,7 @@ Memory packs are YAML files containing pre-built collections of memories. They s # Validate first (offline, no server needed) me pack validate packs/typescript-best-practices.yaml -# Install into the active engine +# Install into the active space me pack install packs/typescript-best-practices.yaml ``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c6ed052..89ab163 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7,17 +7,17 @@ 1. **Check embedding status** -- semantic search requires embeddings. New memories take ~10-30 seconds to get embeddings. Use `me memory get ` and check `hasEmbedding`. 2. **Try fulltext instead** -- fulltext search works immediately after creation. Use `--fulltext` to search by keywords. 3. **Broaden the search** -- remove filters (tree, meta, temporal) to see if results appear without them. -4. **Check access** -- RLS silently filters results. If you're missing memories you know exist, check grants with `me grant check read`. See [Access Control](access-control.md) for details. -5. **Check as superuser** -- superusers bypass all access checks. If results appear for a superuser but not a regular user, the issue is grants. +4. **Check the active space** -- results come only from your active space. Run `me whoami` to confirm it, and `me space use ` to switch. +5. **Check access** -- the server filters results to the tree paths you can read; a missing grant looks like missing results, not an error. List your grants with `me access list ` (or `me access list --path `). See [Access Control](access-control.md) for details. ### "Memory not found" on get or update This can mean either: -- The memory genuinely doesn't exist (wrong ID) -- The memory exists but the current user doesn't have `read` access to its tree path (RLS returns "not found") +- The memory genuinely doesn't exist (wrong ID), or it's in a different space than your active one +- The memory exists but you don't have `read` access to its tree path (access filtering reports it as "not found") -Check access with `me grant check read` or retry as a superuser. +Confirm your active space with `me whoami` and check your grants with `me access list `. ### Embeddings stuck @@ -50,5 +50,6 @@ These all use code `-32000` but are distinguished by `data.code`: | `FORBIDDEN` | Valid credentials, insufficient permissions | Check grants for the required action | | `NOT_FOUND` | Resource doesn't exist (or no access) | Verify the ID and check grants | | `CONFLICT` | Resource already exists (e.g., duplicate slug) | Use a different identifier | +| `LAST_ADMIN` | Operation would leave the space with no effective admin | Promote another user/admin group first | | `RATE_LIMITED` | Too many requests | Back off and retry after the `Retry-After` header | | `VALIDATION_ERROR` | Business logic validation failed | Check the error message for details | diff --git a/docs/typescript-client.md b/docs/typescript-client.md index 376cbf8..4933d7e 100644 --- a/docs/typescript-client.md +++ b/docs/typescript-client.md @@ -8,20 +8,30 @@ The `@memory.build/client` package provides programmatic access to Memory Engine npm install @memory.build/client ``` +## Two clients + +The package exposes two clients, matching the two API endpoints: + +- **`createMemoryClient`** — the space data plane plus space management. Talks to `POST /api/v1/memory/rpc`, carrying the active space in the `X-Me-Space` header. Authenticates with **either** a session token **or** an agent API key. Namespaces: `memory`, `principal`, `group`, `grant`, `invite`. +- **`createUserClient`** — session-only, user-scoped operations. Talks to `POST /api/v1/user/rpc`. Authenticates with a **session token only** (an API key can't manage agents). Methods: `whoami`, plus the `agent`, `apiKey`, and `space` namespaces. + +There is also `createAuthClient` for the OAuth device-flow login that produces a session token. + ## Quick start ```typescript -import { createClient } from "@memory.build/client"; +import { createMemoryClient } from "@memory.build/client"; -const me = createClient({ - url: "https://api.memory.build", - apiKey: "me.xxx.yyy", +const me = createMemoryClient({ + url: "https://api.memory.build", // default + token: sessionTokenOrApiKey, // session token or "me.." + space: "abc123def456", // the X-Me-Space slug }); -// Create a memory +// Create a memory (tree is required — choose share.* or ~.* deliberately) await me.memory.create({ content: "TypeScript was released in 2012", - tree: "knowledge.programming", + tree: "share.knowledge.programming", }); // Search @@ -33,24 +43,32 @@ const { results } = await me.memory.search({ ## Configuration ```typescript -const me = createClient({ +const me = createMemoryClient({ url: "https://api.memory.build", // default - apiKey: "me.xxx.yyy", // format: "me.." - timeout: 30000, // request timeout in ms (default: 30000) - retries: 3, // automatic retries (default: 3) + token: "me.xxx.yyy", // session token or api key (format: "me..") + space: "abc123def456", // active space slug (sent as X-Me-Space) + timeout: 30000, // request timeout in ms (default: 30000) + retries: 3, // automatic retries (default: 3) }); ``` -The client retries on `429`, `500`, `502`, `503`, and `504` responses with exponential backoff and jitter. It respects the `Retry-After` header. +The client retries on `429`, `500`, `502`, `503`, and `504` responses with exponential backoff and jitter, and respects the `Retry-After` header. The token and space can be swapped at runtime: + +```typescript +me.setToken("me.newkey.newsecret"); +me.setSpace("otherslug1234"); +``` ## Memory operations ### create +`tree` is required. Use `share.*` for memories the rest of the space should see, or `~.*` for your private home. + ```typescript const memory = await me.memory.create({ content: "The fact to remember", - tree: "work.projects.acme", // optional hierarchical path + tree: "share.work.projects.acme", // required meta: { source: "meeting-notes" }, // optional JSON metadata temporal: { // optional time range start: "2025-01-01T00:00:00Z", @@ -62,13 +80,13 @@ const memory = await me.memory.create({ ### batchCreate -Create up to 1,000 memories in a single call. +Create up to 1,000 memories in a single call. Each memory requires a `tree`. ```typescript const { ids } = await me.memory.batchCreate({ memories: [ - { content: "First memory", tree: "notes" }, - { content: "Second memory", tree: "notes" }, + { content: "First memory", tree: "share.notes" }, + { content: "Second memory", tree: "share.notes" }, ], }); ``` @@ -102,11 +120,8 @@ const { deleted } = await me.memory.delete({ id: "019..." }); Delete all memories under a tree prefix. ```typescript -// Preview what would be deleted -const { count } = await me.memory.deleteTree({ tree: "old.project", dryRun: true }); - -// Actually delete -const { count: deleted } = await me.memory.deleteTree({ tree: "old.project" }); +const { count } = await me.memory.deleteTree({ tree: "share.old.project", dryRun: true }); +const { count: deleted } = await me.memory.deleteTree({ tree: "share.old.project" }); ``` ### move @@ -115,8 +130,8 @@ Move memories from one tree prefix to another, preserving subtree structure. ```typescript const { count } = await me.memory.move({ - source: "drafts.api", - destination: "published.api", + source: "share.drafts.api", + destination: "share.published.api", }); ``` @@ -126,10 +141,9 @@ View the hierarchical tree structure with counts at each node. ```typescript const { nodes } = await me.memory.tree(); -// [{ path: "work", count: 5 }, { path: "work.projects", count: 3 }, ...] +// [{ path: "share", count: 5 }, { path: "share.work", count: 3 }, ...] -// Scoped to a subtree -const { nodes } = await me.memory.tree({ tree: "work", levels: 2 }); +const { nodes } = await me.memory.tree({ tree: "share.work", levels: 2 }); ``` ## Search @@ -137,14 +151,14 @@ const { nodes } = await me.memory.tree({ tree: "work", levels: 2 }); The `search` method supports keyword, semantic, and hybrid search with multiple filter types. ```typescript -const { results, total } = await me.memory.search({ +const { results } = await me.memory.search({ // Search modes (use one or both for hybrid) semantic: "natural language meaning query", fulltext: "exact keyword BM25 match", // Filters (all optional, combined with AND) grep: "regex.*pattern", // POSIX regex on content - tree: "work.projects.*", // ltree/lquery filter + tree: "share.work.projects.*", // ltree/lquery filter meta: { source: "meeting-notes" }, // JSONB containment temporal: { // time-based filter contains: "2025-06-15T00:00:00Z", // point-in-time @@ -165,119 +179,124 @@ for (const { memory, score } of results) { } ``` -## Error handling +## Space management -The client throws `RpcError` for application errors. Each error has a numeric `code` and an optional string `appCode` for programmatic matching. - -```typescript -import { createClient, RpcError } from "@memory.build/client"; +The memory client also exposes the in-space management namespaces. These require the appropriate authority (admin for roster/groups/invites; `owner@path` for grants). See [Access Control](access-control.md). -try { - await me.memory.get({ id: "nonexistent" }); -} catch (error) { - if (error instanceof RpcError) { - console.error(error.message); // human-readable message +### principal — the roster - if (error.is("NOT_FOUND")) { - // handle missing memory - } - } -} +```typescript +const { principals } = await me.principal.list(); // admin only +await me.principal.add({ principalId: "019..." }); +await me.principal.remove({ principalId: "019..." }); +const { principals } = await me.principal.resolve({ name: "alice@example.com" }); // any member +const { principals } = await me.principal.lookup({ ids: ["019..."] }); // any member ``` -### Error codes +### group -| `appCode` | Meaning | -|-----------|---------| -| `NOT_FOUND` | Resource doesn't exist | -| `UNAUTHORIZED` | Missing or invalid API key | -| `FORBIDDEN` | Insufficient permissions | -| `CONFLICT` | Duplicate or conflicting operation | -| `RATE_LIMITED` | Too many requests | -| `VALIDATION_ERROR` | Invalid input | -| `EMBEDDING_NOT_CONFIGURED` | Semantic search without embedding provider | -| `EMBEDDING_FAILED` | Embedding generation failed | -| `INTERNAL_ERROR` | Server error | - -## Access control +```typescript +const group = await me.group.create({ name: "backend" }); +const { groups } = await me.group.list(); +await me.group.rename({ groupId: group.id, name: "backend-team" }); +await me.group.delete({ groupId: group.id }); +await me.group.addMember({ groupId: group.id, memberId: "019...", admin: false }); +await me.group.removeMember({ groupId: group.id, memberId: "019..." }); +const { members } = await me.group.listMembers({ groupId: group.id }); +const { groups } = await me.group.listForMember({ memberId: "019..." }); +``` -The client exposes namespaces for managing users, grants, roles, and owners. These mirror the [Access Control](access-control.md) system. +### grant — tree access -### Users +Levels are `1` (read), `2` (write), `3` (owner). ```typescript -const user = await me.user.create({ name: "alice" }); -const user = await me.user.get({ id: "019..." }); -const user = await me.user.getByName({ name: "alice" }); -const { users } = await me.user.list(); -await me.user.rename({ id: "019...", name: "bob" }); -await me.user.delete({ id: "019..." }); +await me.grant.set({ principalId: "019...", treePath: "share.work", access: 2 }); +await me.grant.remove({ principalId: "019...", treePath: "share.work" }); +const { grants } = await me.grant.list(); // optionally { principalId } / { treePath } ``` -### Grants +### invite ```typescript -await me.grant.create({ - userId: "019...", - treePath: "team.shared", - actions: ["read", "create"], - withGrantOption: false, -}); -const { grants } = await me.grant.list({ userId: "019..." }); -const { allowed } = await me.grant.check({ - userId: "019...", - treePath: "team.shared", - action: "create", -}); -await me.grant.revoke({ userId: "019...", treePath: "team.shared" }); +// shareAccess is a level number (1=read, 2=write, 3=owner) at the shared root; null/omit = none +const invite = await me.invite.create({ email: "alice@example.com", admin: false, shareAccess: 1 }); +const { invitations } = await me.invite.list(); +await me.invite.revoke({ email: "alice@example.com" }); ``` -### Roles +## User-scoped operations + +Use `createUserClient` (session token only) for identity, agents, API keys, and space discovery. ```typescript -const role = await me.role.create({ name: "editors" }); -await me.role.addMember({ roleId: role.id, memberId: userId }); -await me.role.removeMember({ roleId: role.id, memberId: userId }); -const { members } = await me.role.listMembers({ roleId: role.id }); -const { roles } = await me.role.listForUser({ userId }); -``` +import { createUserClient } from "@memory.build/client"; -### Owners +const user = createUserClient({ token: sessionToken }); -```typescript -await me.owner.set({ userId: "019...", treePath: "team.shared" }); -const owner = await me.owner.get({ treePath: "team.shared" }); -const { owners } = await me.owner.list(); -await me.owner.remove({ treePath: "team.shared" }); -``` +// Identity +const me = await user.whoami(); -### API keys +// Spaces — discover and manage the spaces you belong to +const { spaces } = await user.space.list(); +const space = await user.space.create({ name: "My Space" }); // → { id, slug } +await user.space.rename({ slug: space.slug, name: "Renamed" }); +await user.space.delete({ slug: space.slug }); -```typescript -const { apiKey, rawKey } = await me.apiKey.create({ - userId: "019...", +// Agents — your global service accounts +const agent = await user.agent.create({ name: "ci-bot" }); // → { id } +const { agents } = await user.agent.list(); +await user.agent.rename({ id: agent.id, name: "ci-runner" }); +await user.agent.delete({ id: agent.id }); + +// API keys — global per-agent credentials +const { id, key } = await user.apiKey.create({ + agentId: agent.id, name: "ci-pipeline", expiresAt: "2026-01-01T00:00:00Z", // optional }); -console.log(rawKey); // "me.xxx.yyy" — only shown once - -const { apiKeys } = await me.apiKey.list({ userId: "019..." }); -await me.apiKey.revoke({ id: apiKey.id }); -await me.apiKey.delete({ id: apiKey.id }); +console.log(key); // "me.xxx.yyy" — full key returned once; only its hash is stored +const { apiKeys } = await user.apiKey.list({ memberId: agent.id }); +const apiKeyMeta = await user.apiKey.get({ id }); +await user.apiKey.delete({ id }); ``` -## Protocol +API keys are **global** per-agent credentials, not bound to a space: the same key works in any space the agent has been admitted to (the space comes from `X-Me-Space`). -The client communicates over JSON-RPC 2.0 via a single HTTP endpoint (`POST /api/v1/engine/rpc`). Authentication is via `Authorization: Bearer ` header. +## Error handling -You can make raw RPC calls using the `call` method: +The client throws `RpcError` for application errors. Each error has a numeric `code` and an optional string `appCode` for programmatic matching. ```typescript -const result = await me.call("memory.search", { semantic: "hello" }); +import { createMemoryClient, RpcError } from "@memory.build/client"; + +try { + await me.memory.get({ id: "nonexistent" }); +} catch (error) { + if (error instanceof RpcError) { + console.error(error.message); // human-readable message + if (error.is("NOT_FOUND")) { + // handle missing memory + } + } +} ``` -The API key can be swapped at runtime: +### Error codes -```typescript -me.setApiKey("me.newkey.newsecret"); -``` +| `appCode` | Meaning | +|-----------|---------| +| `NOT_FOUND` | Resource doesn't exist (or not visible to you) | +| `UNAUTHORIZED` | Missing or invalid session token / API key | +| `FORBIDDEN` | Insufficient permissions | +| `CONFLICT` | Duplicate or conflicting operation | +| `LAST_ADMIN` | Operation would leave the space with no effective admin | +| `RATE_LIMITED` | Too many requests | +| `VALIDATION_ERROR` | Invalid input | +| `EMBEDDING_NOT_CONFIGURED` | Semantic search without embedding provider | +| `EMBEDDING_FAILED` | Embedding generation failed | +| `INTERNAL_ERROR` | Server error | + +## Protocol + +Both clients speak JSON-RPC 2.0 over HTTP. The memory client uses `POST /api/v1/memory/rpc` with `Authorization: Bearer ` and a required `X-Me-Space: ` header; the user client uses `POST /api/v1/user/rpc` with a session-token bearer. See the [Access Control](access-control.md) guide for the authority model behind the management namespaces. diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts new file mode 100644 index 0000000..48ff898 --- /dev/null +++ b/e2e/cli.e2e.test.ts @@ -0,0 +1,1050 @@ +// End-to-end CLI integration test. +// +// me (spawned binary) ──HTTP──▶ server (real Bun.serve) ──▶ postgres.js ──▶ ghost test DB +// +// Drives the real `me` CLI as a subprocess against a real in-process server +// (startServer) and the ghost test database, with real OpenAI embeddings. No +// mocks between the CLI and the database. See PLAN_E2E_TESTING.md. +// +// TEST_DATABASE_URL="$(ghost connect testing_me)" ./bun run test:e2e +// +// Boundaries (deliberate): authentication is token injection (provisionUser + +// createSession → ME_SESSION_TOKEN), not `me login`; embeddings hit real OpenAI +// directly (a key is required to run — the suite skips, not fails, without one). + +// Set the space-schema prefix before anything reads it. slugToSchema reads +// SPACE_SCHEMA_PREFIX lazily (per call), so this only needs to run before the +// first call at runtime (in beforeAll) — well after import hoisting. Spaces +// land under metest_ so the existing reclaimer sweeps them. +process.env.SPACE_SCHEMA_PREFIX = "metest_"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { Sql } from "postgres"; +import { encodeProjectDir } from "../packages/cli/importers/claude.ts"; +import { + connect, + resolveTestDatabaseUrl, +} from "../packages/database/migrate/test-utils.ts"; +import { type RunningServer, startServer } from "../packages/server/lib.ts"; +import { provisionUser } from "../packages/server/provision.ts"; + +const OPENAI_KEY = process.env.OPENAI_API_KEY ?? process.env.EMBEDDING_API_KEY; + +const repoRoot = join(import.meta.dir, ".."); +const CLI = join(repoRoot, "packages/cli/index.ts"); + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; // harness's own connection (setup/teardown/assert) +let srv: RunningServer; +let authSchema: string; +let coreSchema: string; +let spaceSlug: string; +let token: string; +let tmpHome: string; + +// TEST_CI disables the conditional skip: in CI this suite always runs +// (missing env fails loudly as test errors, never as a silent skip). +describe.skipIf( + !process.env.TEST_CI && (!OPENAI_KEY || !process.env.TEST_DATABASE_URL), +)("cli e2e", () => { + beforeAll(async () => { + sql = connect(); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); + + // Provision the user (and its default space) BEFORE booting the server, so + // the worker discovers the space at startup — no rediscovery lag for the + // initial space. + const provisioned = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: "e2e@example.test", + name: "E2E", + provider: "github", + accountId: `e2e-${rand()}`, + emailVerified: true, + }, + ); + spaceSlug = provisioned.spaceSlug; + ({ token } = await authStore(sql, authSchema).createSession( + provisioned.userId, + )); + + const embeddingConfig: EmbeddingConfig = { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + // OPENAI_KEY is non-null here (describe.skipIf guards it). + apiKey: OPENAI_KEY as string, + options: {}, + }; + + srv = await startServer({ + port: 0, + databaseUrl: resolveTestDatabaseUrl(), + apiBaseUrl: "http://localhost", // OAuth callbacks unused (token injection) + authSchema, + coreSchema, + migrate: false, // harness already migrated + enableCleanupCron: false, + workerCount: 1, + workerIdleDelayMs: 250, // poll the embed queue fast + workerRefreshIntervalMs: 500, // discover new spaces fast + embeddingConfig, + }); + + tmpHome = await mkdtemp(join(tmpdir(), "me-e2e-")); + }); + + afterAll(async () => { + await srv?.stop(); + // Drop the space schemas this run created (enumerating core.space covers + // CLI-created spaces too), then the auth/core test schemas. + if (sql && coreSchema) { + const spaces = await sql.unsafe(`select slug from ${coreSchema}.space`); + for (const row of spaces) { + await sql.unsafe(`drop schema if exists metest_${row.slug} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); + } + if (tmpHome) await rm(tmpHome, { recursive: true, force: true }); + }); + + // ------------------------------------------------------------------------- + // CLI subprocess helpers + // ------------------------------------------------------------------------- + + function cliEnv(extra: Record = {}): Record { + const env = { ...process.env } as Record; + // Curate: drop any ambient ME_* so the dev's shell can't leak in. + for (const k of [ + "ME_API_KEY", + "ME_SERVER", + "ME_SPACE", + "ME_SESSION_TOKEN", + ]) { + delete env[k]; + } + return { + ...env, + HOME: tmpHome, + XDG_CONFIG_HOME: join(tmpHome, ".config"), + ME_NO_KEYCHAIN: "1", + ME_SERVER: srv.url, + ME_SESSION_TOKEN: token, + ME_SPACE: spaceSlug, + ...extra, + }; + } + + async function me( + args: string[], + extraEnv?: Record, + cwd?: string, + ): Promise<{ stdout: string; stderr: string; code: number }> { + const proc = Bun.spawn([process.execPath, CLI, ...args], { + env: cliEnv(extraEnv), + stdout: "pipe", + stderr: "pipe", + ...(cwd ? { cwd } : {}), + }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const code = await proc.exited; + return { stdout, stderr, code }; + } + + // Like `me`, but pipes `input` to the process's stdin (for `me claude hook`, + // which reads the event JSON from stdin). + async function meStdin( + args: string[], + input: string, + extraEnv?: Record, + ): Promise<{ stdout: string; stderr: string; code: number }> { + const proc = Bun.spawn([process.execPath, CLI, ...args], { + env: cliEnv(extraEnv), + stdin: new TextEncoder().encode(input), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const code = await proc.exited; + return { stdout, stderr, code }; + } + + // Count memories under a tree in this run's space schema. + async function countUnder(treePrefix: string): Promise { + const [row] = await sql.unsafe( + `select count(*)::int as n from metest_${spaceSlug}.memory + where tree <@ $1::ltree`, + [treePrefix], + ); + return (row?.n as number) ?? 0; + } + + // Count memories captured from a given source session id. + async function countBySession(sessionId: string): Promise { + const [row] = await sql.unsafe( + `select count(*)::int as n from metest_${spaceSlug}.memory + where meta->>'source_session_id' = $1`, + [sessionId], + ); + return (row?.n as number) ?? 0; + } + + // Parse the --json stdout of a `me` invocation, asserting success. + async function meJson( + args: string[], + extraEnv?: Record, + ): Promise { + const r = await me([...args, "--json"], extraEnv); + expect( + r.code, + `expected exit 0 for \`me ${args.join(" ")}\`\nstdout: ${r.stdout}\nstderr: ${r.stderr}`, + ).toBe(0); + return JSON.parse(r.stdout) as T; + } + + // Poll the space schema until N memories have a non-null embedding. + async function waitForEmbeddings(count: number, timeoutMs = 30000) { + const schema = `metest_${spaceSlug}`; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const [row] = await sql.unsafe( + `select count(*)::int as n from ${schema}.memory where embedding is not null`, + ); + if ((row?.n ?? 0) >= count) return; + await Bun.sleep(250); + } + throw new Error(`timed out waiting for ${count} embeddings`); + } + + // ------------------------------------------------------------------------- + // Core scenarios + // ------------------------------------------------------------------------- + + test("1. whoami reports the provisioned identity", async () => { + const r = await me(["whoami"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("e2e@example.test"); + }); + + test("2. create + tree round-trip (share namespace)", async () => { + const created = await meJson<{ id: string; tree?: string }>([ + "create", + "the quick brown fox jumps over the lazy dog", + "--tree", + "share", + ]); + expect(created.id).toBeTruthy(); + + const r = await me(["memory", "tree"]); + expect(r.code).toBe(0); + expect(r.stdout.toLowerCase()).toContain("share"); + }); + + test("3. fulltext (BM25) search finds the memory", async () => { + const res = await meJson<{ + total: number; + results: { id: string; content: string }[]; + }>(["search", "--fulltext", "fox"]); + expect(res.total).toBeGreaterThan(0); + expect(res.results.some((m) => m.content.includes("quick brown fox"))).toBe( + true, + ); + }); + + test("4. semantic search ranks a paraphrase near the top", async () => { + // Seed a few more memories to make ranking meaningful. + const seed = (text: string) => meJson(["create", text, "--tree", "share"]); + await seed("a dog chased a cat across the yard"); + await seed("the stock market fell sharply on Tuesday"); + await seed("photosynthesis converts sunlight into energy"); + + // 4 created so far in `share` (1 from scenario 2 + 3 here). Wait for the + // worker to embed them. + await waitForEmbeddings(4); + + const res = await meJson<{ + results: { id: string; content: string }[]; + }>(["search", "--semantic", "wild canine leaps over a sleepy hound"]); + // Recall-based: the fox/dog memories should surface near the top, not the + // stock-market or photosynthesis ones. Assert a relevant item is in top-3. + const top3 = res.results.slice(0, 3).map((m) => m.content); + expect(top3.some((c) => c.includes("fox") || c.includes("dog"))).toBe(true); + }); + + test("5. tree paths reflect ~ (home) and share conventions", async () => { + await meJson(["create", "personal note", "--tree", "~/notes"]); + await meJson(["create", "team note", "--tree", "share/team"]); + + const r = await me(["memory", "tree"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("notes"); + expect(r.stdout).toContain("team"); + }); + + test("6. update + delete round-trip", async () => { + const created = await meJson<{ id: string }>([ + "create", + "ephemeral memory to edit", + "--tree", + "share", + ]); + const updated = await meJson<{ id: string; content: string }>([ + "memory", + "update", + created.id, + "--content", + "edited content", + ]); + expect(updated.content).toBe("edited content"); + + const del = await me(["memory", "delete", created.id, "--yes"]); + expect(del.code).toBe(0); + + // Getting it now fails with a non-zero exit. + const get = await me(["memory", "get", created.id]); + expect(get.code).not.toBe(0); + }); + + // ------------------------------------------------------------------------- + // Extended scenarios + // ------------------------------------------------------------------------- + + test("7. api-key auth works end-to-end (no session token)", async () => { + // Mint the key through the real CLI: create the agent, add it to the + // space, then mint a key for it. + const agent = await meJson<{ id: string }>([ + "agent", + "create", + `bot-${rand()}`, + ]); + await me(["agent", "add", agent.id]); // bring the agent into the space + // Agents join with no grant (their access is clamped to the owner's), so + // grant read on `share` — where the fox memory lives — to make it readable. + await meJson(["access", "grant", agent.id, "share", "r"]); + const key = await meJson<{ id: string; key: string }>([ + "apikey", + "create", + agent.id, + ]); + expect(key.key).toMatch(/^me\./); + + // Search with ONLY the api key — no session token. The agent's global key + // plus X-Me-Space (ME_SPACE) selects the space; this exercises the CLI's + // api-key auth path against the real server end-to-end. + const res = await meJson<{ total: number }>( + ["search", "--fulltext", "fox"], + { ME_API_KEY: key.key, ME_SESSION_TOKEN: "" }, + ); + expect(res.total).toBeGreaterThan(0); + }); + + test("8. `me claude import` backfills work that predates the hook", async () => { + // The scenario: a user does a bunch of Claude Code work BEFORE installing + // the capture hook (no hook fires for it), then installs the hook (which + // begins capturing new sessions live), then runs `me claude import`. The + // pre-install work must be backfilled — the importer has no lower time + // bound tied to hook install; it sweeps every transcript and dedupes by + // deterministic message id. + const root = await mkdtemp(join(tmpdir(), "me-e2e-backfill-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + + // cwd "/work/backfill-proj" → no git repo on disk → slug = basename, so + // both sessions land under the same tree. + const cwd = "/work/backfill-proj"; + const tree = "share.projects.backfill_proj.agent_sessions"; + + const mkMsg = + (sessionId: string) => + (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-02-01T00:00:0${i}.000Z`, + sessionId, + cwd, + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + + const writeTranscript = async (sessionId: string, prefix: string) => { + const m = mkMsg(sessionId); + const lines = [ + m(0, "user", `${prefix} first question`), + m(1, "assistant", `${prefix} first answer`), + m(2, "user", `${prefix} second question`), + m(3, "assistant", `${prefix} second answer`), + ]; + const path = join(projDir, `${sessionId}.jsonl`); + await writeFile(path, lines.map((l) => JSON.stringify(l)).join("\n")); + return path; + }; + + // 1. Pre-install work: a transcript sits on disk; NO hook ever fires for + // it. It must not be in the engine yet. + const oldSession = `pre-install-${rand()}`; + await writeTranscript(oldSession, "old"); + expect(await countBySession(oldSession)).toBe(0); + + // 2. Install the hook (it now captures live) and let it import a NEW + // session — the real `me claude hook` path, reading from stdin. + const newSession = `post-install-${rand()}`; + const newTranscript = await writeTranscript(newSession, "new"); + const hook = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ + transcript_path: newTranscript, + session_id: newSession, + }), + ); + expect(hook.code, hook.stderr).toBe(0); + // The hook captured only the post-install session — the old one is still + // absent (this is exactly the gap `me claude import` must close). + expect(await countBySession(newSession)).toBe(4); + expect(await countBySession(oldSession)).toBe(0); + + // 3. Run the import (canonical spelling; test 9 covers the + // `me claude import` alias). + const imp = await me(["import", "claude", "--source", root]); + expect(imp.code, imp.stderr).toBe(0); + + // 4. The pre-install work is now backfilled, and the hook's live capture + // was not duplicated. + expect(await countBySession(oldSession)).toBe(4); + expect(await countBySession(newSession)).toBe(4); + expect(await countUnder(tree)).toBe(8); + + await rm(root, { recursive: true, force: true }); + }); + + test("8b. `me claude init` backfills sessions and writes a CLAUDE.md pointer", async () => { + // `me claude init` is the one-shot setup command. Two steps exercised + // here: + // 1. import THIS project's existing sessions (sessions whose recorded + // cwd is at/under init's cwd — init is per-project setup; the + // machine-wide sweep is `me import claude`); + // 2. record the project's memory location in the project's CLAUDE.md + // (the project = init's cwd; not a git repo here → CLAUDE.md lands in + // that dir, slug = its basename). + // The project we run `init` in — a non-git temp dir with a known basename + // so the derived slug is predictable. CLAUDE.md will be written here, and + // the transcript's session records this dir as its cwd, so the session + // tree and the CLAUDE.md pointer name the same project. + const projectRoot = await mkdtemp(join(tmpdir(), "me-e2e-initcwd-")); + const projectDir = join(projectRoot, "initcwd"); + await mkdir(projectDir, { recursive: true }); + // The recorded session cwd must be the REAL path (as Claude Code would + // record it): macOS tmpdir is a symlink (/var/folders → /private/var), + // and init filters against the resolved process.cwd(). + const projectCwd = await realpath(projectDir); + + const sessionId = `init-${rand()}`; + const foreignId = `foreign-${rand()}`; + const tree = "share.projects.initcwd.agent_sessions"; + const mkMsg = ( + sid: string, + cwd: string, + i: number, + type: "user" | "assistant", + text: string, + ) => ({ + type, + uuid: `${sid}-${type}-${i}`, + timestamp: `2026-03-01T00:00:0${i}.000Z`, + sessionId: sid, + cwd, + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + // Mirror Claude Code's on-disk layout: each transcript lives in a + // directory named after the session cwd (encoded) — the scoped import + // prunes by that name, so a literal fixture dir would never be scanned. + const writeTranscript = async ( + sid: string, + cwd: string, + prefix: string, + ) => { + const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); + await mkdir(dir, { recursive: true }); + await writeFile( + join(dir, `${sid}.jsonl`), + [ + mkMsg(sid, cwd, 0, "user", `${prefix} first question`), + mkMsg(sid, cwd, 1, "assistant", `${prefix} first answer`), + mkMsg(sid, cwd, 2, "user", `${prefix} second question`), + mkMsg(sid, cwd, 3, "assistant", `${prefix} second answer`), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + }; + await writeTranscript(sessionId, projectCwd, "init"); + // A session from a DIFFERENT project must not be swept up by init. + await writeTranscript(foreignId, "/work/other-proj", "foreign"); + + // Pre-init: nothing captured, no CLAUDE.md. + expect(await countBySession(sessionId)).toBe(0); + + // Run `init` FROM the project dir so its cwd → slug → CLAUDE.md location. + const init = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + projectDir, + ); + expect(init.code, init.stderr).toBe(0); + + // Step 1: this project's session was backfilled; the foreign one wasn't. + expect(await countBySession(sessionId)).toBe(4); + expect(await countUnder(tree)).toBe(4); + expect(await countBySession(foreignId)).toBe(0); + + // Step 2: CLAUDE.md now points at this project's memories. + const claudeMd = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); + expect(claudeMd).toContain("memory-engine:start"); + expect(claudeMd).toContain("share.projects.initcwd"); + expect(claudeMd).toContain("share.projects.initcwd.agent_sessions"); + expect(claudeMd).toContain("share.projects.initcwd.git_history"); + + // Re-running is idempotent: still exactly one managed block. + const init2 = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + projectDir, + ); + expect(init2.code, init2.stderr).toBe(0); + const claudeMd2 = await readFile(join(projectDir, "CLAUDE.md"), "utf8"); + expect(claudeMd2.split("memory-engine:start").length - 1).toBe(1); + + for (const cwd of [projectCwd, "/work/other-proj"]) { + await rm(join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)), { + recursive: true, + force: true, + }); + } + await rm(projectRoot, { recursive: true, force: true }); + }); + + test("8c. `me claude init` honors --skip-transcript-import / --skip-claude-md", async () => { + // Non-interactive (piped) init runs every step except those turned off by + // a --skip- flag. Verify each flag suppresses exactly its step. + // Each case gets its own project dir + a transcript recorded IN that dir + // (init's import is scoped to the project it runs in), stored under the + // Claude Code encoded-cwd directory layout the scoped import prunes by. + const transcriptDirs: string[] = []; + const mkProject = async (name: string) => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-skip-")); + const dir = join(root, name); + await mkdir(dir, { recursive: true }); + return { root, dir }; + }; + const writeTranscript = async (sid: string, cwd: string) => { + const mkMsg = (i: number, type: "user" | "assistant") => ({ + type, + uuid: `${sid}-${type}-${i}`, + timestamp: `2026-04-01T00:00:0${i}.000Z`, + sessionId: sid, + cwd, + message: + type === "user" + ? { content: `q${i}` } + : { + content: [{ type: "text", text: `a${i}` }], + model: "claude-x", + }, + }); + const dir = join(tmpHome, ".claude", "projects", encodeProjectDir(cwd)); + transcriptDirs.push(dir); + await mkdir(dir, { recursive: true }); + await writeFile( + join(dir, `${sid}.jsonl`), + [ + mkMsg(0, "user"), + mkMsg(1, "assistant"), + mkMsg(2, "user"), + mkMsg(3, "assistant"), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + }; + + // --skip-transcript-import: CLAUDE.md is written, but this project's + // session is NOT imported (it would have been without the flag). + const a = await mkProject("skipimport"); + const sessionA = `skipa-${rand()}`; + await writeTranscript(sessionA, await realpath(a.dir)); + const r1 = await me( + ["claude", "init", "--skip-transcript-import", "--skip-plugin-install"], + undefined, + a.dir, + ); + expect(r1.code, r1.stderr).toBe(0); + expect(await countBySession(sessionA)).toBe(0); + expect(existsSync(join(a.dir, "CLAUDE.md"))).toBe(true); + + // --skip-claude-md: the project's session imports, but no CLAUDE.md. + const b = await mkProject("skipclaudemd"); + const sessionB = `skipb-${rand()}`; + await writeTranscript(sessionB, await realpath(b.dir)); + const r2 = await me( + ["claude", "init", "--skip-claude-md", "--skip-plugin-install"], + undefined, + b.dir, + ); + expect(r2.code, r2.stderr).toBe(0); + expect(await countBySession(sessionB)).toBe(4); + expect(existsSync(join(b.dir, "CLAUDE.md"))).toBe(false); + + for (const dir of transcriptDirs) { + await rm(dir, { recursive: true, force: true }); + } + await rm(a.root, { recursive: true, force: true }); + await rm(b.root, { recursive: true, force: true }); + }); + + // Run git in `dir`, isolated from the developer's git config (gpg + // signing, hooks, templates), with deterministic commit dates. + async function git( + dir: string, + args: string[], + dateIso?: string, + extraEnv?: Record, + ): Promise { + const proc = Bun.spawn( + [ + "git", + "-C", + dir, + "-c", + "user.name=E2E", + "-c", + "user.email=e2e@example.test", + "-c", + "commit.gpgsign=false", + ...args, + ], + { + env: { + ...process.env, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + ...(dateIso + ? { GIT_AUTHOR_DATE: dateIso, GIT_COMMITTER_DATE: dateIso } + : {}), + // Spawned hooks inherit this env — the git-hook test merges + // cliEnv() here so the hook's `me import git` can reach the server. + ...(extraEnv ?? {}), + }, + stdout: "pipe", + stderr: "pipe", + }, + ); + const code = await proc.exited; + if (code !== 0) { + const stderr = await new Response(proc.stderr).text(); + throw new Error(`git ${args.join(" ")} failed: ${stderr}`); + } + } + + test("8d. `me import git` imports commit history, idempotently and incrementally", async () => { + // A real repo with a known-basename root so the slug (no remote → + // basename) and therefore the tree are predictable. + const root = await mkdtemp(join(tmpdir(), "me-e2e-git-")); + const name = `gitproj${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + const commitFile = async (file: string, msg: string, dateIso: string) => { + await writeFile(join(repo, file), `${msg}\n`); + await git(repo, ["add", "."], dateIso); + await git(repo, ["commit", "-q", "-m", msg], dateIso); + }; + await commitFile("a.txt", "feat: add a", "2026-05-01T10:00:00Z"); + await commitFile("b.txt", "fix: adjust b", "2026-05-02T10:00:00Z"); + await commitFile("c.txt", "docs: describe c", "2026-05-03T10:00:00Z"); + + // 1. First import: all three commits land under the project tree. + const first = await meJson<{ + inserted: number; + commitsWalked: number; + tree: string; + }>(["import", "git", repo]); + expect(first.tree).toBe(tree); + expect(first.commitsWalked).toBe(3); + expect(first.inserted).toBe(3); + expect(await countUnder(tree)).toBe(3); + + // Spot-check one record's shape: type/sha meta + commit-date temporal + + // file list in the content. + const [row] = await sql.unsafe( + `select content, meta from metest_${spaceSlug}.memory + where tree = $1::ltree and content like 'fix: adjust b%'`, + [tree], + ); + expect(row?.meta?.type).toBe("git_commit"); + expect(row?.meta?.sha).toMatch(/^[0-9a-f]{40}$/); + expect(row?.meta?.author_email).toBe("e2e@example.test"); + expect(row?.content).toContain("Files:"); + expect(row?.content).toContain("b.txt (+1 -0)"); + + // 2. Plain re-run: the high-water commit is HEAD → incremental walk of + // an empty range; nothing re-sent, nothing duplicated. + const rerun = await meJson<{ inserted: number; commitsWalked: number }>([ + "import", + "git", + repo, + ]); + expect(rerun.commitsWalked).toBe(0); + expect(rerun.inserted).toBe(0); + expect(await countUnder(tree)).toBe(3); + + // 3. --full re-run: walks everything; deterministic ids make the server + // skip every row (`ON CONFLICT DO NOTHING`). + const full = await meJson<{ + inserted: number; + skipped: number; + commitsWalked: number; + }>(["import", "git", "--full", repo]); + expect(full.commitsWalked).toBe(3); + expect(full.inserted).toBe(0); + expect(full.skipped).toBe(3); + expect(await countUnder(tree)).toBe(3); + + // 4. New work: one regular commit + one body-less merge. The next plain + // run walks only the new range, imports the commit, and drops the + // boilerplate merge. + await git(repo, ["checkout", "-q", "-b", "feat"], undefined); + await commitFile("d.txt", "feat: add d", "2026-05-04T10:00:00Z"); + await git(repo, ["checkout", "-q", "main"]); + await git( + repo, + ["merge", "-q", "--no-ff", "feat", "-m", "Merge branch 'feat'"], + "2026-05-05T10:00:00Z", + ); + const incr = await meJson<{ + inserted: number; + commitsWalked: number; + skippedMerges: number; + range?: string; + }>(["import", "git", repo]); + expect(incr.range).toMatch(/^[0-9a-f]{40}\.\.HEAD$/); + expect(incr.commitsWalked).toBe(2); + expect(incr.inserted).toBe(1); + expect(incr.skippedMerges).toBe(1); + expect(await countUnder(tree)).toBe(4); + + await rm(root, { recursive: true, force: true }); + }); + + test("8e. `me claude init` runs the git step; --skip-git-import suppresses it", async () => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-gitinit-")); + const name = `gitinit${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + await writeFile(join(repo, "x.txt"), "x\n"); + await git(repo, ["add", "."], "2026-05-01T10:00:00Z"); + await git( + repo, + ["commit", "-q", "-m", "feat: initial"], + "2026-05-01T10:00:00Z", + ); + + // --skip-git-import: no commit memories. + const skipped = await me( + ["claude", "init", "--skip-git-import", "--skip-plugin-install"], + undefined, + repo, + ); + expect(skipped.code, skipped.stderr).toBe(0); + expect(await countUnder(tree)).toBe(0); + + // Plain init (non-interactive baseline) imports the repo's history and + // the CLAUDE.md pointer names the git_history node. + const init = await me( + ["claude", "init", "--skip-plugin-install"], + undefined, + repo, + ); + expect(init.code, init.stderr).toBe(0); + expect(await countUnder(tree)).toBe(1); + const claudeMd = await readFile(join(repo, "CLAUDE.md"), "utf8"); + expect(claudeMd).toContain(`${tree}\``); + + await rm(root, { recursive: true, force: true }); + }); + + test("8f. `me import git-hook` captures new commits via post-commit", async () => { + const root = await mkdtemp(join(tmpdir(), "me-e2e-githook-")); + const name = `githook${rand()}`; + const repo = join(root, name); + await mkdir(repo, { recursive: true }); + const tree = `share.projects.${name}.git_history`; + + await git(repo, ["init", "-q", "-b", "main"]); + await writeFile(join(repo, "a.txt"), "a\n"); + await git(repo, ["add", "."], "2026-05-01T10:00:00Z"); + await git( + repo, + ["commit", "-q", "-m", "feat: first"], + "2026-05-01T10:00:00Z", + ); + + // Install: managed block written, executable, embeds the source invocation. + const install = await me(["import", "git-hook", repo]); + expect(install.code, install.stderr).toBe(0); + const hookFile = join(repo, ".git", "hooks", "post-commit"); + const hook = await readFile(hookFile, "utf8"); + expect(hook).toContain(">>> memory-engine"); + expect(hook).toContain("import git"); + expect(hook).toContain(process.execPath); // bun + index.ts invocation + const { mode } = await stat(hookFile); + expect(mode & 0o111).not.toBe(0); + + // Re-install is idempotent: still exactly one managed block. + const again = await me(["import", "git-hook", repo]); + expect(again.code, again.stderr).toBe(0); + const hook2 = await readFile(hookFile, "utf8"); + expect(hook2.split(">>> memory-engine").length - 1).toBe(1); + + // A commit fires the hook; its background incremental import catches up + // the whole history (both commits). The hook child inherits the commit's + // env, so merge cliEnv() in. + await writeFile(join(repo, "b.txt"), "b\n"); + await git(repo, ["add", "."], "2026-05-02T10:00:00Z", cliEnv()); + await git( + repo, + ["commit", "-q", "-m", "feat: second"], + "2026-05-02T10:00:00Z", + cliEnv(), + ); + const deadline = Date.now() + 30000; + while ((await countUnder(tree)) < 2 && Date.now() < deadline) { + await Bun.sleep(250); + } + expect(await countUnder(tree)).toBe(2); + + // --remove deletes the managed block (and here the whole file, since the + // block was its only content). + const removed = await me(["import", "git-hook", "--remove", repo]); + expect(removed.code, removed.stderr).toBe(0); + expect(existsSync(hookFile)).toBe(false); + + await rm(root, { recursive: true, force: true }); + }); + + test("9. claude capture hook ↔ `me claude import` are cross-idempotent", async () => { + // A minimal Claude Code session transcript on disk. The importer scans + // //*.jsonl; the hook reads the file directly. + const sessionId = `xact-${rand()}`; + const root = await mkdtemp(join(tmpdir(), "me-e2e-transcript-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + const transcript = join(projDir, `${sessionId}.jsonl`); + // Two user turns so the importer doesn't skip it as a trivial session + // (the hook captures regardless; this makes both paths process all four). + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-02-01T00:00:0${i}.000Z`, + sessionId, + cwd: "/work/idempotent-proj", + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + const lines = [ + mkMsg(0, "user", "first question"), + mkMsg(1, "assistant", "first answer"), + mkMsg(2, "user", "second question"), + mkMsg(3, "assistant", "second answer"), + ]; + await writeFile(transcript, lines.map((l) => JSON.stringify(l)).join("\n")); + + // cwd "/work/idempotent-proj" → no git repo on disk → slug = basename. + const tree = "share.projects.idempotent_proj.agent_sessions"; + + // 1. Live capture via the real hook (reads transcript_path from stdin, + // auths with the session, writes via importTranscriptFile). + const hook = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ transcript_path: transcript, session_id: sessionId }), + ); + expect(hook.code, hook.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + // 2. `me claude import` over the SAME transcript → no new rows (same tree + + // deterministic ids ⇒ the importer dedupes against the hook's writes). + const imp = await me(["claude", "import", "--source", root]); + expect(imp.code, imp.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + // 3. Re-run the hook → still idempotent. + const hook2 = await meStdin( + ["claude", "hook", "--event", "stop"], + JSON.stringify({ transcript_path: transcript, session_id: sessionId }), + ); + expect(hook2.code, hook2.stderr).toBe(0); + expect(await countUnder(tree)).toBe(4); + + await rm(root, { recursive: true, force: true }); + }); + + test("9b. a stale importer_version is re-rendered in place on re-import", async () => { + // The server's conditional upsert: re-importing a session rewrites any + // row whose stored meta.importer_version differs from the current + // importer's, and skips the rest — no client-side existing-state read. + const sessionId = `stale-${rand()}`; + const root = await mkdtemp(join(tmpdir(), "me-e2e-stale-")); + const projDir = join(root, "proj"); + await mkdir(projDir, { recursive: true }); + const mkMsg = (i: number, type: "user" | "assistant", text: string) => ({ + type, + uuid: `${sessionId}-${type}-${i}`, + timestamp: `2026-05-01T00:00:0${i}.000Z`, + sessionId, + cwd: "/work/stale-proj", + message: + type === "user" + ? { content: text } + : { content: [{ type: "text", text }], model: "claude-x" }, + }); + await writeFile( + join(projDir, `${sessionId}.jsonl`), + [ + mkMsg(0, "user", "stale first question"), + mkMsg(1, "assistant", "stale first answer"), + mkMsg(2, "user", "stale second question"), + mkMsg(3, "assistant", "stale second answer"), + ] + .map((l) => JSON.stringify(l)) + .join("\n"), + ); + + const first = await meJson<{ inserted: number }>([ + "import", + "claude", + "--source", + root, + ]); + expect(first.inserted).toBe(4); + + // Rewind one row to look like an older importer build wrote it. + const [stale] = await sql.unsafe( + `update metest_${spaceSlug}.memory + set content = 'STALE RENDER', + meta = jsonb_set(meta, '{importer_version}', '"0"') + where meta->>'source_session_id' = $1 + and meta->>'source_message_id' = $2 + returning id`, + [sessionId, `${sessionId}-user-0`], + ); + expect(stale?.id).toBeDefined(); + + // Re-import: exactly the stale row is rewritten, the rest skip. + const second = await meJson<{ + inserted: number; + updated: number; + skipped: number; + failed: number; + }>(["import", "claude", "--source", root]); + expect(second.inserted).toBe(0); + expect(second.updated).toBe(1); + expect(second.skipped).toBe(3); + expect(second.failed).toBe(0); + + const [row] = await sql.unsafe( + `select content, meta->>'importer_version' as v + from metest_${spaceSlug}.memory where id = $1`, + [stale?.id as string], + ); + expect(row?.content).toBe("stale first question"); + expect(row?.v).toBe("1"); + + await rm(root, { recursive: true, force: true }); + }); + + test("9c. `me import` group: no bare default, memories ≡ memory import", async () => { + // Bare `me import` is a group, not the old file-import alias: it prints + // the subcommand list and exits non-zero. + const bare = await me(["import"]); + expect(bare.code).not.toBe(0); + expect(bare.stdout + bare.stderr).toContain("memories"); + + // Old muscle memory `me import ` no longer parses. + const fileArg = await me(["import", "nosuch.md"]); + expect(fileArg.code).not.toBe(0); + expect(fileArg.stderr).toContain("unknown command"); + + // The file importer lives at `me import memories`, with + // `me memory import` as its alias — both write the same records. + const record = (i: number) => + JSON.stringify({ + content: `import group probe ${i}`, + tree: "share.importgroup", + }); + const viaGroup = await meStdin(["import", "memories", "-"], record(1)); + expect(viaGroup.code, viaGroup.stderr).toBe(0); + const viaAlias = await meStdin(["memory", "import", "-"], record(2)); + expect(viaAlias.code, viaAlias.stderr).toBe(0); + expect(await countUnder("share.importgroup")).toBe(2); + }); + + test("10. failure modes: bad space and missing auth exit non-zero", async () => { + const badSpace = await me(["search", "--fulltext", "fox"], { + ME_SPACE: "doesnotexist1", + }); + expect(badSpace.code).not.toBe(0); + + const noAuth = await me(["whoami"], { ME_SESSION_TOKEN: "" }); + expect(noAuth.code).not.toBe(0); + }); +}); diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 0000000..af57ff7 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,12 @@ +{ + "name": "@memory.build/e2e", + "version": "0.2.5", + "private": true, + "type": "module", + "dependencies": { + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", + "@memory.build/embedding": "workspace:*", + "postgres": "^3.4.9" + } +} diff --git a/package.json b/package.json index 29e8f2f..16a4561 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,14 @@ "private": true, "workspaces": [ "packages/*", - "scripts" + "scripts", + "e2e" ], "scripts": { "build": "./bun run --filter '@memory.build/cli' build", "build:all": "./bun scripts/build-all.ts", - "check": "./bun i --silent && ./bun run typecheck && ./bun run lint --write && ./bun run test --only-failures", + "check": "./bun i --silent && ./bun scripts/bundle-web-assets.ts && ./bun run typecheck && ./bun run lint --write && ./bun run test:unit", + "check:full": "./bun run check && ./bun run test --only-failures && TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-postgresql://postgres@127.0.0.1:5432/postgres}\" ./bun run test:e2e", "clean": "rm -rf packages/cli/dist dist", "docs": "./bun --filter @memory.build/docs-site dev", "docs:build": "./bun --filter @memory.build/docs-site build", @@ -21,6 +23,7 @@ "install:local": "./bun scripts/install-local.ts", "lint": "biome check", "me": "./bun run packages/cli/index.ts", + "migrate:db": "./bun scripts/migrate-db.ts", "pg": "./bun run pg:build && docker run -d --name me-postgres -e POSTGRES_HOST_AUTH_METHOD=trust -p 127.0.0.1:5432:5432 me-postgres", "pg:build": "docker build -t me-postgres -f docker/Dockerfile.postgres docker/", "pg:rm": "docker rm -f me-postgres", @@ -29,7 +32,12 @@ "release:server": "./bun scripts/release-server.ts", "server": "./bun run packages/server/index.ts", "setup": "./bun scripts/setup.ts", - "test": "./bun test packages", + "test": "TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-postgresql://postgres@127.0.0.1:5432/postgres}\" ./bun test packages --timeout 30000 --parallel=2", + "test:db": "./bun run test:db:clean && find packages -name '*.integration.test.ts' -print0 | xargs -0 ./bun test --parallel=2 --timeout 30000", + "test:db:clean": "./bun scripts/clean-test-schemas.ts", + "test:db:clean:all": "./bun scripts/clean-test-schemas.ts --all", + "test:e2e": "./bun run test:db:clean && ./bun test --timeout 60000 ./e2e", + "test:unit": "find packages -name '*.test.ts' ! -name '*.integration.test.ts' -print0 | xargs -0 ./bun test --parallel=2", "typecheck": "tsc --noEmit" }, "devDependencies": { diff --git a/packages/accounts/db.integration.test.ts b/packages/accounts/db.integration.test.ts deleted file mode 100644 index f15700a..0000000 --- a/packages/accounts/db.integration.test.ts +++ /dev/null @@ -1,684 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { type AccountsDB, createAccountsDB } from "./db"; -import { TestDatabase } from "./migrate/test-utils"; -import { AccountsError } from "./types"; - -// Test master key (32 bytes for AES-256) -const TEST_MASTER_KEY = Buffer.from( - "0123456789abcdef0123456789abcdef", - "utf-8", -); - -let testDb: TestDatabase; -let db: AccountsDB; - -beforeAll(async () => { - testDb = await TestDatabase.create(); - - db = createAccountsDB(testDb.sql, testDb.schema, { - masterKey: TEST_MASTER_KEY, - }); - - // Create and activate an encryption key for tests - const keyId = await db.createDataKey(); - await db.activateDataKey(keyId); -}); - -afterAll(async () => { - await testDb.dispose(); -}); - -// --------------------------------------------------------------------------- -// Identity tests -// --------------------------------------------------------------------------- - -describe("identity", () => { - test("create and get identity", async () => { - const identity = await db.createIdentity({ - email: "test@example.com", - name: "Test User", - }); - - expect(identity.id).toBeDefined(); - expect(identity.email).toBe("test@example.com"); - expect(identity.name).toBe("Test User"); - - const fetched = await db.getIdentity(identity.id); - expect(fetched).toEqual(identity); - }); - - test("get identity by email", async () => { - const identity = await db.createIdentity({ - email: "byemail@example.com", - name: "By Email", - }); - - const fetched = await db.getIdentityByEmail("byemail@example.com"); - expect(fetched?.id).toBe(identity.id); - }); - - test("update identity", async () => { - const identity = await db.createIdentity({ - email: "update@example.com", - name: "Original Name", - }); - - const updated = await db.updateIdentity(identity.id, { name: "New Name" }); - expect(updated).toBe(true); - - const fetched = await db.getIdentity(identity.id); - expect(fetched?.name).toBe("New Name"); - }); - - test("delete identity", async () => { - const identity = await db.createIdentity({ - email: "delete@example.com", - name: "To Delete", - }); - - const deleted = await db.deleteIdentity(identity.id); - expect(deleted).toBe(true); - - const fetched = await db.getIdentity(identity.id); - expect(fetched).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// Org tests -// --------------------------------------------------------------------------- - -describe("org", () => { - test("create and get org", async () => { - const org = await db.createOrg({ - name: "Test Organization", - }); - - expect(org.id).toBeDefined(); - expect(org.slug).toMatch(/^[a-z0-9]{12}$/); - expect(org.name).toBe("Test Organization"); - - const fetched = await db.getOrg(org.id); - expect(fetched).toEqual(org); - }); - - test("get org by slug", async () => { - const org = await db.createOrg({ - name: "By Slug", - }); - - const fetched = await db.getOrgBySlug(org.slug); - expect(fetched?.id).toBe(org.id); - }); - - test("update org name", async () => { - const org = await db.createOrg({ name: "Original Name" }); - - const updated = await db.updateOrg(org.id, { name: "Renamed" }); - expect(updated).toBe(true); - - const fetched = await db.getOrg(org.id); - expect(fetched?.name).toBe("Renamed"); - // Slug should be unchanged - expect(fetched?.slug).toBe(org.slug); - // updated_at should be set - expect(fetched?.updatedAt).not.toBeNull(); - }); - - test("update org returns false for unknown id", async () => { - const result = await db.updateOrg("019d694f-79f6-7595-8faf-b70b01c11f98", { - name: "Whatever", - }); - expect(result).toBe(false); - }); - - test("update org with no fields is a no-op", async () => { - const org = await db.createOrg({ name: "No Fields" }); - const result = await db.updateOrg(org.id, {}); - expect(result).toBe(false); - - const fetched = await db.getOrg(org.id); - expect(fetched?.name).toBe("No Fields"); - }); - - test("list orgs by identity", async () => { - const identity = await db.createIdentity({ - email: "orglist@example.com", - name: "Org List User", - }); - - const org1 = await db.createOrg({ name: "Org 1" }); - const org2 = await db.createOrg({ name: "Org 2" }); - - await db.addMember(org1.id, identity.id, "owner"); - await db.addMember(org2.id, identity.id, "member"); - - const orgs = await db.listOrgsByIdentity(identity.id); - expect(orgs.length).toBe(2); - expect(orgs.map((o) => o.id).sort()).toEqual([org1.id, org2.id].sort()); - }); -}); - -// --------------------------------------------------------------------------- -// OrgMember tests -// --------------------------------------------------------------------------- - -describe("org-member", () => { - test("add and list members", async () => { - const org = await db.createOrg({ - name: "Member Test", - }); - const identity = await db.createIdentity({ - email: "member@example.com", - name: "Member", - }); - - const member = await db.addMember(org.id, identity.id, "admin"); - expect(member.orgId).toBe(org.id); - expect(member.identityId).toBe(identity.id); - expect(member.role).toBe("admin"); - - const members = await db.listMembers(org.id); - expect(members.length).toBe(1); - }); - - test("update role", async () => { - const org = await db.createOrg({ - name: "Role Update", - }); - const identity = await db.createIdentity({ - email: "roleupdate@example.com", - name: "Role Update", - }); - - await db.addMember(org.id, identity.id, "member"); - await db.updateRole(org.id, identity.id, "admin"); - - const member = await db.getMember(org.id, identity.id); - expect(member?.role).toBe("admin"); - }); - - test("cannot remove last owner", async () => { - const org = await db.createOrg({ name: "Last Owner" }); - const identity = await db.createIdentity({ - email: "lastowner@example.com", - name: "Last Owner", - }); - - await db.addMember(org.id, identity.id, "owner"); - - await expect(db.removeMember(org.id, identity.id)).rejects.toThrow( - AccountsError, - ); - }); - - test("cannot demote last owner", async () => { - const org = await db.createOrg({ - name: "Demote Owner", - }); - const identity = await db.createIdentity({ - email: "demoteowner@example.com", - name: "Demote Owner", - }); - - await db.addMember(org.id, identity.id, "owner"); - - await expect(db.updateRole(org.id, identity.id, "admin")).rejects.toThrow( - AccountsError, - ); - }); - - test("can remove owner if another owner exists", async () => { - const org = await db.createOrg({ - name: "Multi Owner", - }); - const owner1 = await db.createIdentity({ - email: "owner1@example.com", - name: "Owner 1", - }); - const owner2 = await db.createIdentity({ - email: "owner2@example.com", - name: "Owner 2", - }); - - await db.addMember(org.id, owner1.id, "owner"); - await db.addMember(org.id, owner2.id, "owner"); - - const removed = await db.removeMember(org.id, owner1.id); - expect(removed).toBe(true); - - const owners = await db.listOwners(org.id); - expect(owners.length).toBe(1); - expect(owners[0]?.identityId).toBe(owner2.id); - }); - - test("delete org cascades to org_member even when sole owner exists", async () => { - // Regression: previously the org_member_owner_check trigger refused - // the cascade, making it impossible to delete an org you owned. The - // invariant now lives in removeMember/updateRole so the cascade - // proceeds unimpeded. - const org = await db.createOrg({ name: "Sole Owner Cascade" }); - const identity = await db.createIdentity({ - email: "sole-owner-cascade@example.com", - name: "Sole Owner", - }); - await db.addMember(org.id, identity.id, "owner"); - - const deleted = await db.deleteOrg(org.id); - expect(deleted).toBe(true); - - const members = await db.listMembers(org.id); - expect(members.length).toBe(0); - }); -}); - -// --------------------------------------------------------------------------- -// Engine tests -// --------------------------------------------------------------------------- - -describe("engine", () => { - test("create engine with generated slug", async () => { - const org = await db.createOrg({ name: "Engine Org" }); - - const engine = await db.createEngine({ - orgId: org.id, - name: "My Engine", - }); - - expect(engine.id).toBeDefined(); - expect(engine.slug).toMatch(/^[a-z0-9]{12}$/); - expect(engine.name).toBe("My Engine"); - expect(engine.status).toBe("active"); - expect(engine.shardId).toBe(1); - }); - - test("get engine by slug", async () => { - const org = await db.createOrg({ - name: "Engine Slug", - }); - const engine = await db.createEngine({ - orgId: org.id, - name: "Slug Engine", - }); - - const fetched = await db.getEngineBySlug(engine.slug); - expect(fetched?.id).toBe(engine.id); - }); - - test("update engine status", async () => { - const org = await db.createOrg({ name: "Status" }); - const engine = await db.createEngine({ - orgId: org.id, - name: "Status Engine", - }); - - await db.updateEngine(engine.id, { status: "suspended" }); - - const fetched = await db.getEngine(engine.id); - expect(fetched?.status).toBe("suspended"); - }); - - test("update engine name", async () => { - const org = await db.createOrg({ name: "Rename" }); - const engine = await db.createEngine({ - orgId: org.id, - name: "Old Engine Name", - }); - - const updated = await db.updateEngine(engine.id, { - name: "New Engine Name", - }); - expect(updated).toBe(true); - - const fetched = await db.getEngine(engine.id); - expect(fetched?.name).toBe("New Engine Name"); - // Slug must remain stable across renames - expect(fetched?.slug).toBe(engine.slug); - expect(fetched?.updatedAt).not.toBeNull(); - }); - - test("update engine name conflicts with sibling in same org", async () => { - const org = await db.createOrg({ name: "Sibling Conflict" }); - await db.createEngine({ orgId: org.id, name: "Existing" }); - const target = await db.createEngine({ orgId: org.id, name: "Target" }); - - // Renaming "Target" to "Existing" should hit the (org_id, name) unique - // constraint and throw a Postgres error. - await expect( - db.updateEngine(target.id, { name: "Existing" }), - ).rejects.toThrow(); - }); - - test("list engines by org", async () => { - const org = await db.createOrg({ - name: "List Engines", - }); - await db.createEngine({ orgId: org.id, name: "Engine A" }); - await db.createEngine({ orgId: org.id, name: "Engine B" }); - - const engines = await db.listEnginesByOrg(org.id); - expect(engines.length).toBe(2); - }); - - test("createEngine stores custom language", async () => { - const org = await db.createOrg({ name: "Lang Org" }); - - const engine = await db.createEngine({ - orgId: org.id, - name: "German Engine", - language: "german", - }); - - expect(engine.language).toBe("german"); - - const fetched = await db.getEngine(engine.id); - expect(fetched?.language).toBe("german"); - }); - - test("createEngine defaults language to english", async () => { - const org = await db.createOrg({ - name: "Default Lang Org", - }); - - const engine = await db.createEngine({ - orgId: org.id, - name: "Default Engine", - }); - - expect(engine.language).toBe("english"); - }); -}); - -// --------------------------------------------------------------------------- -// Session tests -// --------------------------------------------------------------------------- - -describe("session", () => { - test("create and validate session", async () => { - const identity = await db.createIdentity({ - email: "session@example.com", - name: "Session User", - }); - - const { session, rawToken } = await db.createSession({ - identityId: identity.id, - }); - - expect(session.identityId).toBe(identity.id); - expect(rawToken).toBeDefined(); - - const result = await db.validateSession(rawToken); - expect(result?.session.id).toBe(session.id); - expect(result?.identity.id).toBe(identity.id); - }); - - test("invalid token returns null", async () => { - const result = await db.validateSession("invalid-token"); - expect(result).toBeNull(); - }); - - test("delete session", async () => { - const identity = await db.createIdentity({ - email: "deletesession@example.com", - name: "Delete Session", - }); - - const { session, rawToken } = await db.createSession({ - identityId: identity.id, - }); - - await db.deleteSession(session.id); - - const result = await db.validateSession(rawToken); - expect(result).toBeNull(); - }); - - test("delete all sessions for identity", async () => { - const identity = await db.createIdentity({ - email: "allsessions@example.com", - name: "All Sessions", - }); - - await db.createSession({ identityId: identity.id }); - await db.createSession({ identityId: identity.id }); - - const count = await db.deleteSessionsByIdentity(identity.id); - expect(count).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// Invitation tests -// --------------------------------------------------------------------------- - -describe("invitation", () => { - test("create and find invitation by token", async () => { - const org = await db.createOrg({ name: "Invite Org" }); - const inviter = await db.createIdentity({ - email: "inviter@example.com", - name: "Inviter", - }); - - const { invitation, rawToken } = await db.createInvitation({ - orgId: org.id, - email: "invitee@example.com", - role: "member", - invitedBy: inviter.id, - }); - - expect(invitation.orgId).toBe(org.id); - expect(invitation.email).toBe("invitee@example.com"); - expect(rawToken).toBeDefined(); - - const found = await db.getInvitationByToken(rawToken); - expect(found?.id).toBe(invitation.id); - }); - - test("accept invitation", async () => { - const org = await db.createOrg({ name: "Accept Org" }); - const inviter = await db.createIdentity({ - email: "acceptinviter@example.com", - name: "Inviter", - }); - - const { invitation } = await db.createInvitation({ - orgId: org.id, - email: "acceptee@example.com", - role: "admin", - invitedBy: inviter.id, - }); - - const accepted = await db.acceptInvitation(invitation.id); - expect(accepted?.acceptedAt).toBeDefined(); - }); - - test("list pending invitations", async () => { - const org = await db.createOrg({ - name: "Pending Org", - }); - const inviter = await db.createIdentity({ - email: "pendinginviter@example.com", - name: "Inviter", - }); - - await db.createInvitation({ - orgId: org.id, - email: "pending1@example.com", - role: "member", - invitedBy: inviter.id, - }); - await db.createInvitation({ - orgId: org.id, - email: "pending2@example.com", - role: "member", - invitedBy: inviter.id, - }); - - const pending = await db.listPendingInvitations(org.id); - expect(pending.length).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// OAuth tests -// --------------------------------------------------------------------------- - -describe("oauth", () => { - test("link and get oauth account", async () => { - const identity = await db.createIdentity({ - email: "oauth@example.com", - name: "OAuth User", - }); - - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-123", - email: "oauth@example.com", - accessToken: "access-token-123", - refreshToken: "refresh-token-456", - }); - - expect(oauth.provider).toBe("github"); - expect(oauth.providerAccountId).toBe("gh-123"); - - const fetched = await db.getOAuthAccount("github", "gh-123"); - expect(fetched?.id).toBe(oauth.id); - }); - - test("get decrypted tokens", async () => { - const identity = await db.createIdentity({ - email: "oauthtokens@example.com", - name: "OAuth Tokens", - }); - - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "google", - providerAccountId: "google-abc", - accessToken: "my-access-token", - refreshToken: "my-refresh-token", - }); - - const tokens = await db.getOAuthTokens(oauth.id); - expect(tokens?.accessToken).toBe("my-access-token"); - expect(tokens?.refreshToken).toBe("my-refresh-token"); - }); - - test("refresh oauth tokens", async () => { - const identity = await db.createIdentity({ - email: "refreshtokens@example.com", - name: "Refresh Tokens", - }); - - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-refresh", - accessToken: "old-access", - refreshToken: "old-refresh", - }); - - await db.refreshOAuthTokens(oauth.id, { - accessToken: "new-access", - refreshToken: "new-refresh", - }); - - const tokens = await db.getOAuthTokens(oauth.id); - expect(tokens?.accessToken).toBe("new-access"); - expect(tokens?.refreshToken).toBe("new-refresh"); - }); - - test("list oauth accounts by identity", async () => { - const identity = await db.createIdentity({ - email: "multioauth@example.com", - name: "Multi OAuth", - }); - - await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-multi", - accessToken: "token1", - }); - await db.linkOAuthAccount({ - identityId: identity.id, - provider: "google", - providerAccountId: "google-multi", - accessToken: "token2", - }); - - const accounts = await db.getOAuthAccountsByIdentity(identity.id); - expect(accounts.length).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// Encryption key rotation test -// --------------------------------------------------------------------------- - -describe("encryption key rotation", () => { - test("can rotate encryption keys", async () => { - const identity = await db.createIdentity({ - email: "rotation@example.com", - name: "Rotation Test", - }); - - // Link with current key - const oauth = await db.linkOAuthAccount({ - identityId: identity.id, - provider: "github", - providerAccountId: "gh-rotation", - accessToken: "original-token", - }); - - // Create and activate new key - const newKeyId = await db.createDataKey(); - await db.activateDataKey(newKeyId); - - // Old tokens should still decrypt (using old key stored with them) - const tokens = await db.getOAuthTokens(oauth.id); - expect(tokens?.accessToken).toBe("original-token"); - - // New tokens will use the new key - await db.refreshOAuthTokens(oauth.id, { accessToken: "rotated-token" }); - - const newTokens = await db.getOAuthTokens(oauth.id); - expect(newTokens?.accessToken).toBe("rotated-token"); - }); -}); - -// --------------------------------------------------------------------------- -// Transaction test -// --------------------------------------------------------------------------- - -describe("transactions", () => { - test("withTransaction commits on success", async () => { - const result = await db.withTransaction(async (txDb) => { - const identity = await txDb.createIdentity({ - email: "txsuccess@example.com", - name: "TX Success", - }); - return identity; - }); - - const fetched = await db.getIdentity(result.id); - expect(fetched).toBeDefined(); - }); - - test("withTransaction rolls back on error", async () => { - const email = "txrollback@example.com"; - - try { - await db.withTransaction(async (txDb) => { - await txDb.createIdentity({ email, name: "TX Rollback" }); - throw new Error("Intentional error"); - }); - } catch { - // Expected - } - - const fetched = await db.getIdentityByEmail(email); - expect(fetched).toBeNull(); - }); -}); diff --git a/packages/accounts/db.ts b/packages/accounts/db.ts deleted file mode 100644 index 9fb02a2..0000000 --- a/packages/accounts/db.ts +++ /dev/null @@ -1,113 +0,0 @@ -import type { SQL } from "bun"; -import { - type DeviceAuthOps, - deriveContext, - deviceAuthOps, - type EngineOps, - engineOps, - type IdentityOps, - type InvitationOps, - identityOps, - invitationOps, - type OAuthOps, - type OrgMemberOps, - type OrgOps, - oauthOps, - orgMemberOps, - orgOps, - type SessionOps, - sessionOps, - setLocalAccountsTimeouts, -} from "./ops"; -import type { AccountsContext } from "./types"; -import { createAccountsCrypto } from "./util/crypto"; - -export interface CreateAccountsDBOptions { - masterKey: Buffer; -} - -type AllOps = DeviceAuthOps & - IdentityOps & - OrgOps & - OrgMemberOps & - EngineOps & - InvitationOps & - OAuthOps & - SessionOps; - -export interface AccountsDB extends AllOps { - /** Create a new encryption data key (does not activate) */ - createDataKey(): Promise; - /** Activate an encryption data key */ - activateDataKey(keyId: number): Promise; - /** Execute operations within a transaction */ - withTransaction(fn: (db: AccountsDB) => Promise): Promise; -} - -function composeOps(ctx: AccountsContext): AllOps { - return { - ...deviceAuthOps(ctx), - ...identityOps(ctx), - ...orgOps(ctx), - ...orgMemberOps(ctx), - ...engineOps(ctx), - ...invitationOps(ctx), - ...oauthOps(ctx), - ...sessionOps(ctx), - }; -} - -export function createAccountsDB( - sql: SQL, - schema: string, - options: CreateAccountsDBOptions, -): AccountsDB { - const crypto = createAccountsCrypto(options.masterKey, { sql, schema }); - - const ctx: AccountsContext = { - sql, - schema, - inTransaction: false, - crypto, - }; - - const ops = composeOps(ctx); - - const db: AccountsDB = { - ...ops, - - createDataKey(): Promise { - return crypto.createDataKey(); - }, - - activateDataKey(keyId: number): Promise { - return crypto.activateDataKey(keyId); - }, - - async withTransaction(fn: (db: AccountsDB) => Promise): Promise { - return sql.begin(async (tx) => { - await setLocalAccountsTimeouts(tx); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - - const txCrypto = createAccountsCrypto(options.masterKey, { - sql: tx, - schema, - }); - const txCtx = deriveContext({ ...ctx, crypto: txCrypto }, tx); - const txOps = composeOps(txCtx); - - const txDb: AccountsDB = { - ...txOps, - createDataKey: () => txCrypto.createDataKey(), - activateDataKey: (keyId) => txCrypto.activateDataKey(keyId), - withTransaction: (nestedFn: (db: AccountsDB) => Promise) => - nestedFn(txDb), - }; - - return fn(txDb); - }); - }, - }; - - return db; -} diff --git a/packages/accounts/index.ts b/packages/accounts/index.ts deleted file mode 100644 index da9c2b1..0000000 --- a/packages/accounts/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { - type AccountsDB, - type CreateAccountsDBOptions, - createAccountsDB, -} from "./db"; -export * from "./types"; -export { generateToken, tokenHash } from "./util/hash"; -export { generateSlug } from "./util/slug"; diff --git a/packages/accounts/migrate/index.ts b/packages/accounts/migrate/index.ts deleted file mode 100644 index 29f70de..0000000 --- a/packages/accounts/migrate/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type { MigrateResult } from "./runner"; -export { dryRun, getMigrations, getVersion, migrate } from "./runner"; -export type { AccountsConfig, ResolvedConfig } from "./template"; -export { defaultConfig, resolveConfig, template } from "./template"; diff --git a/packages/accounts/migrate/migrate.integration.test.ts b/packages/accounts/migrate/migrate.integration.test.ts deleted file mode 100644 index 2417af0..0000000 --- a/packages/accounts/migrate/migrate.integration.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { - afterAll, - afterEach, - beforeAll, - describe, - expect, - test, -} from "bun:test"; -import { SQL } from "bun"; -import { dryRun, getMigrations, getVersion, migrate } from "./runner"; -import { - getDatabaseVersion, - schemaExists, - TestDatabase, - tableExists, -} from "./test-utils"; - -const adminUrl = "postgresql://postgres@localhost:5432/postgres"; - -describe("integration: migrate", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("creates schema and infrastructure tables on first run", async () => { - const schema = testSchema(); - const result = await migrate(sql, { schema }, "0.1.0"); - - expect(result.status).toBe("ok"); - expect(await schemaExists(sql, schema)).toBe(true); - expect(await tableExists(sql, schema, "version")).toBe(true); - expect(await tableExists(sql, schema, "migration")).toBe(true); - }); - - test("is idempotent", async () => { - const schema = testSchema(); - - const result1 = await migrate(sql, { schema }, "0.1.0"); - expect(result1.status).toBe("ok"); - - const result2 = await migrate(sql, { schema }, "0.1.0"); - expect(result2.status).toBe("ok"); - expect(result2.applied).toHaveLength(0); - }); - - test("tracks version correctly", async () => { - const schema = testSchema(); - - await migrate(sql, { schema }, "0.1.0"); - expect(await getDatabaseVersion(sql, schema)).toBe("0.1.0"); - - await migrate(sql, { schema }, "0.2.0"); - expect(await getDatabaseVersion(sql, schema)).toBe("0.2.0"); - }); - - test("rejects downgrade", async () => { - const schema = testSchema(); - - await migrate(sql, { schema }, "0.2.0"); - - try { - await migrate(sql, { schema }, "0.1.0"); - expect.unreachable("should have thrown"); - } catch (error) { - expect((error as Error).message).toContain("Server version (0.1.0)"); - expect((error as Error).message).toContain( - "older than database version (0.2.0)", - ); - } - }); - - test("version table has single-row constraint", async () => { - const schema = testSchema(); - await migrate(sql, { schema }, "0.1.0"); - - try { - await sql.unsafe( - `insert into ${schema}.version (version) values ('0.2.0')`, - ); - expect.unreachable("should have thrown"); - } catch (error) { - // Expected: unique constraint violation - expect(error).toBeTruthy(); - } - }); - - test("rejects migration by non-owner", async () => { - const schema = testSchema(); - - // First run creates the schema - await migrate(sql, { schema }, "0.1.0"); - - // Change owner to postgres (different from current user in some setups) - // This test may not trigger on all setups - the important thing is the code path exists - // The ownership check is in scaffold() - }); -}); - -describe("integration: dryRun", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("shows all pending for new schema", async () => { - const schema = testSchema(); - const result = await dryRun(sql, { schema }); - - // No migrations defined yet (scaffold handles infrastructure) - expect(result.pending.length).toBe(getMigrations().length); - expect(result.applied).toHaveLength(0); - }); - - test("shows none pending after migration", async () => { - const schema = testSchema(); - await migrate(sql, { schema }, "0.1.0"); - - const result = await dryRun(sql, { schema }); - - expect(result.pending).toHaveLength(0); - expect(result.applied.length).toBe(getMigrations().length); - }); -}); - -describe("integration: getVersion", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("returns current version", async () => { - const schema = testSchema(); - await migrate(sql, { schema }, "1.2.3"); - - const version = await getVersion(sql, { schema }); - expect(version).toBe("1.2.3"); - }); -}); - -describe("integration: TestDatabase", () => { - test("creates isolated schema", async () => { - const db = await TestDatabase.create(adminUrl, "0.1.0"); - - try { - expect(db.schema).toMatch(/^accounts_test_/); - expect(await schemaExists(db.sql, db.schema)).toBe(true); - expect(await tableExists(db.sql, db.schema, "migration")).toBe(true); - expect(await tableExists(db.sql, db.schema, "version")).toBe(true); - } finally { - await db.dispose(); - } - }); - - test("dispose drops schema", async () => { - const db = await TestDatabase.create(adminUrl, "0.1.0"); - const schema = db.schema; - - const sql = new SQL(adminUrl); - try { - expect(await schemaExists(sql, schema)).toBe(true); - await db.dispose(); - expect(await schemaExists(sql, schema)).toBe(false); - } finally { - await sql.close(); - } - }); -}); - -describe("integration: advisory locks", () => { - let sql: SQL; - const testSchemas: string[] = []; - - beforeAll(() => { - sql = new SQL(adminUrl); - }); - - afterEach(async () => { - for (const schema of testSchemas) { - await sql.unsafe(`drop schema if exists ${schema} cascade`); - } - testSchemas.length = 0; - }); - - afterAll(async () => { - await sql.close(); - }); - - function testSchema(): string { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - testSchemas.push(schema); - return schema; - } - - test("concurrent migrations on same schema - handles gracefully", async () => { - const schema = testSchema(); - - const results = await Promise.all([ - migrate(sql, { schema }, "0.1.0"), - migrate(sql, { schema }, "0.1.0"), - migrate(sql, { schema }, "0.1.0"), - ]); - - // All should complete (ok or skipped) - for (const result of results) { - expect(["ok", "skipped"]).toContain(result.status); - } - - // Schema should exist and be properly set up - expect(await schemaExists(sql, schema)).toBe(true); - expect(await tableExists(sql, schema, "version")).toBe(true); - }); -}); diff --git a/packages/accounts/migrate/migrations/001_updated_at.sql b/packages/accounts/migrate/migrations/001_updated_at.sql deleted file mode 100644 index f2c6856..0000000 --- a/packages/accounts/migrate/migrations/001_updated_at.sql +++ /dev/null @@ -1,13 +0,0 @@ --- extensions required by the accounts schema -create extension if not exists citext; - --- generic trigger function to update updated_at timestamp -create function {{schema}}.update_updated_at() -returns trigger -as $func$ -begin - new.updated_at = pg_catalog.now(); - return new; -end; -$func$ language plpgsql volatile security definer -set search_path to {{schema}}, pg_temp; diff --git a/packages/accounts/migrate/migrations/002_core_tables.sql b/packages/accounts/migrate/migrations/002_core_tables.sql deleted file mode 100644 index 92a40b1..0000000 --- a/packages/accounts/migrate/migrations/002_core_tables.sql +++ /dev/null @@ -1,58 +0,0 @@ --- ===== Shard (minimal, for future scaling) ===== -create table {{schema}}.shard -( id int primary key -); - --- seed default shard -insert into {{schema}}.shard (id) values (1); - --- ===== Org (billing/ownership entity) ===== -create table {{schema}}.org -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, slug text unique not null check (slug ~ '^[a-z0-9]{12}$') -, name text not null -, created_at timestamptz not null default now() -, updated_at timestamptz -); - -create trigger org_updated_at - before update on {{schema}}.org - for each row - execute function {{schema}}.update_updated_at(); - --- ===== Identity (human who can log in) ===== -create table {{schema}}.identity -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, email citext unique not null -, name text not null -, created_at timestamptz not null default now() -, updated_at timestamptz -); - -create trigger identity_updated_at - before update on {{schema}}.identity - for each row - execute function {{schema}}.update_updated_at(); - --- ===== Engine (memory engine instance) ===== -create table {{schema}}.engine -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, org_id uuid not null references {{schema}}.org on delete cascade -, slug text unique not null check (slug ~ '^[a-z0-9]{12}$') -, name text not null -, shard_id int not null references {{schema}}.shard -, status text not null default 'active' check (status in ('active', 'suspended', 'deleted')) -, language text not null default 'english' check (language ~ '^[a-z_]+$') -, created_at timestamptz not null default now() -, updated_at timestamptz -, unique (org_id, name) -); - -create index idx_engine_org on {{schema}}.engine (org_id); -create index idx_engine_shard on {{schema}}.engine (shard_id); -create index idx_engine_status on {{schema}}.engine (status) where status <> 'active'; - -create trigger engine_updated_at - before update on {{schema}}.engine - for each row - execute function {{schema}}.update_updated_at(); diff --git a/packages/accounts/migrate/migrations/003_membership.sql b/packages/accounts/migrate/migrations/003_membership.sql deleted file mode 100644 index 9feba1a..0000000 --- a/packages/accounts/migrate/migrations/003_membership.sql +++ /dev/null @@ -1,10 +0,0 @@ --- ===== Org Member (identity membership in org) ===== -create table {{schema}}.org_member -( org_id uuid not null references {{schema}}.org on delete cascade -, identity_id uuid not null references {{schema}}.identity on delete cascade -, role text not null check (role in ('owner', 'admin', 'member')) -, created_at timestamptz not null default now() -, primary key (org_id, identity_id) -); - -create index idx_org_member_identity on {{schema}}.org_member (identity_id); diff --git a/packages/accounts/migrate/migrations/004_invitations.sql b/packages/accounts/migrate/migrations/004_invitations.sql deleted file mode 100644 index 130682c..0000000 --- a/packages/accounts/migrate/migrations/004_invitations.sql +++ /dev/null @@ -1,17 +0,0 @@ --- ===== Invitation (pending org invitations) ===== -create table {{schema}}.invitation -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, org_id uuid not null references {{schema}}.org on delete cascade -, email citext not null -, role text not null check (role in ('owner', 'admin', 'member')) -, token text unique not null -, invited_by uuid not null references {{schema}}.identity -, expires_at timestamptz not null -, accepted_at timestamptz -, created_at timestamptz not null default now() -, unique (org_id, email) -); - -create index idx_invitation_token on {{schema}}.invitation (token) where accepted_at is null; -create index idx_invitation_org on {{schema}}.invitation (org_id) where accepted_at is null; -create index idx_invitation_email on {{schema}}.invitation (email) where accepted_at is null; diff --git a/packages/accounts/migrate/migrations/005_auth.sql b/packages/accounts/migrate/migrations/005_auth.sql deleted file mode 100644 index 27545c0..0000000 --- a/packages/accounts/migrate/migrations/005_auth.sql +++ /dev/null @@ -1,34 +0,0 @@ --- ===== OAuth Account (provider links) ===== -create table {{schema}}.oauth_account -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, identity_id uuid not null references {{schema}}.identity on delete cascade -, provider text not null check (provider in ('google', 'github')) -, provider_account_id text not null -, email citext -, access_token text -, refresh_token text -, token_expires_at timestamptz -, created_at timestamptz not null default now() -, updated_at timestamptz -, unique (provider, provider_account_id) -); - -create index idx_oauth_account_identity on {{schema}}.oauth_account (identity_id); - -create trigger oauth_account_updated_at - before update on {{schema}}.oauth_account - for each row - execute function {{schema}}.update_updated_at(); - --- ===== Session (for OAuth flow) ===== -create table {{schema}}.session -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, identity_id uuid not null references {{schema}}.identity on delete cascade -, token text unique not null -, expires_at timestamptz not null -, created_at timestamptz not null default now() -); - -create index idx_session_identity on {{schema}}.session (identity_id); -create index idx_session_token on {{schema}}.session (token); -create index idx_session_expires on {{schema}}.session (expires_at); diff --git a/packages/accounts/migrate/migrations/006_ops_support.sql b/packages/accounts/migrate/migrations/006_ops_support.sql deleted file mode 100644 index 90dbda2..0000000 --- a/packages/accounts/migrate/migrations/006_ops_support.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Encryption keys for envelope encryption -create table {{schema}}.encryption_key -( id int primary key generated always as identity -, key_ciphertext bytea not null -, active boolean not null default false -, created_at timestamptz not null default now() -); - --- Only one active key at a time -create unique index idx_encryption_key_active - on {{schema}}.encryption_key (active) where active = true; - --- Track which key encrypted OAuth tokens -alter table {{schema}}.oauth_account - add column encryption_key_id int references {{schema}}.encryption_key(id); - --- Trigger to prevent removing the last owner from an org -create function {{schema}}.check_org_has_owner() -returns trigger as $func$ -begin - if (TG_OP = 'DELETE' and OLD.role = 'owner') - or (TG_OP = 'UPDATE' and OLD.role = 'owner' and NEW.role <> 'owner') - then - if not exists ( - select 1 from {{schema}}.org_member - where org_id = OLD.org_id - and role = 'owner' - and identity_id <> OLD.identity_id - ) then - raise exception 'org_must_have_owner' - using errcode = 'P0001', - hint = 'Cannot remove the last owner from an organization'; - end if; - end if; - - if TG_OP = 'DELETE' then - return OLD; - end if; - return NEW; -end; -$func$ language plpgsql; - -create trigger org_member_owner_check - before delete or update on {{schema}}.org_member - for each row - execute function {{schema}}.check_org_has_owner(); diff --git a/packages/accounts/migrate/migrations/007_device_authorization.sql b/packages/accounts/migrate/migrations/007_device_authorization.sql deleted file mode 100644 index 476284d..0000000 --- a/packages/accounts/migrate/migrations/007_device_authorization.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Device authorization for OAuth device flow (RFC 8628) --- Stores state during the ~15 minute device flow window - -create table {{schema}}.device_authorization ( - device_code text primary key, -- URL-safe base64, 32 bytes - user_code text not null unique, -- XXXX-XXXX format - provider text not null, -- 'google' | 'github' - oauth_state text not null unique, -- CSRF protection - expires_at timestamptz not null, -- 15 minute TTL - last_poll timestamptz, -- rate limiting - identity_id uuid references {{schema}}.identity(id) on delete cascade, -- set when authorized - denied boolean not null default false, -- user denied access - created_at timestamptz not null default now() -); diff --git a/packages/accounts/migrate/migrations/008_drop_org_owner_trigger.sql b/packages/accounts/migrate/migrations/008_drop_org_owner_trigger.sql deleted file mode 100644 index 260efbf..0000000 --- a/packages/accounts/migrate/migrations/008_drop_org_owner_trigger.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Remove the "must keep at least one owner" trigger on org_member. --- --- The trigger fired before any DELETE or UPDATE on org_member, including --- the rows cascaded by `delete from org` (since org_member.org_id has --- `references org on delete cascade`). When an org was being deleted in --- its entirety, the cascade removed the org's owner row too, and the --- trigger refused — making it impossible to delete an org that you owned. --- --- The invariant is now enforced at the application layer in --- packages/accounts/ops/org-member.ts (removeMember + updateRole), where --- it can distinguish member-management flows from a cascading org delete. -drop trigger if exists org_member_owner_check on {{schema}}.org_member; -drop function if exists {{schema}}.check_org_has_owner(); diff --git a/packages/accounts/migrate/migrations/009_session_lookup.sql b/packages/accounts/migrate/migrations/009_session_lookup.sql deleted file mode 100644 index c1097dd..0000000 --- a/packages/accounts/migrate/migrations/009_session_lookup.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Session and invitation tokens are stored as their sha256 digest in --- token_hash, used as a unique-indexed lookup key. Raw tokens are 256-bit --- CSPRNG output, so sha256 alone provides preimage resistance equivalent --- to argon2 against an offline DB dump — without paying ~60ms per verify --- and without the O(n) scan-and-verify pattern the previous schema forced. - --- Drop existing rows: we don't store raw tokens, so we cannot derive --- token_hash for them. All current CLI sessions become invalid (next --- command yields a 401, "Invalid or expired session"; user runs `me login`). --- All pending invitations must be re-issued. -truncate {{schema}}.session; -truncate {{schema}}.invitation; - --- `drop column` cascades to dependent indexes and constraints. This removes --- session_token_key (unique constraint) and idx_session_token, plus --- invitation_token_key (unique constraint) and idx_invitation_token. -alter table {{schema}}.session drop column token; -alter table {{schema}}.invitation drop column token; - -alter table {{schema}}.session - add column token_hash bytea not null; -alter table {{schema}}.invitation - add column token_hash bytea not null; - -create unique index session_token_hash_uniq - on {{schema}}.session (token_hash); - -create unique index invitation_token_hash_uniq - on {{schema}}.invitation (token_hash) - where accepted_at is null; diff --git a/packages/accounts/migrate/runner.test.ts b/packages/accounts/migrate/runner.test.ts deleted file mode 100644 index 6ffc14f..0000000 --- a/packages/accounts/migrate/runner.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { getMigrations } from "./runner"; - -describe("getMigrations", () => { - test("returns an array", () => { - expect(Array.isArray(getMigrations())).toBe(true); - }); - - test("migrations are sorted by name", () => { - const names = getMigrations().map((m) => m.name); - const sorted = [...names].sort(); - expect(names).toEqual(sorted); - }); - - test("migration names match NNN_name pattern", () => { - for (const { name } of getMigrations()) { - expect(name).toMatch(/^\d{3}_\w+$/); - } - }); - - // Note: scaffold() handles infrastructure (schema, version, migration tables) - // Domain migrations will be added to the migrations array as needed -}); diff --git a/packages/accounts/migrate/runner.ts b/packages/accounts/migrate/runner.ts deleted file mode 100644 index 8cf8ca8..0000000 --- a/packages/accounts/migrate/runner.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { type SQL, semver } from "bun"; -import { setLocalAccountsTimeouts } from "../ops/_tx"; -import migration001 from "./migrations/001_updated_at.sql" with { - type: "text", -}; -import migration002 from "./migrations/002_core_tables.sql" with { - type: "text", -}; -import migration003 from "./migrations/003_membership.sql" with { - type: "text", -}; -import migration004 from "./migrations/004_invitations.sql" with { - type: "text", -}; -import migration005 from "./migrations/005_auth.sql" with { type: "text" }; -import migration006 from "./migrations/006_ops_support.sql" with { - type: "text", -}; -import migration007 from "./migrations/007_device_authorization.sql" with { - type: "text", -}; -import migration008 from "./migrations/008_drop_org_owner_trigger.sql" with { - type: "text", -}; -import migration009 from "./migrations/009_session_lookup.sql" with { - type: "text", -}; -import { type AccountsConfig, resolveConfig, template } from "./template"; - -interface Migration { - name: string; - sql: string; -} - -const migrations: Migration[] = [ - { name: "001_updated_at", sql: migration001 }, - { name: "002_core_tables", sql: migration002 }, - { name: "003_membership", sql: migration003 }, - { name: "004_invitations", sql: migration004 }, - { name: "005_auth", sql: migration005 }, - { name: "006_ops_support", sql: migration006 }, - { name: "007_device_authorization", sql: migration007 }, - { name: "008_drop_org_owner_trigger", sql: migration008 }, - { name: "009_session_lookup", sql: migration009 }, -]; - -export interface MigrateResult { - schema: string; - status: "ok" | "skipped" | "error"; - applied: string[]; - error?: Error; -} - -const MAX_LOCK_RETRIES = 5; -const BASE_DELAY_MS = 100; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** - * Scaffold creates the migration infrastructure: schema, version table, migration table. - * This runs before migrations and is idempotent - safe to call multiple times. - * Also validates ownership to prevent migrating a schema you don't own. - */ -async function scaffold(tx: SQL, schema: string): Promise { - await tx.unsafe(` - do $block$ - declare - _owner oid; - _user oid; - begin - select pg_catalog.to_regrole(current_user)::oid - into strict _user - ; - - select n.nspowner into _owner - from pg_catalog.pg_namespace n - where n.nspname = '${schema}' - ; - - if _owner is null then - -- schema doesn't exist, create infrastructure - create schema ${schema}; - - -- version table (single row, tracks overall schema version) - create table ${schema}.version - ( version text not null check (version ~ '^\\d+\\.\\d+\\.\\d+$') - , at timestamptz not null default now() - ); - create unique index on ${schema}.version ((true)); - insert into ${schema}.version (version) values ('0.0.0'); - - -- migration table - create table ${schema}.migration - ( name text not null primary key - , applied_at_version text not null - , applied_at timestamptz not null default pg_catalog.clock_timestamp() - ); - - elsif _owner is distinct from _user then - raise exception 'only the owner of the ${schema} schema can run database migrations'; - end if - ; - end - $block$ - `); -} - -export async function migrate( - sql: SQL, - config?: AccountsConfig, - serverVersion = "0.0.0", -): Promise { - const resolved = resolveConfig(config); - const { schema } = resolved; - - return await sql.begin(async (tx) => { - await setLocalAccountsTimeouts(tx); - - // Acquire advisory lock with retry - const [{ lock_id }] = - await tx`select hashtext(${schema})::bigint as lock_id`; - - let acquired = false; - for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { - const [result] = - await tx`select pg_try_advisory_xact_lock(${lock_id}) as acquired`; - if (result.acquired) { - acquired = true; - break; - } - if (attempt < MAX_LOCK_RETRIES - 1) { - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } - - if (!acquired) { - return { schema, status: "skipped" as const, applied: [] }; - } - - // Scaffold creates schema + version + migration tables (idempotent) - await scaffold(tx, schema); - - // Check version - reject downgrades - const [{ version: dbVersion }] = await tx.unsafe( - `select version from ${schema}.version`, - ); - - const cmp = semver.order(serverVersion, dbVersion); - if (cmp < 0) { - throw new Error( - `Server version (${serverVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the server.", - ); - } - - // Run migrations - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - const applied: string[] = []; - - for (const migration of sorted) { - const [existing] = await tx.unsafe( - `select 1 from ${schema}.migration where name = $1`, - [migration.name], - ); - - if (existing) { - continue; - } - - const renderedSql = template(migration.sql, resolved); - await tx.unsafe(renderedSql); - await tx.unsafe( - `insert into ${schema}.migration (name, applied_at_version) values ($1, $2)`, - [migration.name, serverVersion], - ); - applied.push(migration.name); - } - - // Update version if app version is newer - if (cmp > 0) { - await tx.unsafe(`update ${schema}.version set version = $1, at = now()`, [ - serverVersion, - ]); - } - - return { schema, status: "ok" as const, applied }; - }); -} - -export async function dryRun( - sql: SQL, - config?: AccountsConfig, -): Promise<{ pending: string[]; applied: string[] }> { - const { schema } = resolveConfig(config); - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - - // Check if migration table exists - const [{ exists }] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = 'migration' - ) as exists - `; - - if (!exists) { - return { - pending: sorted.map((m) => m.name), - applied: [], - }; - } - - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - const appliedSet = new Set(rows.map((r: { name: string }) => r.name)); - const applied = sorted - .filter((m) => appliedSet.has(m.name)) - .map((m) => m.name); - const pending = sorted - .filter((m) => !appliedSet.has(m.name)) - .map((m) => m.name); - - return { pending, applied }; -} - -export async function getVersion( - sql: SQL, - config?: AccountsConfig, -): Promise { - const { schema } = resolveConfig(config); - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row.version; -} - -export function getMigrations(): ReadonlyArray<{ name: string }> { - return [...migrations] - .sort((a, b) => a.name.localeCompare(b.name)) - .map(({ name }) => ({ name })); -} diff --git a/packages/accounts/migrate/template.test.ts b/packages/accounts/migrate/template.test.ts deleted file mode 100644 index 89fe385..0000000 --- a/packages/accounts/migrate/template.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { defaultConfig, resolveConfig, template } from "./template"; - -describe("template function", () => { - test("replaces single variable", () => { - const sql = "CREATE TABLE {{schema}}.foo (id uuid)"; - const result = template(sql, { schema: "accounts" }); - expect(result).toBe("CREATE TABLE accounts.foo (id uuid)"); - }); - - test("replaces same variable multiple times", () => { - const sql = "{{schema}}.a and {{schema}}.b"; - const result = template(sql, { schema: "test" }); - expect(result).toBe("test.a and test.b"); - }); - - test("throws on missing variable", () => { - const sql = "CREATE TABLE {{missing}}.foo"; - expect(() => template(sql, {})).toThrow( - "Missing template variable: missing", - ); - }); - - test("handles no variables", () => { - const sql = "CREATE TABLE foo (id uuid)"; - const result = template(sql, {}); - expect(result).toBe("CREATE TABLE foo (id uuid)"); - }); - - test("handles numeric values", () => { - const sql = "LIMIT {{limit}}"; - const result = template(sql, { limit: 100 }); - expect(result).toBe("LIMIT 100"); - }); -}); - -describe("config", () => { - test("defaultConfig has schema = accounts", () => { - expect(defaultConfig.schema).toBe("accounts"); - }); - - test("resolveConfig uses default schema", () => { - const resolved = resolveConfig(); - expect(resolved.schema).toBe("accounts"); - }); - - test("resolveConfig allows schema override", () => { - const resolved = resolveConfig({ schema: "accounts_test" }); - expect(resolved.schema).toBe("accounts_test"); - }); -}); diff --git a/packages/accounts/migrate/template.ts b/packages/accounts/migrate/template.ts deleted file mode 100644 index d5a6194..0000000 --- a/packages/accounts/migrate/template.ts +++ /dev/null @@ -1,22 +0,0 @@ -export function template(sql: string, vars: Record): string { - return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { - if (!(key in vars)) { - throw new Error(`Missing template variable: ${key}`); - } - return String(vars[key]); - }); -} - -export interface AccountsConfig { - schema?: string; -} - -export type ResolvedConfig = Required; - -export const defaultConfig: ResolvedConfig = { - schema: "accounts", -}; - -export function resolveConfig(config?: AccountsConfig): ResolvedConfig { - return { ...defaultConfig, ...config }; -} diff --git a/packages/accounts/migrate/test-utils.ts b/packages/accounts/migrate/test-utils.ts deleted file mode 100644 index 36271a5..0000000 --- a/packages/accounts/migrate/test-utils.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { SQL } from "bun"; -import { migrate } from "./runner"; -import type { AccountsConfig } from "./template"; - -function assertSafeIdentifier(name: string): void { - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { - throw new Error(`Unsafe database identifier: ${name}`); - } -} - -export class TestDatabase { - schema: string; - sql: SQL; - - private constructor(schema: string, sql: SQL) { - this.schema = schema; - this.sql = sql; - } - - static async create( - adminUrl = "postgresql://postgres@localhost:5432/postgres", - serverVersion = "0.1.0", - ): Promise { - const schema = `accounts_test_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - assertSafeIdentifier(schema); - - const sql = new SQL(adminUrl); - const config: AccountsConfig = { schema }; - - await migrate(sql, config, serverVersion); - - return new TestDatabase(schema, sql); - } - - async dispose(): Promise { - assertSafeIdentifier(this.schema); - await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); - await this.sql.close(); - } -} - -export async function getAppliedMigrations( - sql: SQL, - schema: string, -): Promise { - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - return rows.map((r: { name: string }) => r.name); -} - -export async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = ${table} - ) as exists - `; - return row.exists; -} - -export async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.schemata - where schema_name = ${name} - ) as exists - `; - return row.exists; -} - -export async function getDatabaseVersion( - sql: SQL, - schema: string, -): Promise { - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row.version; -} diff --git a/packages/accounts/ops/_tx.ts b/packages/accounts/ops/_tx.ts deleted file mode 100644 index 1b73e76..0000000 --- a/packages/accounts/ops/_tx.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Transaction helper for accounts operations - * - * Simpler than engine's _tx.ts - no RLS roles needed since accounts - * uses application-level authorization. - */ - -import { span } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import type { AccountsContext } from "../types"; - -const ACCOUNTS_STATEMENT_TIMEOUT = - process.env.ACCOUNTS_STATEMENT_TIMEOUT ?? "25s"; -const ACCOUNTS_LOCK_TIMEOUT = process.env.ACCOUNTS_LOCK_TIMEOUT ?? "5s"; -const ACCOUNTS_TRANSACTION_TIMEOUT = - process.env.ACCOUNTS_TRANSACTION_TIMEOUT ?? "30s"; -const ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT = - process.env.ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? "30s"; - -/** - * Bound accounts transactions with transaction-local GUCs so pooled - * connections do not retain settings. - */ -export async function setLocalAccountsTimeouts(sql: SQL): Promise { - await sql.unsafe("SELECT set_config('statement_timeout', $1, true)", [ - ACCOUNTS_STATEMENT_TIMEOUT, - ]); - await sql.unsafe("SELECT set_config('lock_timeout', $1, true)", [ - ACCOUNTS_LOCK_TIMEOUT, - ]); - await sql.unsafe("SELECT set_config('transaction_timeout', $1, true)", [ - ACCOUNTS_TRANSACTION_TIMEOUT, - ]); - await sql.unsafe( - "SELECT set_config('idle_in_transaction_session_timeout', $1, true)", - [ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT], - ); -} - -/** - * Execute a function within a transaction context. - * - * If already inside a transaction (ctx.inTransaction is true), runs directly. - * Otherwise opens a new transaction with search_path set. - */ -export async function withTx( - ctx: AccountsContext, - operation: string, - fn: (sql: SQL) => Promise, -): Promise { - if (ctx.inTransaction) { - return fn(ctx.sql); - } - - return span(`accounts.${operation}`, { - attributes: { - "db.schema": ctx.schema, - "db.operation": operation, - }, - callback: () => - ctx.sql.begin(async (tx) => { - await setLocalAccountsTimeouts(tx); - await tx.unsafe(`SET LOCAL search_path TO ${ctx.schema}, public`); - return fn(tx); - }), - }); -} - -/** - * Create a derived context for use inside withTransaction() - */ -export function deriveContext(ctx: AccountsContext, tx: SQL): AccountsContext { - return { - ...ctx, - sql: tx, - inTransaction: true, - }; -} diff --git a/packages/accounts/ops/device-auth.ts b/packages/accounts/ops/device-auth.ts deleted file mode 100644 index 881f666..0000000 --- a/packages/accounts/ops/device-auth.ts +++ /dev/null @@ -1,207 +0,0 @@ -import type { - AccountsContext, - CreateDeviceAuthParams, - DeviceAuthorization, - DeviceProvider, -} from "../types"; -import { withTx } from "./_tx"; - -interface DeviceAuthRow { - device_code: string; - user_code: string; - provider: string; - oauth_state: string; - expires_at: Date; - last_poll: Date | null; - identity_id: string | null; - denied: boolean; - created_at: Date; -} - -function rowToDeviceAuth(row: DeviceAuthRow): DeviceAuthorization { - return { - deviceCode: row.device_code, - userCode: row.user_code, - provider: row.provider as DeviceProvider, - oauthState: row.oauth_state, - expiresAt: row.expires_at, - lastPoll: row.last_poll, - identityId: row.identity_id, - denied: row.denied, - createdAt: row.created_at, - }; -} - -export function deviceAuthOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - /** - * Create a new device authorization. - */ - async create(params: CreateDeviceAuthParams): Promise { - const { deviceCode, userCode, provider, oauthState, expiresAt } = params; - - return withTx(ctx, "createDeviceAuth", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.device_authorization - (device_code, user_code, provider, oauth_state, expires_at) - values - (${deviceCode}, ${userCode}, ${provider}, ${oauthState}, ${expiresAt}) - returning * - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create device authorization"); - } - return rowToDeviceAuth(row); - }); - }, - - /** - * Get device authorization by device code (for CLI polling). - * Returns null if not found or expired. - */ - async getByDeviceCode( - deviceCode: string, - ): Promise { - return withTx(ctx, "getDeviceByCode", async (sql) => { - const rows = await sql` - select * from ${sql.unsafe(schema)}.device_authorization - where device_code = ${deviceCode} - and expires_at > now() - `; - const row = rows[0]; - return row ? rowToDeviceAuth(row) : null; - }); - }, - - /** - * Get device authorization by user code (for browser code entry). - * Normalizes input: uppercase, removes hyphens, reconstructs format. - * Returns null if not found or expired. - */ - async getByUserCode(userCode: string): Promise { - // Normalize: uppercase, remove hyphen, reconstruct XXXX-XXXX - const normalized = userCode.toUpperCase().replace(/-/g, ""); - const formatted = `${normalized.slice(0, 4)}-${normalized.slice(4)}`; - - return withTx(ctx, "getDeviceByUserCode", async (sql) => { - const rows = await sql` - select * from ${sql.unsafe(schema)}.device_authorization - where user_code = ${formatted} - and expires_at > now() - `; - const row = rows[0]; - return row ? rowToDeviceAuth(row) : null; - }); - }, - - /** - * Get device authorization by OAuth state (for callback). - * Returns null if not found or expired. - */ - async getByOAuthState( - oauthState: string, - ): Promise { - return withTx(ctx, "getDeviceByOAuthState", async (sql) => { - const rows = await sql` - select * from ${sql.unsafe(schema)}.device_authorization - where oauth_state = ${oauthState} - and expires_at > now() - `; - const row = rows[0]; - return row ? rowToDeviceAuth(row) : null; - }); - }, - - /** - * Update last poll time for rate limiting. - * Returns the time since last poll in milliseconds, or null if first poll. - */ - async updateLastPoll(deviceCode: string): Promise { - return withTx(ctx, "updateDeviceLastPoll", async (sql) => { - const rows = await sql<{ last_poll: Date | null }[]>` - update ${sql.unsafe(schema)}.device_authorization - set last_poll = now() - where device_code = ${deviceCode} - and expires_at > now() - returning ( - select last_poll from ${sql.unsafe(schema)}.device_authorization - where device_code = ${deviceCode} - ) as last_poll - `; - const row = rows[0]; - if (!row?.last_poll) { - return null; // First poll or not found - } - return Date.now() - row.last_poll.getTime(); - }); - }, - - /** - * Mark device as authorized with an identity. - * Returns true if updated, false if not found/expired. - */ - async authorize(deviceCode: string, identityId: string): Promise { - return withTx(ctx, "authorizeDevice", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.device_authorization - set identity_id = ${identityId} - where device_code = ${deviceCode} - and expires_at > now() - and identity_id is null - and denied = false - `; - return result.count > 0; - }); - }, - - /** - * Mark device as denied. - * Returns true if updated, false if not found/expired. - */ - async deny(deviceCode: string): Promise { - return withTx(ctx, "denyDevice", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.device_authorization - set denied = true - where device_code = ${deviceCode} - and expires_at > now() - and identity_id is null - `; - return result.count > 0; - }); - }, - - /** - * Delete a device authorization (cleanup after completion). - * Returns true if deleted. - */ - async delete(deviceCode: string): Promise { - return withTx(ctx, "deleteDevice", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.device_authorization - where device_code = ${deviceCode} - `; - return result.count > 0; - }); - }, - - /** - * Delete all expired device authorizations. - * Called by cron job. Returns count deleted. - */ - async deleteExpired(): Promise { - return withTx(ctx, "deleteExpiredDevices", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.device_authorization - where expires_at <= now() - `; - return result.count; - }); - }, - }; -} - -export type DeviceAuthOps = ReturnType; diff --git a/packages/accounts/ops/engine.ts b/packages/accounts/ops/engine.ts deleted file mode 100644 index 64437aa..0000000 --- a/packages/accounts/ops/engine.ts +++ /dev/null @@ -1,143 +0,0 @@ -import type { - AccountsContext, - CreateEngineParams, - Engine, - EngineStatus, -} from "../types"; -import { generateSlug } from "../util/slug"; -import { withTx } from "./_tx"; - -interface EngineRow { - id: string; - org_id: string; - slug: string; - name: string; - shard_id: number; - status: EngineStatus; - language: string; - created_at: Date; - updated_at: Date | null; -} - -function rowToEngine(row: EngineRow): Engine { - return { - id: row.id, - orgId: row.org_id, - slug: row.slug, - name: row.name, - shardId: row.shard_id, - status: row.status, - language: row.language, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function engineOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createEngine(params: CreateEngineParams): Promise { - const { id, orgId, name, shardId = 1, language = "english" } = params; - const slug = generateSlug(); - - return withTx(ctx, "createEngine", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.engine (id, org_id, slug, name, shard_id, language) - values (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${orgId}, ${slug}, ${name}, ${shardId}, ${language}) - returning id, org_id, slug, name, shard_id, status, language, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create engine"); - } - return rowToEngine(row); - }); - }, - - async getEngine(id: string): Promise { - return withTx(ctx, "getEngine", async (sql) => { - const [row] = await sql` - select id, org_id, slug, name, shard_id, status, language, created_at, updated_at - from ${sql.unsafe(schema)}.engine - where id = ${id} - `; - return row ? rowToEngine(row) : null; - }); - }, - - async getEngineBySlug(slug: string): Promise { - return withTx(ctx, "getEngineBySlug", async (sql) => { - const [row] = await sql` - select id, org_id, slug, name, shard_id, status, language, created_at, updated_at - from ${sql.unsafe(schema)}.engine - where slug = ${slug} - `; - return row ? rowToEngine(row) : null; - }); - }, - - async updateEngine( - id: string, - params: { name?: string; status?: EngineStatus }, - ): Promise { - const { name, status } = params; - if (name === undefined && status === undefined) { - return false; - } - - return withTx(ctx, "updateEngine", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.engine - set - ${name !== undefined ? sql`name = ${name},` : sql``} - ${status !== undefined ? sql`status = ${status},` : sql``} - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async listEnginesByOrg(orgId: string): Promise { - return withTx(ctx, "listEnginesByOrg", async (sql) => { - const rows = await sql` - select id, org_id, slug, name, shard_id, status, language, created_at, updated_at - from ${sql.unsafe(schema)}.engine - where org_id = ${orgId} - order by created_at - `; - return rows.map(rowToEngine); - }); - }, - - /** - * Hard-delete an engine row. Returns true if a row was deleted, false - * if no row matched. Caller is responsible for first dropping the - * engine schema; this only removes the accounts-side metadata. - */ - async deleteEngine(id: string): Promise { - return withTx(ctx, "deleteEngine", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.engine - where id = ${id} - `; - return result.count > 0; - }); - }, - - /** List all active engines across all orgs (for embedding worker discovery) */ - async listActiveEngines(): Promise<{ slug: string; shardId: number }[]> { - return withTx(ctx, "listActiveEngines", async (sql) => { - const rows = await sql<{ slug: string; shard_id: number }[]>` - select slug, shard_id - from ${sql.unsafe(schema)}.engine - where status = 'active' - `; - return rows.map((r) => ({ slug: r.slug, shardId: r.shard_id })); - }); - }, - }; -} - -export type EngineOps = ReturnType; diff --git a/packages/accounts/ops/identity.ts b/packages/accounts/ops/identity.ts deleted file mode 100644 index 04057ed..0000000 --- a/packages/accounts/ops/identity.ts +++ /dev/null @@ -1,99 +0,0 @@ -import type { AccountsContext, CreateIdentityParams, Identity } from "../types"; -import { withTx } from "./_tx"; - -interface IdentityRow { - id: string; - email: string; - name: string; - created_at: Date; - updated_at: Date | null; -} - -function rowToIdentity(row: IdentityRow): Identity { - return { - id: row.id, - email: row.email, - name: row.name, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function identityOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createIdentity(params: CreateIdentityParams): Promise { - const { id, email, name } = params; - - return withTx(ctx, "createIdentity", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.identity (id, email, name) - values (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${email}, ${name}) - returning id, email, name, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create identity"); - } - return rowToIdentity(row); - }); - }, - - async getIdentity(id: string): Promise { - return withTx(ctx, "getIdentity", async (sql) => { - const [row] = await sql` - select id, email, name, created_at, updated_at - from ${sql.unsafe(schema)}.identity - where id = ${id} - `; - return row ? rowToIdentity(row) : null; - }); - }, - - async getIdentityByEmail(email: string): Promise { - return withTx(ctx, "getIdentityByEmail", async (sql) => { - const [row] = await sql` - select id, email, name, created_at, updated_at - from ${sql.unsafe(schema)}.identity - where email = ${email} - `; - return row ? rowToIdentity(row) : null; - }); - }, - - async updateIdentity( - id: string, - params: { name?: string; email?: string }, - ): Promise { - const { name, email } = params; - if (name === undefined && email === undefined) { - return false; - } - - return withTx(ctx, "updateIdentity", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.identity - set - ${name !== undefined ? sql`name = ${name},` : sql``} - ${email !== undefined ? sql`email = ${email},` : sql``} - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async deleteIdentity(id: string): Promise { - return withTx(ctx, "deleteIdentity", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.identity - where id = ${id} - `; - return result.count > 0; - }); - }, - }; -} - -export type IdentityOps = ReturnType; diff --git a/packages/accounts/ops/index.ts b/packages/accounts/ops/index.ts deleted file mode 100644 index 49f7fd8..0000000 --- a/packages/accounts/ops/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { deriveContext, setLocalAccountsTimeouts, withTx } from "./_tx"; -export { type DeviceAuthOps, deviceAuthOps } from "./device-auth"; -export { type EngineOps, engineOps } from "./engine"; -export { type IdentityOps, identityOps } from "./identity"; -export { type InvitationOps, invitationOps } from "./invitation"; -export { type OAuthOps, oauthOps } from "./oauth"; -export { type OrgOps, orgOps } from "./org"; -export { type OrgMemberOps, orgMemberOps } from "./org-member"; -export { type SessionOps, sessionOps } from "./session"; diff --git a/packages/accounts/ops/invitation.ts b/packages/accounts/ops/invitation.ts deleted file mode 100644 index 3526606..0000000 --- a/packages/accounts/ops/invitation.ts +++ /dev/null @@ -1,134 +0,0 @@ -import type { - AccountsContext, - CreateInvitationParams, - CreateInvitationResult, - Invitation, - OrgRole, -} from "../types"; -import { generateToken, tokenHash } from "../util/hash"; -import { withTx } from "./_tx"; - -interface InvitationRow { - id: string; - org_id: string; - email: string; - role: OrgRole; - invited_by: string; - expires_at: Date; - accepted_at: Date | null; - created_at: Date; -} - -function rowToInvitation(row: InvitationRow): Invitation { - return { - id: row.id, - orgId: row.org_id, - email: row.email, - role: row.role, - invitedBy: row.invited_by, - expiresAt: row.expires_at, - acceptedAt: row.accepted_at, - createdAt: row.created_at, - }; -} - -export function invitationOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createInvitation( - params: CreateInvitationParams, - ): Promise { - const { orgId, email, role, invitedBy, expiresInDays = 7 } = params; - - const rawToken = generateToken(); - const hash = tokenHash(rawToken); - const expiresAt = new Date(); - expiresAt.setDate(expiresAt.getDate() + expiresInDays); - - return withTx(ctx, "createInvitation", async (sql) => { - // Upsert: replace existing pending invitation for same org+email - const rows = await sql` - insert into ${sql.unsafe(schema)}.invitation - (org_id, email, role, token_hash, invited_by, expires_at) - values (${orgId}, ${email}, ${role}, ${hash}, ${invitedBy}, ${expiresAt}) - on conflict (org_id, email) - do update set - role = excluded.role, - token_hash = excluded.token_hash, - invited_by = excluded.invited_by, - expires_at = excluded.expires_at, - accepted_at = null - returning id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create invitation"); - } - return { - invitation: rowToInvitation(row), - rawToken, - }; - }); - }, - - async getInvitationByToken(rawToken: string): Promise { - const hash = tokenHash(rawToken); - - return withTx(ctx, "getInvitationByToken", async (sql) => { - // Single indexed lookup on the partial unique index: - // invitation_token_hash_uniq (token_hash) where accepted_at is null. - const rows = await sql` - select id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - from ${sql.unsafe(schema)}.invitation - where token_hash = ${hash} - and accepted_at is null - and expires_at > now() - limit 1 - `; - const row = rows[0]; - return row ? rowToInvitation(row) : null; - }); - }, - - async acceptInvitation(id: string): Promise { - return withTx(ctx, "acceptInvitation", async (sql) => { - const rows = await sql` - update ${sql.unsafe(schema)}.invitation - set accepted_at = now() - where id = ${id} - and accepted_at is null - returning id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - `; - const row = rows[0]; - return row ? rowToInvitation(row) : null; - }); - }, - - async revokeInvitation(id: string): Promise { - return withTx(ctx, "revokeInvitation", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.invitation - where id = ${id} - `; - return result.count > 0; - }); - }, - - async listPendingInvitations(orgId: string): Promise { - return withTx(ctx, "listPendingInvitations", async (sql) => { - const rows = await sql` - select id, org_id, email, role, invited_by, expires_at, accepted_at, created_at - from ${sql.unsafe(schema)}.invitation - where org_id = ${orgId} - and accepted_at is null - and expires_at > now() - order by created_at - `; - return rows.map(rowToInvitation); - }); - }, - }; -} - -export type InvitationOps = ReturnType; diff --git a/packages/accounts/ops/oauth.ts b/packages/accounts/ops/oauth.ts deleted file mode 100644 index 34d80d7..0000000 --- a/packages/accounts/ops/oauth.ts +++ /dev/null @@ -1,178 +0,0 @@ -import type { - AccountsContext, - LinkOAuthParams, - OAuthAccount, - OAuthProvider, -} from "../types"; -import { withTx } from "./_tx"; - -interface OAuthAccountRow { - id: string; - identity_id: string; - provider: OAuthProvider; - provider_account_id: string; - email: string | null; - access_token: string | null; - refresh_token: string | null; - encryption_key_id: number | null; - token_expires_at: Date | null; - created_at: Date; - updated_at: Date | null; -} - -function rowToOAuthAccount(row: OAuthAccountRow): OAuthAccount { - return { - id: row.id, - identityId: row.identity_id, - provider: row.provider, - providerAccountId: row.provider_account_id, - email: row.email, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function oauthOps(ctx: AccountsContext) { - const { schema, crypto } = ctx; - - return { - async linkOAuthAccount(params: LinkOAuthParams): Promise { - const { - identityId, - provider, - providerAccountId, - email, - accessToken, - refreshToken, - tokenExpiresAt, - } = params; - - // Encrypt tokens - const { ciphertext: encryptedAccess, keyId } = - await crypto.encrypt(accessToken); - const encryptedRefresh = refreshToken - ? (await crypto.encrypt(refreshToken)).ciphertext - : null; - - return withTx(ctx, "linkOAuthAccount", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.oauth_account - (identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at) - values (${identityId}, ${provider}, ${providerAccountId}, ${email ?? null}, ${encryptedAccess}, ${encryptedRefresh}, ${keyId}, ${tokenExpiresAt ?? null}) - on conflict (provider, provider_account_id) - do update set - identity_id = excluded.identity_id, - email = excluded.email, - access_token = excluded.access_token, - refresh_token = excluded.refresh_token, - encryption_key_id = excluded.encryption_key_id, - token_expires_at = excluded.token_expires_at, - updated_at = now() - returning id, identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to link OAuth account"); - } - return rowToOAuthAccount(row); - }); - }, - - async getOAuthAccount( - provider: OAuthProvider, - providerAccountId: string, - ): Promise { - return withTx(ctx, "getOAuthAccount", async (sql) => { - const [row] = await sql` - select id, identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at, created_at, updated_at - from ${sql.unsafe(schema)}.oauth_account - where provider = ${provider} and provider_account_id = ${providerAccountId} - `; - return row ? rowToOAuthAccount(row) : null; - }); - }, - - async getOAuthAccountsByIdentity( - identityId: string, - ): Promise { - return withTx(ctx, "getOAuthAccountsByIdentity", async (sql) => { - const rows = await sql` - select id, identity_id, provider, provider_account_id, email, access_token, refresh_token, encryption_key_id, token_expires_at, created_at, updated_at - from ${sql.unsafe(schema)}.oauth_account - where identity_id = ${identityId} - order by created_at - `; - return rows.map(rowToOAuthAccount); - }); - }, - - async unlinkOAuthAccount(id: string): Promise { - return withTx(ctx, "unlinkOAuthAccount", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.oauth_account - where id = ${id} - `; - return result.count > 0; - }); - }, - - async refreshOAuthTokens( - id: string, - params: { - accessToken: string; - refreshToken?: string; - tokenExpiresAt?: Date; - }, - ): Promise { - const { accessToken, refreshToken, tokenExpiresAt } = params; - - const { ciphertext: encryptedAccess, keyId } = - await crypto.encrypt(accessToken); - const encryptedRefresh = refreshToken - ? (await crypto.encrypt(refreshToken)).ciphertext - : undefined; - - return withTx(ctx, "refreshOAuthTokens", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.oauth_account - set - access_token = ${encryptedAccess}, - ${encryptedRefresh !== undefined ? sql`refresh_token = ${encryptedRefresh},` : sql``} - encryption_key_id = ${keyId}, - ${tokenExpiresAt !== undefined ? sql`token_expires_at = ${tokenExpiresAt},` : sql``} - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async getOAuthTokens( - id: string, - ): Promise<{ accessToken: string; refreshToken: string | null } | null> { - return withTx(ctx, "getOAuthTokens", async (sql) => { - const [row] = await sql` - select access_token, refresh_token, encryption_key_id - from ${sql.unsafe(schema)}.oauth_account - where id = ${id} - `; - - if (!row?.access_token || !row.encryption_key_id) { - return null; - } - - const accessToken = await crypto.decrypt( - row.access_token, - row.encryption_key_id, - ); - const refreshToken = row.refresh_token - ? await crypto.decrypt(row.refresh_token, row.encryption_key_id) - : null; - - return { accessToken, refreshToken }; - }); - }, - }; -} - -export type OAuthOps = ReturnType; diff --git a/packages/accounts/ops/org-member.ts b/packages/accounts/ops/org-member.ts deleted file mode 100644 index 9ea7e2b..0000000 --- a/packages/accounts/ops/org-member.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { - type AccountsContext, - AccountsError, - type OrgMember, - type OrgRole, -} from "../types"; -import { withTx } from "./_tx"; - -interface OrgMemberRow { - org_id: string; - identity_id: string; - role: OrgRole; - created_at: Date; - name: string; - email: string; -} - -function rowToOrgMember(row: OrgMemberRow): OrgMember { - return { - orgId: row.org_id, - identityId: row.identity_id, - role: row.role, - createdAt: row.created_at, - name: row.name, - email: row.email, - }; -} - -/** - * Verify that removing or demoting `identityId` from `orgId` would leave at - * least one other owner in place. Throws ORG_MUST_HAVE_OWNER otherwise. - * - * The query takes `for update` row locks on every other owner row in the - * org. This closes the race where two concurrent transactions, each - * removing a different owner, both observe the other's row as still - * present and proceed — leaving the org with zero owners. With `for - * update`, the second transaction blocks on the first's row locks, sees - * the post-commit count, and correctly fails. The previous DB trigger - * had the same race; this hardens it. - */ -async function assertAnotherOwnerExists( - sql: import("bun").SQL, - schema: string, - orgId: string, - identityId: string, -): Promise { - const rows = await sql<{ identity_id: string }[]>` - select identity_id - from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} - and role = 'owner' - and identity_id <> ${identityId} - for update - `; - if (rows.length === 0) { - throw new AccountsError( - "ORG_MUST_HAVE_OWNER", - "Cannot remove the last owner from an organization", - ); - } -} - -export function orgMemberOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async addMember( - orgId: string, - identityId: string, - role: OrgRole, - ): Promise { - return withTx(ctx, "addMember", async (sql) => { - const rows = await sql` - with inserted as ( - insert into ${sql.unsafe(schema)}.org_member (org_id, identity_id, role) - values (${orgId}, ${identityId}, ${role}) - returning org_id, identity_id, role, created_at - ) - select i.*, id.name, id.email - from inserted i - join ${sql.unsafe(schema)}.identity id on id.id = i.identity_id - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to add member"); - } - return rowToOrgMember(row); - }); - }, - - async removeMember(orgId: string, identityId: string): Promise { - return withTx(ctx, "removeMember", async (sql) => { - // Fetch the target row with `for update` so the row stays stable - // under our feet while we decide whether to delete it. - const [target] = await sql<{ role: OrgRole }[]>` - select role - from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} and identity_id = ${identityId} - for update - `; - if (!target) return false; - - if (target.role === "owner") { - await assertAnotherOwnerExists(sql, schema, orgId, identityId); - } - - const result = await sql` - delete from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} and identity_id = ${identityId} - `; - return result.count > 0; - }); - }, - - async updateRole( - orgId: string, - identityId: string, - newRole: OrgRole, - ): Promise { - return withTx(ctx, "updateRole", async (sql) => { - const [target] = await sql<{ role: OrgRole }[]>` - select role - from ${sql.unsafe(schema)}.org_member - where org_id = ${orgId} and identity_id = ${identityId} - for update - `; - if (!target) return false; - - // Only block the transition that would orphan the org: an owner - // being changed to anything other than owner. - if (target.role === "owner" && newRole !== "owner") { - await assertAnotherOwnerExists(sql, schema, orgId, identityId); - } - - const result = await sql` - update ${sql.unsafe(schema)}.org_member - set role = ${newRole} - where org_id = ${orgId} and identity_id = ${identityId} - `; - return result.count > 0; - }); - }, - - async getMember( - orgId: string, - identityId: string, - ): Promise { - return withTx(ctx, "getMember", async (sql) => { - const [row] = await sql` - select m.org_id, m.identity_id, m.role, m.created_at, id.name, id.email - from ${sql.unsafe(schema)}.org_member m - join ${sql.unsafe(schema)}.identity id on id.id = m.identity_id - where m.org_id = ${orgId} and m.identity_id = ${identityId} - `; - return row ? rowToOrgMember(row) : null; - }); - }, - - async listMembers(orgId: string): Promise { - return withTx(ctx, "listMembers", async (sql) => { - const rows = await sql` - select m.org_id, m.identity_id, m.role, m.created_at, id.name, id.email - from ${sql.unsafe(schema)}.org_member m - join ${sql.unsafe(schema)}.identity id on id.id = m.identity_id - where m.org_id = ${orgId} - order by m.created_at - `; - return rows.map(rowToOrgMember); - }); - }, - - async countOwnedOrgs(identityId: string): Promise { - return withTx(ctx, "countOwnedOrgs", async (sql) => { - const [row] = await sql<{ count: number }[]>` - select count(*)::int as count - from ${sql.unsafe(schema)}.org_member - where identity_id = ${identityId} and role = 'owner' - `; - return row?.count ?? 0; - }); - }, - - async listOwners(orgId: string): Promise { - return withTx(ctx, "listOwners", async (sql) => { - const rows = await sql` - select m.org_id, m.identity_id, m.role, m.created_at, id.name, id.email - from ${sql.unsafe(schema)}.org_member m - join ${sql.unsafe(schema)}.identity id on id.id = m.identity_id - where m.org_id = ${orgId} and m.role = 'owner' - order by m.created_at - `; - return rows.map(rowToOrgMember); - }); - }, - }; -} - -export type OrgMemberOps = ReturnType; diff --git a/packages/accounts/ops/org.ts b/packages/accounts/ops/org.ts deleted file mode 100644 index 96f828b..0000000 --- a/packages/accounts/ops/org.ts +++ /dev/null @@ -1,110 +0,0 @@ -import type { AccountsContext, CreateOrgParams, Org } from "../types"; -import { generateSlug } from "../util/slug"; -import { withTx } from "./_tx"; - -interface OrgRow { - id: string; - slug: string; - name: string; - created_at: Date; - updated_at: Date | null; -} - -function rowToOrg(row: OrgRow): Org { - return { - id: row.id, - slug: row.slug, - name: row.name, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function orgOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createOrg(params: CreateOrgParams): Promise { - const { id, name } = params; - const slug = generateSlug(); - - return withTx(ctx, "createOrg", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.org (id, slug, name) - values (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${slug}, ${name}) - returning id, slug, name, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create org"); - } - return rowToOrg(row); - }); - }, - - async getOrg(id: string): Promise { - return withTx(ctx, "getOrg", async (sql) => { - const [row] = await sql` - select id, slug, name, created_at, updated_at - from ${sql.unsafe(schema)}.org - where id = ${id} - `; - return row ? rowToOrg(row) : null; - }); - }, - - async getOrgBySlug(slug: string): Promise { - return withTx(ctx, "getOrgBySlug", async (sql) => { - const [row] = await sql` - select id, slug, name, created_at, updated_at - from ${sql.unsafe(schema)}.org - where slug = ${slug} - `; - return row ? rowToOrg(row) : null; - }); - }, - - async updateOrg(id: string, params: { name?: string }): Promise { - const { name } = params; - if (name === undefined) { - return false; - } - - return withTx(ctx, "updateOrg", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.org - set - name = ${name}, - updated_at = now() - where id = ${id} - `; - return result.count > 0; - }); - }, - - async deleteOrg(id: string): Promise { - return withTx(ctx, "deleteOrg", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.org - where id = ${id} - `; - return result.count > 0; - }); - }, - - async listOrgsByIdentity(identityId: string): Promise { - return withTx(ctx, "listOrgsByIdentity", async (sql) => { - const rows = await sql` - select o.id, o.slug, o.name, o.created_at, o.updated_at - from ${sql.unsafe(schema)}.org o - inner join ${sql.unsafe(schema)}.org_member m on m.org_id = o.id - where m.identity_id = ${identityId} - order by o.created_at - `; - return rows.map(rowToOrg); - }); - }, - }; -} - -export type OrgOps = ReturnType; diff --git a/packages/accounts/ops/session.ts b/packages/accounts/ops/session.ts deleted file mode 100644 index ed4119b..0000000 --- a/packages/accounts/ops/session.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { - AccountsContext, - CreateSessionParams, - CreateSessionResult, - Identity, - Session, -} from "../types"; -import { generateToken, tokenHash } from "../util/hash"; -import { withTx } from "./_tx"; - -interface SessionRow { - id: string; - identity_id: string; - expires_at: Date; - created_at: Date; -} - -interface SessionWithIdentityRow extends SessionRow { - identity_email: string; - identity_name: string; - identity_created_at: Date; - identity_updated_at: Date | null; -} - -function rowToSession(row: SessionRow): Session { - return { - id: row.id, - identityId: row.identity_id, - expiresAt: row.expires_at, - createdAt: row.created_at, - }; -} - -export function sessionOps(ctx: AccountsContext) { - const { schema } = ctx; - - return { - async createSession( - params: CreateSessionParams, - ): Promise { - const { identityId, expiresInDays = 30 } = params; - - const rawToken = generateToken(); - const hash = tokenHash(rawToken); - const expiresAt = new Date(); - expiresAt.setDate(expiresAt.getDate() + expiresInDays); - - return withTx(ctx, "createSession", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.session (identity_id, token_hash, expires_at) - values (${identityId}, ${hash}, ${expiresAt}) - returning id, identity_id, expires_at, created_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create session"); - } - return { - session: rowToSession(row), - rawToken, - }; - }); - }, - - async validateSession( - rawToken: string, - ): Promise<{ session: Session; identity: Identity } | null> { - const hash = tokenHash(rawToken); - - return withTx(ctx, "validateSession", async (sql) => { - // Single indexed lookup: token_hash is unique. No loop, no argon2. - const rows = await sql` - select - s.id, s.identity_id, s.expires_at, s.created_at, - i.email as identity_email, i.name as identity_name, - i.created_at as identity_created_at, i.updated_at as identity_updated_at - from ${sql.unsafe(schema)}.session s - inner join ${sql.unsafe(schema)}.identity i on i.id = s.identity_id - where s.token_hash = ${hash} - and s.expires_at > now() - limit 1 - `; - const row = rows[0]; - if (!row) { - return null; - } - return { - session: rowToSession(row), - identity: { - id: row.identity_id, - email: row.identity_email, - name: row.identity_name, - createdAt: row.identity_created_at, - updatedAt: row.identity_updated_at, - }, - }; - }); - }, - - async deleteSession(id: string): Promise { - return withTx(ctx, "deleteSession", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.session - where id = ${id} - `; - return result.count > 0; - }); - }, - - async deleteSessionsByIdentity(identityId: string): Promise { - return withTx(ctx, "deleteSessionsByIdentity", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.session - where identity_id = ${identityId} - `; - return result.count; - }); - }, - - async cleanupExpiredSessions(): Promise { - return withTx(ctx, "cleanupExpiredSessions", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.session - where expires_at <= now() - `; - return result.count; - }); - }, - }; -} - -export type SessionOps = ReturnType; diff --git a/packages/accounts/package.json b/packages/accounts/package.json deleted file mode 100644 index 0bab4d5..0000000 --- a/packages/accounts/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "@memory.build/accounts", - "version": "0.2.5", - "private": true, - "type": "module", - "dependencies": { - "@pydantic/logfire-node": "^0.13.1" - } -} diff --git a/packages/accounts/types.ts b/packages/accounts/types.ts deleted file mode 100644 index e11b732..0000000 --- a/packages/accounts/types.ts +++ /dev/null @@ -1,236 +0,0 @@ -import type { SQL } from "bun"; - -// ============================================================================= -// Context -// ============================================================================= - -export interface AccountsCrypto { - encrypt(plaintext: string): Promise<{ ciphertext: string; keyId: number }>; - decrypt(ciphertext: string, keyId: number): Promise; - createDataKey(): Promise; - activateDataKey(keyId: number): Promise; -} - -export interface AccountsContext { - sql: SQL; - schema: string; - inTransaction: boolean; - crypto: AccountsCrypto; -} - -// ============================================================================= -// Errors -// ============================================================================= - -export type AccountsErrorCode = - | "ORG_MUST_HAVE_OWNER" - | "IDENTITY_NOT_FOUND" - | "ORG_NOT_FOUND" - | "ENGINE_NOT_FOUND" - | "INVITATION_NOT_FOUND" - | "INVITATION_EXPIRED" - | "INVITATION_ALREADY_ACCEPTED" - | "SESSION_NOT_FOUND" - | "SESSION_EXPIRED" - | "OAUTH_ACCOUNT_NOT_FOUND" - | "DUPLICATE_SLUG" - | "DUPLICATE_EMAIL" - | "ENCRYPTION_KEY_NOT_FOUND" - | "NO_ACTIVE_ENCRYPTION_KEY"; - -export class AccountsError extends Error { - constructor( - public code: AccountsErrorCode, - message: string, - ) { - super(message); - this.name = "AccountsError"; - } -} - -// ============================================================================= -// Identity -// ============================================================================= - -export interface Identity { - id: string; - email: string; - name: string; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateIdentityParams { - id?: string; - email: string; - name: string; -} - -// ============================================================================= -// Org -// ============================================================================= - -export interface Org { - id: string; - slug: string; - name: string; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateOrgParams { - id?: string; - name: string; -} - -// ============================================================================= -// OrgMember -// ============================================================================= - -export type OrgRole = "owner" | "admin" | "member"; - -export interface OrgMember { - orgId: string; - identityId: string; - role: OrgRole; - createdAt: Date; - name: string; - email: string; -} - -// ============================================================================= -// Engine -// ============================================================================= - -export type EngineStatus = "active" | "suspended" | "deleted"; - -export interface Engine { - id: string; - orgId: string; - slug: string; - name: string; - shardId: number; - status: EngineStatus; - language: string; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateEngineParams { - id?: string; - orgId: string; - name: string; - shardId?: number; - language?: string; // defaults to 'english' -} - -// ============================================================================= -// Invitation -// ============================================================================= - -export interface Invitation { - id: string; - orgId: string; - email: string; - role: OrgRole; - invitedBy: string; - expiresAt: Date; - acceptedAt: Date | null; - createdAt: Date; -} - -export interface CreateInvitationParams { - orgId: string; - email: string; - role: OrgRole; - invitedBy: string; - expiresInDays?: number; -} - -export interface CreateInvitationResult { - invitation: Invitation; - rawToken: string; -} - -// ============================================================================= -// OAuthAccount -// ============================================================================= - -export type OAuthProvider = "google" | "github"; - -export interface OAuthAccount { - id: string; - identityId: string; - provider: OAuthProvider; - providerAccountId: string; - email: string | null; - createdAt: Date; - updatedAt: Date | null; -} - -export interface LinkOAuthParams { - identityId: string; - provider: OAuthProvider; - providerAccountId: string; - email?: string; - accessToken: string; - refreshToken?: string; - tokenExpiresAt?: Date; -} - -// ============================================================================= -// Session -// ============================================================================= - -export interface Session { - id: string; - identityId: string; - expiresAt: Date; - createdAt: Date; -} - -export interface CreateSessionParams { - identityId: string; - expiresInDays?: number; -} - -export interface CreateSessionResult { - session: Session; - rawToken: string; -} - -// ============================================================================= -// EncryptionKey -// ============================================================================= - -export interface EncryptionKey { - id: number; - active: boolean; - createdAt: Date; -} - -// ============================================================================= -// DeviceAuthorization (OAuth Device Flow) -// ============================================================================= - -export type DeviceProvider = "google" | "github"; - -export interface DeviceAuthorization { - deviceCode: string; - userCode: string; - provider: DeviceProvider; - oauthState: string; - expiresAt: Date; - lastPoll: Date | null; - identityId: string | null; - denied: boolean; - createdAt: Date; -} - -export interface CreateDeviceAuthParams { - deviceCode: string; - userCode: string; - provider: DeviceProvider; - oauthState: string; - expiresAt: Date; -} diff --git a/packages/accounts/util/crypto.ts b/packages/accounts/util/crypto.ts deleted file mode 100644 index f8a2338..0000000 --- a/packages/accounts/util/crypto.ts +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Envelope encryption for OAuth tokens - * - * Uses a master key (from environment) to encrypt data keys stored in the DB. - * Data keys encrypt the actual OAuth tokens. This allows key rotation without - * re-encrypting all tokens at once. - * - * Algorithm: AES-256-GCM - * Ciphertext format: {iv}:{ciphertext}:{authTag} (all base64) - */ - -import type { SQL } from "bun"; -import type { AccountsCrypto } from "../types"; - -const ALGORITHM = "AES-GCM"; -const KEY_LENGTH = 256; -const IV_LENGTH = 12; -const TAG_LENGTH = 128; - -interface EncryptionKeyRow { - id: number; - key_ciphertext: Buffer; - active: boolean; - created_at: Date; -} - -/** - * Import a raw key for AES-GCM operations - */ -async function importKey(keyBytes: Buffer | Uint8Array): Promise { - // Copy into a fresh ArrayBuffer for Web Crypto API compatibility - // (Buffer's underlying buffer may be SharedArrayBuffer which Web Crypto rejects) - const bytes = new Uint8Array(keyBytes.length); - bytes.set(keyBytes); - - return crypto.subtle.importKey( - "raw", - bytes, - { name: ALGORITHM, length: KEY_LENGTH }, - false, - ["encrypt", "decrypt"], - ); -} - -/** - * Encrypt plaintext using AES-256-GCM - */ -async function encryptAES(key: CryptoKey, plaintext: string): Promise { - const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH)); - const encoder = new TextEncoder(); - const data = encoder.encode(plaintext); - - const ciphertext = await crypto.subtle.encrypt( - { name: ALGORITHM, iv, tagLength: TAG_LENGTH }, - key, - data, - ); - - // Split ciphertext and auth tag (last 16 bytes) - const ciphertextBytes = new Uint8Array(ciphertext.slice(0, -16)); - const authTag = new Uint8Array(ciphertext.slice(-16)); - - const ivB64 = btoa(String.fromCharCode(...iv)); - const ciphertextB64 = btoa(String.fromCharCode(...ciphertextBytes)); - const tagB64 = btoa(String.fromCharCode(...authTag)); - - return `${ivB64}:${ciphertextB64}:${tagB64}`; -} - -/** - * Decrypt ciphertext using AES-256-GCM - */ -async function decryptAES(key: CryptoKey, ciphertext: string): Promise { - const [ivB64, ciphertextB64, tagB64] = ciphertext.split(":"); - if (!ivB64 || !ciphertextB64 || !tagB64) { - throw new Error("Invalid ciphertext format"); - } - - const iv = Uint8Array.from(atob(ivB64), (c) => c.charCodeAt(0)); - const ciphertextBytes = Uint8Array.from(atob(ciphertextB64), (c) => - c.charCodeAt(0), - ); - const authTag = Uint8Array.from(atob(tagB64), (c) => c.charCodeAt(0)); - - // Combine ciphertext and auth tag for Web Crypto API - const combined = new Uint8Array(ciphertextBytes.length + authTag.length); - combined.set(ciphertextBytes); - combined.set(authTag, ciphertextBytes.length); - - const plaintext = await crypto.subtle.decrypt( - { name: ALGORITHM, iv, tagLength: TAG_LENGTH }, - key, - combined, - ); - - return new TextDecoder().decode(plaintext); -} - -/** - * Create an AccountsCrypto instance for envelope encryption - */ -export function createAccountsCrypto( - masterKey: Buffer, - ctx: { sql: SQL; schema: string }, -): AccountsCrypto { - // Cache for decrypted data keys - const keyCache = new Map(); - let masterCryptoKey: CryptoKey | null = null; - - async function getMasterKey(): Promise { - if (!masterCryptoKey) { - masterCryptoKey = await importKey(masterKey); - } - return masterCryptoKey; - } - - async function getDataKey(keyId: number): Promise { - const cached = keyCache.get(keyId); - if (cached) { - return cached; - } - - const { sql, schema } = ctx; - const [row] = await sql` - select id, key_ciphertext, active, created_at - from ${sql.unsafe(schema)}.encryption_key - where id = ${keyId} - `; - - if (!row) { - throw new Error(`Encryption key ${keyId} not found`); - } - - const master = await getMasterKey(); - const dataKeyBytes = await decryptAES( - master, - row.key_ciphertext.toString("utf-8"), - ); - const dataKey = await importKey(Buffer.from(dataKeyBytes, "base64")); - - keyCache.set(keyId, dataKey); - return dataKey; - } - - async function getActiveKeyId(): Promise { - const { sql, schema } = ctx; - const [row] = await sql<{ id: number }[]>` - select id from ${sql.unsafe(schema)}.encryption_key - where active = true - `; - - if (!row) { - throw new Error("No active encryption key"); - } - - return row.id; - } - - return { - async encrypt( - plaintext: string, - ): Promise<{ ciphertext: string; keyId: number }> { - const keyId = await getActiveKeyId(); - const dataKey = await getDataKey(keyId); - const ciphertext = await encryptAES(dataKey, plaintext); - return { ciphertext, keyId }; - }, - - async decrypt(ciphertext: string, keyId: number): Promise { - const dataKey = await getDataKey(keyId); - return decryptAES(dataKey, ciphertext); - }, - - async createDataKey(): Promise { - const { sql, schema } = ctx; - - // Generate a random 256-bit key - const dataKeyBytes = crypto.getRandomValues(new Uint8Array(32)); - const dataKeyB64 = btoa(String.fromCharCode(...dataKeyBytes)); - - // Encrypt with master key - const master = await getMasterKey(); - const keyCiphertext = await encryptAES(master, dataKeyB64); - - const [row] = await sql<{ id: number }[]>` - insert into ${sql.unsafe(schema)}.encryption_key (key_ciphertext, active) - values (${keyCiphertext}::bytea, false) - returning id - `; - - if (!row) { - throw new Error("Failed to create encryption key"); - } - - return row.id; - }, - - async activateDataKey(keyId: number): Promise { - const { sql, schema } = ctx; - - // Deactivate all keys, then activate the specified one - await sql` - update ${sql.unsafe(schema)}.encryption_key - set active = false - where active = true - `; - - const result = await sql` - update ${sql.unsafe(schema)}.encryption_key - set active = true - where id = ${keyId} - `; - - if (result.count === 0) { - throw new Error(`Encryption key ${keyId} not found`); - } - - // Clear cache since active key changed - keyCache.clear(); - }, - }; -} diff --git a/packages/accounts/util/hash.test.ts b/packages/accounts/util/hash.test.ts deleted file mode 100644 index 0b37721..0000000 --- a/packages/accounts/util/hash.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { generateToken, tokenHash } from "./hash"; - -describe("generateToken", () => { - test("produces a base64url string of the expected length", () => { - const token = generateToken(); - // 32 bytes base64url-encoded with padding stripped: ceil(32 / 3) * 4 = 44, - // minus the trailing '=' padding character → 43 chars. - expect(token).toHaveLength(43); - expect(token).toMatch(/^[A-Za-z0-9_-]+$/); - }); - - test("produces unique values across calls", () => { - const seen = new Set(); - for (let i = 0; i < 100; i++) { - seen.add(generateToken()); - } - expect(seen.size).toBe(100); - }); -}); - -describe("tokenHash", () => { - test("returns 32 raw bytes", () => { - const hash = tokenHash("anything"); - expect(hash).toBeInstanceOf(Buffer); - expect(hash.length).toBe(32); - }); - - test("is deterministic", () => { - const a = tokenHash("hello world"); - const b = tokenHash("hello world"); - expect(a.equals(b)).toBe(true); - }); - - test("differs for different inputs", () => { - const a = tokenHash("hello world"); - const b = tokenHash("hello world!"); - expect(a.equals(b)).toBe(false); - }); - - test("matches the published sha256 of a known string", () => { - // Known value: sha256("abc") = ba7816bf8f01cfea414140de5dae2223 - // b00361a396177a9cb410ff61f20015ad - const hex = tokenHash("abc").toString("hex"); - expect(hex).toBe( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - ); - }); -}); diff --git a/packages/accounts/util/hash.ts b/packages/accounts/util/hash.ts deleted file mode 100644 index 37c9ccf..0000000 --- a/packages/accounts/util/hash.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Token utilities for session and invitation tokens. - * - * Tokens are 256-bit CSPRNG output (base64url-encoded). We store sha256(token) - * in a unique-indexed column and look up by it directly — no slow-hash verifier - * is needed because the token's entropy alone defeats offline preimage attacks. - * See migration 009_session_lookup.sql for the rationale. - */ - -/** - * Generate a random token (32 bytes, base64url encoded). - */ -export function generateToken(): string { - const bytes = crypto.getRandomValues(new Uint8Array(32)); - return btoa(String.fromCharCode(...bytes)) - .replace(/\+/g, "-") - .replace(/\//g, "_") - .replace(/=/g, ""); -} - -/** - * Compute the lookup hash stored in the session/invitation `token_hash` column. - * Returns 32 raw bytes suitable for binding directly to a `bytea` parameter. - */ -export function tokenHash(rawToken: string): Buffer { - return new Bun.CryptoHasher("sha256").update(rawToken).digest(); -} diff --git a/packages/accounts/util/slug.ts b/packages/accounts/util/slug.ts deleted file mode 100644 index 4bb81b5..0000000 --- a/packages/accounts/util/slug.ts +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Slug generation - * - * Generates a 12-character alphanumeric slug (a-z, 0-9) for org and engine identification. - */ - -const SLUG_LENGTH = 12; -const SLUG_CHARSET = "abcdefghijklmnopqrstuvwxyz0123456789"; - -/** - * Generate a random slug (12 lowercase alphanumeric chars) - */ -export function generateSlug(): string { - const bytes = crypto.getRandomValues(new Uint8Array(SLUG_LENGTH)); - let result = ""; - for (const byte of bytes) { - result += SLUG_CHARSET[byte % SLUG_CHARSET.length]; - } - return result; -} diff --git a/packages/auth/db.integration.test.ts b/packages/auth/db.integration.test.ts new file mode 100644 index 0000000..ff337ec --- /dev/null +++ b/packages/auth/db.integration.test.ts @@ -0,0 +1,150 @@ +// Integration tests for the auth runtime layer (authStore). +// +// Provisions a throwaway auth_test_ schema via migrateAuth and exercises +// the wrappers against the real SQL functions. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/auth/db.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateAuth } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { type AuthStore, authStore } from "./db"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +const randomAuthSchema = () => { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += ALPHABET[b % 36]; + return `auth_test_${s}`; +}; +const email = () => `fn_${crypto.randomUUID().slice(0, 8)}@example.com`; + +let sql: Sql; +let schema: string; +let db: AuthStore; + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + schema = randomAuthSchema(); + await migrateAuth(sql, { schema }); + db = authStore(sql, schema); +}); + +afterAll(async () => { + if (schema) await sql.unsafe(`drop schema if exists ${schema} cascade`); + await sql.end(); +}); + +test("createUser + getUser + getUserByEmail (case-insensitive)", async () => { + const e = email(); + const id = await db.createUser(e, "Alice", { emailVerified: true }); + + const byId = await db.getUser(id); + expect(byId?.id).toBe(id); + expect(byId?.email).toBe(e); + expect(byId?.emailVerified).toBe(true); + + const byEmail = await db.getUserByEmail(e.toUpperCase()); + expect(byEmail?.id).toBe(id); + + expect(await db.getUserByEmail(email())).toBeNull(); +}); + +test("createSession returns a token that validateSession accepts", async () => { + const id = await db.createUser(email(), "Bob"); + const { sessionId, token } = await db.createSession(id); + expect(token.length).toBeGreaterThan(20); + + const v = await db.validateSession(token); + expect(v?.sessionId).toBe(sessionId); + expect(v?.userId).toBe(id); + + // wrong token → null + expect(await db.validateSession("not-a-real-token")).toBeNull(); + + // after delete → null + expect(await db.deleteSession(sessionId)).toBe(true); + expect(await db.validateSession(token)).toBeNull(); +}); + +test("validateSession rolls a near-expiry session forward, then throttles", async () => { + const id = await db.createUser(email(), "Rolling"); + // Start with a 1-day session so it's inside the refresh threshold (<6d left). + const { token } = await db.createSession(id, { expiresInDays: 1 }); + + // First use slides the expiry forward to ~7 days out (the window). + const before = await db.validateSession(token); + expect(before).not.toBeNull(); + const sixDaysOut = Date.now() + 6 * 24 * 60 * 60 * 1000; + expect(before?.expiresAt.getTime()).toBeGreaterThan(sixDaysOut); + + // A full-window session is NOT bumped again on the next use (≤ once/day + // throttle): the stored expiry is unchanged, not sliding on every call. + const after = await db.validateSession(token); + expect(after?.expiresAt.getTime()).toBe(before?.expiresAt.getTime()); +}); + +test("deleteSessionsByUser revokes all of a user's sessions", async () => { + const id = await db.createUser(email(), "Carol"); + const a = await db.createSession(id); + const b = await db.createSession(id); + + expect(await db.deleteSessionsByUser(id)).toBe(2); + expect(await db.validateSession(a.token)).toBeNull(); + expect(await db.validateSession(b.token)).toBeNull(); +}); + +test("upsertAccount + getAccountByProvider", async () => { + const id = await db.createUser(email(), "Dave"); + const acct = crypto.randomUUID(); + + await db.upsertAccount(id, "github", acct); + const found = await db.getAccountByProvider("github", acct); + expect(found?.userId).toBe(id); + expect(found?.providerId).toBe("github"); + + // idempotent + await db.upsertAccount(id, "github", acct); + expect((await db.getAccountsByUser(id)).length).toBe(1); + + expect( + await db.getAccountByProvider("github", crypto.randomUUID()), + ).toBeNull(); +}); + +test("device flow: create → lookup (normalized code) → poll → authorize", async () => { + const id = await db.createUser(email(), "Erin"); + const { deviceCode, userCode, oauthState } = + await db.createDeviceAuth("google"); + + // user_code lookup tolerates lowercase / missing hyphen + const denorm = userCode.toLowerCase().replace("-", ""); + const byUserCode = await db.getDeviceByUserCode(denorm); + expect(byUserCode?.deviceCode).toBe(deviceCode); + + const byState = await db.getDeviceByOAuthState(oauthState); + expect(byState?.deviceCode).toBe(deviceCode); + + // pending → bind (callback) → still pending → approve (consent) → authorized + expect((await db.pollDevice(deviceCode, 0)).status).toBe("pending"); + expect(await db.bindDeviceUser(deviceCode, id)).toBe(true); + expect((await db.pollDevice(deviceCode, 0)).status).toBe("pending"); + expect(await db.approveDevice(deviceCode)).toBe(true); + const poll = await db.pollDevice(deviceCode, 0); + expect(poll.status).toBe("approved"); + expect(poll.userId).toBe(id); +}); + +test("withTransaction rolls back on error", async () => { + const e = email(); + await expect( + db.withTransaction(async (tx) => { + await tx.createUser(e, "Rollback"); + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + expect(await db.getUserByEmail(e)).toBeNull(); +}); diff --git a/packages/auth/db.ts b/packages/auth/db.ts new file mode 100644 index 0000000..248ef1b --- /dev/null +++ b/packages/auth/db.ts @@ -0,0 +1,301 @@ +import { AUTH_SCHEMA } from "@memory.build/database"; +import type { Sql } from "postgres"; +import { + DEVICE_CODE_EXPIRY_SECONDS, + generateDeviceCode, + generateOAuthState, + generateSessionToken, + generateUserCode, + hashSessionToken, + normalizeUserCode, +} from "./token"; +import type { + Account, + CreatedDeviceAuth, + CreatedSession, + CreateUserOptions, + DevicePollResult, + DevicePollStatus, + DeviceStatus, + OAuthProvider, + User, + ValidatedSession, +} from "./types"; + +// Initial session lifetime at login. Sessions are rolling: validate_session +// slides expiry forward to now + this window on use (throttled to ~once/day), +// with no absolute cap — better-auth's model (expiresIn=7d, updateAge=1d). Keep +// this in sync with the window in auth migrate 002_session.sql validate_session. +const SESSION_EXPIRY_DAYS = 7; + +/** + * The auth control-plane data layer. + * + * Thin wrappers over the auth schema SQL functions; every method calls a + * function (none query auth tables directly). Token generation/hashing is the + * only TS-side logic (the DB stores only hashes). + */ +export interface AuthStore { + createUser( + email: string, + name: string, + opts?: CreateUserOptions, + ): Promise; + getUser(id: string): Promise; + getUserByEmail(email: string): Promise; + + /** Mint a session; returns the one-time raw token (only its hash is stored). */ + createSession( + userId: string, + opts?: { expiresInDays?: number }, + ): Promise; + validateSession(token: string): Promise; + deleteSession(id: string): Promise; + deleteSessionsByUser(userId: string): Promise; + cleanupExpiredSessions(): Promise; + + upsertAccount( + userId: string, + providerId: OAuthProvider, + accountId: string, + ): Promise; + getAccountByProvider( + providerId: OAuthProvider, + accountId: string, + ): Promise; + getAccountsByUser(userId: string): Promise; + unlinkAccount(id: string): Promise; + + /** Start a device flow; generates the codes and returns them. */ + createDeviceAuth(provider: OAuthProvider): Promise; + getDeviceByUserCode(userCode: string): Promise; + getDeviceByOAuthState(oauthState: string): Promise; + /** Resolve the poll state machine in one call (see poll_device). */ + pollDevice( + deviceCode: string, + minIntervalSecs?: number, + ): Promise; + /** Callback bound the resolved user (status stays 'pending' until consent). */ + bindDeviceUser(deviceCode: string, userId: string): Promise; + /** Consent: approve the bound device (→ 'approved'). */ + approveDevice(deviceCode: string): Promise; + /** Consent denied, or OAuth failed (→ 'denied'). */ + denyDevice(deviceCode: string): Promise; + deleteDevice(deviceCode: string): Promise; + deleteExpiredDevices(): Promise; + + withTransaction(fn: (db: AuthStore) => Promise): Promise; +} + +/** A device authorization row (the get_device_by_* lookups). */ +export interface DeviceAuthRow { + deviceCode: string; + userCode: string; + provider: OAuthProvider; + oauthState: string; + expiresAt: Date; + lastPoll: Date | null; + userId: string | null; + status: DeviceStatus; + createdAt: Date; +} + +function mapUser(row: Record): User { + return { + id: row.id as string, + email: row.email as string, + name: row.name as string, + emailVerified: Boolean(row.email_verified), + image: (row.image as string | null) ?? null, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapAccount(row: Record): Account { + return { + id: row.id as string, + userId: row.user_id as string, + providerId: row.provider_id as OAuthProvider, + accountId: row.account_id as string, + }; +} + +function mapDevice(row: Record): DeviceAuthRow { + return { + deviceCode: row.device_code as string, + userCode: row.user_code as string, + provider: row.provider as OAuthProvider, + oauthState: row.oauth_state as string, + expiresAt: row.expires_at as Date, + lastPoll: (row.last_poll as Date | null) ?? null, + userId: (row.user_id as string | null) ?? null, + status: row.status as DeviceStatus, + createdAt: row.created_at as Date, + }; +} + +export function authStore(sql: Sql, schema: string = AUTH_SCHEMA): AuthStore { + const sch = sql(schema); + + const db: AuthStore = { + async createUser(email, name, opts) { + const [row] = await sql` + select ${sch}.create_user( + ${email}, ${name}, ${opts?.emailVerified ?? false}, ${opts?.image ?? null} + ) as id`; + if (!row) throw new Error("create_user returned no row"); + return row.id as string; + }, + + async getUser(id) { + const [row] = await sql`select * from ${sch}.get_user(${id})`; + return row ? mapUser(row) : null; + }, + + async getUserByEmail(email) { + const [row] = await sql`select * from ${sch}.get_user_by_email(${email})`; + return row ? mapUser(row) : null; + }, + + async createSession(userId, opts) { + const token = generateSessionToken(); + const tokenHash = hashSessionToken(token); + const days = opts?.expiresInDays ?? SESSION_EXPIRY_DAYS; + const expiresAt = new Date(Date.now() + days * 24 * 60 * 60 * 1000); + const [row] = await sql` + select ${sch}.create_session(${userId}, ${tokenHash}, ${expiresAt}) as id`; + if (!row) throw new Error("create_session returned no row"); + return { sessionId: row.id as string, token }; + }, + + async validateSession(token) { + const tokenHash = hashSessionToken(token); + const [row] = await sql` + select * from ${sch}.validate_session(${tokenHash})`; + if (!row) return null; + return { + sessionId: row.session_id as string, + userId: row.user_id as string, + email: row.email as string, + name: row.name as string, + expiresAt: row.expires_at as Date, + }; + }, + + async deleteSession(id) { + const [row] = await sql`select ${sch}.delete_session(${id}) as ok`; + return Boolean(row?.ok); + }, + + async deleteSessionsByUser(userId) { + const [row] = await sql` + select ${sch}.delete_sessions_by_user(${userId}) as n`; + return Number(row?.n); + }, + + async cleanupExpiredSessions() { + const [row] = await sql`select ${sch}.cleanup_expired_sessions() as n`; + return Number(row?.n); + }, + + async upsertAccount(userId, providerId, accountId) { + const [row] = await sql` + select ${sch}.upsert_account(${userId}, ${providerId}, ${accountId}) as id`; + if (!row) throw new Error("upsert_account returned no row"); + return row.id as string; + }, + + async getAccountByProvider(providerId, accountId) { + const [row] = await sql` + select * from ${sch}.get_account_by_provider(${providerId}, ${accountId})`; + return row ? mapAccount(row) : null; + }, + + async getAccountsByUser(userId) { + const rows = await sql` + select * from ${sch}.get_accounts_by_user(${userId})`; + return rows.map(mapAccount); + }, + + async unlinkAccount(id) { + const [row] = await sql`select ${sch}.unlink_account(${id}) as ok`; + return Boolean(row?.ok); + }, + + async createDeviceAuth(provider) { + const deviceCode = generateDeviceCode(); + const userCode = generateUserCode(); + const oauthState = generateOAuthState(); + const expiresAt = new Date( + Date.now() + DEVICE_CODE_EXPIRY_SECONDS * 1000, + ); + await sql` + select ${sch}.create_device_auth( + ${deviceCode}, ${userCode}, ${provider}, ${oauthState}, ${expiresAt} + )`; + return { + deviceCode, + userCode, + oauthState, + expiresIn: DEVICE_CODE_EXPIRY_SECONDS, + }; + }, + + async getDeviceByUserCode(userCode) { + const [row] = await sql` + select * from ${sch}.get_device_by_user_code(${normalizeUserCode(userCode)})`; + return row ? mapDevice(row) : null; + }, + + async getDeviceByOAuthState(oauthState) { + const [row] = await sql` + select * from ${sch}.get_device_by_oauth_state(${oauthState})`; + return row ? mapDevice(row) : null; + }, + + async pollDevice(deviceCode, minIntervalSecs) { + const [row] = await sql` + select * from ${sch}.poll_device(${deviceCode}, ${minIntervalSecs ?? 5})`; + return { + status: (row?.status as DevicePollStatus) ?? "expired", + userId: (row?.user_id as string | null) ?? null, + }; + }, + + async bindDeviceUser(deviceCode, userId) { + const [row] = await sql` + select ${sch}.bind_device_user(${deviceCode}, ${userId}) as ok`; + return Boolean(row?.ok); + }, + + async approveDevice(deviceCode) { + const [row] = await sql` + select ${sch}.approve_device(${deviceCode}) as ok`; + return Boolean(row?.ok); + }, + + async denyDevice(deviceCode) { + const [row] = await sql`select ${sch}.deny_device(${deviceCode}) as ok`; + return Boolean(row?.ok); + }, + + async deleteDevice(deviceCode) { + const [row] = await sql`select ${sch}.delete_device(${deviceCode}) as ok`; + return Boolean(row?.ok); + }, + + async deleteExpiredDevices() { + const [row] = await sql`select ${sch}.delete_expired_devices() as n`; + return Number(row?.n); + }, + + async withTransaction(fn: (db: AuthStore) => Promise): Promise { + return sql.begin((tx) => + fn(authStore(tx as unknown as Sql, schema)), + ) as Promise; + }, + }; + + return db; +} diff --git a/packages/auth/index.ts b/packages/auth/index.ts new file mode 100644 index 0000000..cd864fd --- /dev/null +++ b/packages/auth/index.ts @@ -0,0 +1,22 @@ +export { type AuthStore, authStore, type DeviceAuthRow } from "./db"; +export { + DEVICE_CODE_EXPIRY_SECONDS, + generateDeviceCode, + generateOAuthState, + generateSessionToken, + generateUserCode, + hashSessionToken, + normalizeUserCode, +} from "./token"; +export type { + Account, + CreatedDeviceAuth, + CreatedSession, + CreateUserOptions, + DevicePollResult, + DevicePollStatus, + DeviceStatus, + OAuthProvider, + User, + ValidatedSession, +} from "./types"; diff --git a/packages/auth/package.json b/packages/auth/package.json new file mode 100644 index 0000000..d5011b2 --- /dev/null +++ b/packages/auth/package.json @@ -0,0 +1,10 @@ +{ + "name": "@memory.build/auth", + "version": "0.0.0", + "private": true, + "type": "module", + "dependencies": { + "@memory.build/database": "workspace:*", + "postgres": "^3.4.9" + } +} diff --git a/packages/auth/token.ts b/packages/auth/token.ts new file mode 100644 index 0000000..3bbea94 --- /dev/null +++ b/packages/auth/token.ts @@ -0,0 +1,62 @@ +/** + * Token + device-code helpers for the auth layer. + * + * Session tokens are 256-bit CSPRNG values; we store sha256(token) (bytea) and + * look up by hash — entropy alone defeats offline preimage attacks, so a fast + * hash is sufficient and a DB read never yields usable bearer tokens. + */ + +const SESSION_TOKEN_BYTES = 32; +const DEVICE_CODE_BYTES = 32; +const OAUTH_STATE_BYTES = 16; + +/** User code: 8 chars, excluding ambiguous 0/O/1/I/L, shown as XXXX-XXXX. */ +const USER_CODE_CHARS = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"; + +/** Device authorization lifetime (15 minutes), in seconds. */ +export const DEVICE_CODE_EXPIRY_SECONDS = 15 * 60; + +function base64url(bytes: Uint8Array): string { + return btoa(String.fromCharCode(...bytes)) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=/g, ""); +} + +function randomBase64url(byteLength: number): string { + return base64url(crypto.getRandomValues(new Uint8Array(byteLength))); +} + +/** Generate a random 256-bit session token (base64url). */ +export function generateSessionToken(): string { + return randomBase64url(SESSION_TOKEN_BYTES); +} + +/** sha256(token) as raw bytes, for the `token_hash` bytea column. */ +export function hashSessionToken(token: string): Buffer { + return new Bun.CryptoHasher("sha256").update(token).digest(); +} + +/** Device flow: the CLI polling secret (base64url, 32 bytes). */ +export function generateDeviceCode(): string { + return randomBase64url(DEVICE_CODE_BYTES); +} + +/** Device flow: the OAuth `state` (CSRF binding, base64url, 16 bytes). */ +export function generateOAuthState(): string { + return randomBase64url(OAUTH_STATE_BYTES); +} + +/** Device flow: the human-entered code, formatted XXXX-XXXX. */ +export function generateUserCode(): string { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let code = ""; + for (const b of bytes) code += USER_CODE_CHARS[b % USER_CODE_CHARS.length]; + return `${code.slice(0, 4)}-${code.slice(4)}`; +} + +/** Normalize user-entered codes (uppercase, strip hyphens, re-hyphenate). */ +export function normalizeUserCode(input: string): string { + const c = input.toUpperCase().replace(/-/g, ""); + return `${c.slice(0, 4)}-${c.slice(4)}`; +} diff --git a/packages/auth/types.ts b/packages/auth/types.ts new file mode 100644 index 0000000..af81145 --- /dev/null +++ b/packages/auth/types.ts @@ -0,0 +1,68 @@ +/** + * Types for the auth runtime layer (authStore). + * + * Thin wrappers over the auth schema SQL functions + * (packages/database/auth/migrate/idempotent/*.sql). No table queries in TS. + */ + +export type OAuthProvider = "google" | "github"; + +export interface User { + id: string; + email: string; + name: string; + emailVerified: boolean; + image: string | null; + createdAt: Date; + updatedAt: Date | null; +} + +export interface CreateUserOptions { + emailVerified?: boolean; + image?: string; +} + +/** What validate_session returns: the session plus its user. */ +export interface ValidatedSession { + sessionId: string; + userId: string; + email: string; + name: string; + expiresAt: Date; +} + +/** A freshly minted session — the raw token is returned once, only its hash is stored. */ +export interface CreatedSession { + sessionId: string; + token: string; +} + +export interface Account { + id: string; + userId: string; + providerId: OAuthProvider; + accountId: string; +} + +export interface CreatedDeviceAuth { + deviceCode: string; + userCode: string; + oauthState: string; + /** Seconds until the device authorization expires. */ + expiresIn: number; +} + +/** The stored device_authorization state (better-auth-shaped). */ +export type DeviceStatus = "pending" | "approved" | "denied"; + +/** + * The poll-result vocabulary returned by poll_device: the stored DeviceStatus + * (pending|approved|denied) passed straight through, plus two poll-only outcomes. + */ +export type DevicePollStatus = DeviceStatus | "expired" | "slow_down"; + +export interface DevicePollResult { + status: DevicePollStatus; + /** Set only when status === "approved". */ + userId: string | null; +} diff --git a/packages/claude-plugin/.claude-plugin/plugin.json b/packages/claude-plugin/.claude-plugin/plugin.json index 63d7eca..ea6c91d 100644 --- a/packages/claude-plugin/.claude-plugin/plugin.json +++ b/packages/claude-plugin/.claude-plugin/plugin.json @@ -12,10 +12,10 @@ "userConfig": { "api_key": { "type": "string", - "title": "API key", - "description": "Memory Engine API key. Create one with `me apikey create` (or have an admin create one with restricted privileges for your agent).", + "title": "API key (optional)", + "description": "Optional. Leave blank to use your `me login` session — the plugin falls back to it automatically. Set an API key to attribute captures to a dedicated agent instead: create one with `me apikey create` (or have an admin create one with restricted privileges).", "sensitive": true, - "required": true + "required": false }, "server": { "type": "string", @@ -24,12 +24,25 @@ "default": "https://api.memory.build", "required": true }, - "tree_prefix": { + "space": { + "type": "string", + "title": "Space (optional)", + "description": "Space slug to capture into (the X-Me-Space). Leave blank to use your active space (`me space use` / ME_SPACE). Pin it for unattended or project-scope installs so captures always land in the same space; if blank and no active space is set, captures are skipped.", + "required": false + }, + "tree_root": { "type": "string", - "title": "Tree prefix", - "description": "Ltree path prefix where captured prompts/responses are stored (e.g. `claude_code.sessions`).", - "default": "claude_code.sessions", + "title": "Tree root", + "description": "Ltree root under which captures are nested as `..agent_sessions` (the same layout `me ... import` uses, so live and imported sessions share a node per project). Defaults under `share` so your login session can write there; use a `~`-prefixed root to keep captures in your private home instead.", + "default": "share.projects", "required": true + }, + "content_mode": { + "type": "string", + "title": "Content mode", + "description": "What to capture per message. `default` stores the user + assistant text (recommended). `full_transcript` also stores reasoning and tool calls/results as their own memories — more complete but much larger/noisier (and may include sensitive tool output).", + "default": "default", + "required": false } } } diff --git a/packages/claude-plugin/.mcp.json b/packages/claude-plugin/.mcp.json index 677100c..c518cbf 100644 --- a/packages/claude-plugin/.mcp.json +++ b/packages/claude-plugin/.mcp.json @@ -7,7 +7,9 @@ "--server", "${user_config.server}", "--api-key", - "${user_config.api_key}" + "${user_config.api_key}", + "--space", + "${user_config.space}" ] } } diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index ef6fff7..9979c33 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -5,9 +5,12 @@ Captures your Claude Code conversations to [Memory Engine](https://memory.build) ## Components - **MCP server** (`me mcp`) — memory tools (search, create, get, update, delete, tree, import, export, etc.) available to the agent during sessions. -- **Hooks**: - - `UserPromptSubmit` captures your prompt as a memory. - - `Stop` captures the agent's final response as a memory. +- **Hooks** — capture the session transcript as memories, one per message: + - `Stop` fires after each turn and imports the session-so-far. + - `SessionEnd` does a final import when the session closes. + - This is the **same parse + write as `me … import`** (incremental: each fire + only writes messages new since the last), so live captures and bulk imports + land in the same place with the same metadata. - **Async, best-effort**. Hooks never block your session; failures log to stderr and exit 0. ## Prerequisites @@ -18,35 +21,30 @@ Captures your Claude Code conversations to [Memory Engine](https://memory.build) curl -fsSL https://install.memory.build | sh ``` -2. **Logged in** to a Memory Engine instance with an active engine: +2. **Logged in** to a Memory Engine instance, with an active space selected: ```bash me login - me whoami # confirms identity + active engine + me space use # select the space to capture into + me whoami # confirms identity + active space ``` -3. **An API key** for the plugin. When you `me login`, an admin key for your identity is issued automatically and stored in your local credentials — you can look it up with: + That login session is all the plugin needs — `api_key` is **optional** (see below). - ```bash - # inspect your credentials file (contains the key for the active engine) - cat ~/.config/me/credentials.yaml - ``` - - To paste that key into the plugin is the simplest path. - -### Restricting the plugin's privileges (optional but recommended) +### Using a dedicated agent key (optional) -The key you configure for the plugin does **not** have to be your admin key. You can issue a separate, scoped-down key and paste *that* into the plugin, so the agent only has access to the tree paths you want it to touch: +By default the plugin uses your `me login` session, so captures are attributed to **you**. To attribute captures to a separate, scoped-down **agent** identity instead — so it only touches the tree paths you allow — mint an api key and paste *that* into the plugin's `api_key`: ```bash -# 1. Create a dedicated engine user for the agent -me user create claude-code-agent +# 1. Create a dedicated agent +me agent create claude-code-agent -# 2. Grant it just the access it needs — in this example, read+create on +# 2. Add it to the space and grant just the access it needs — e.g. read+write on # the capture subtree (grants cover all descendant paths via ltree) -me grant create claude-code-agent claude_code.sessions read create +me agent add claude-code-agent +me access grant claude-code-agent share.projects w -# 3. Issue an API key for that user +# 3. Mint an API key for that agent me apikey create claude-code-agent plugin-key # → prints the raw key once; paste it into the plugin's api_key config ``` @@ -69,39 +67,36 @@ claude plugin install memory-engine@memory-engine --scope local # this repo, ## Configure -The plugin needs three values: `api_key`, `server`, and `tree_prefix`. Claude Code does not prompt for them at install time — you configure them from inside a session. +Every value is optional if you're logged in with an active space — the plugin falls back to your `me login` session and `me space use` space. Claude Code does not prompt at install time; configure from inside a session. ```text claude # start a session /plugin # open the plugin manager # → Installed → memory-engine → Configure -# → api_key (sensitive — stored in keychain) -# → server (default https://api.memory.build) -# → tree_prefix (default claude_code.sessions) +# → space (OPTIONAL — blank = your active space; pin for project/shared installs) +# → api_key (OPTIONAL, sensitive — blank = use your `me login` session) +# → server (default https://api.memory.build) +# → tree_root (default share.projects; captures nest at ..agent_sessions) +# → content_mode (default | full_transcript — see below) # → values take effect immediately; no restart required ``` -Sensitive values (the api_key) go to your system keychain. Non-sensitive values go to the `settings.json` for the scope you installed in. +Leave `api_key` blank to use your `me login` session (captures attributed to you); set it to use a dedicated agent key (see above). Leave `space` blank to capture into your active space; pin it for unattended or project-scope installs (a blank space with no active space set means captures are silently skipped). `content_mode` is `default` (user + assistant text — recommended) or `full_transcript` (also stores reasoning and tool calls/results as their own memories — more complete, but larger/noisier and may include sensitive tool output). Sensitive values (the api_key) go to your system keychain; non-sensitive values go to the `settings.json` for the scope you installed in. ## Verify -After configuring, send a prompt in Claude Code, then check that capture happened: +After a session (each turn's `Stop`, or `SessionEnd`), check that capture happened: ```bash -me memory search --tree "claude_code.*" --limit 5 +me memory search --tree "share.projects.*" --limit 5 ``` -You should see your recent prompts (and, after the agent finishes, its response). What gets stored: +Capture is the **same path as `me … import`** — one memory per message, with the +identical layout and metadata, so live and imported sessions interleave cleanly: -- **Tree**: whatever you set in `tree_prefix` (default: `claude_code.sessions`) -- **Metadata**: - - `type`: `user_prompt` or `agent_response` - - `session_id`: Claude Code's session UUID - - `project`: derived from `git remote get-url origin` in the session cwd (falls back to cwd basename) - - `cwd`: working directory when the hook fired - - `source`: `"claude-code"` - - `me_version`: the `me` CLI version that created the memory -- **Temporal**: ISO timestamp of hook invocation +- **Tree**: `..agent_sessions` (default root `share.projects`) — one node per project. +- **Metadata**: the importer's `source_*` schema — `type: agent_session`, `source_tool: "claude"`, `source_session_id`, `source_message_id`, `source_message_role` (`user`/`assistant`), `source_project_slug` (from the git `origin` remote, else cwd basename), `content_mode`, `importer_version`, and (when available) `source_cwd` / `source_git_repo` / `source_model` / … See the full table in [agent session imports](https://docs.memory.build/cli/agent-session-imports). +- **Temporal**: each memory's `start` is the **message** timestamp. ## Multi-scope / multi-engine @@ -139,11 +134,11 @@ Claude Code handles the cleanup. Your captured memories and API keys are preserv ## Troubleshooting -**`[memory-engine] CLAUDE_PLUGIN_OPTION_API_KEY not set` in stderr** -The hook ran but userConfig isn't filled in. Open `/plugin → memory-engine → Configure` and set the api_key. +**`[memory-engine] no credentials` in stderr** +The hook ran but found neither a `me login` session nor a configured api_key. Run `me login` (and `me space use `), or open `/plugin → memory-engine → Configure` and set the api_key + space. -**`Plugin option "X" isn't set` in Claude Code's error panel** -A required userConfig value is missing for either a hook or the MCP server. Configure all three: api_key, server, tree_prefix. +**Hook fires but no memories appear, no error** +With everything optional, a hook silently skips when it can't resolve a space — no `space` configured *and* no active space set (`me space use`). Either pin `space` in `/plugin → Configure` or run `me space use `. **Hook fires but no memories appear** - Confirm the api_key is valid: @@ -154,4 +149,4 @@ A required userConfig value is missing for either a hook or the MCP server. Conf - Confirm `me` is on PATH from inside the Claude session: ask Claude to run `which me`. **MCP server shows "failed" in `/plugin`** -Usually means api_key or server is missing from userConfig. Fix the configuration, then pick "Reconnect" from the plugin menu (or restart the session). +Usually means there are no credentials to resolve: you're not logged in (`me login`) and no api_key is set, or `me` isn't on PATH. Fix it, then pick "Reconnect" from the plugin menu (or restart the session). diff --git a/packages/claude-plugin/hooks/hooks.json b/packages/claude-plugin/hooks/hooks.json index bf2f955..196e65c 100644 --- a/packages/claude-plugin/hooks/hooks.json +++ b/packages/claude-plugin/hooks/hooks.json @@ -1,26 +1,26 @@ { "description": "Memory Engine capture hooks", "hooks": { - "UserPromptSubmit": [ + "Stop": [ { "hooks": [ { "type": "command", - "command": "me claude hook --event user-prompt-submit", + "command": "me claude hook --event stop", "async": true, - "timeout": 30 + "timeout": 60 } ] } ], - "Stop": [ + "SessionEnd": [ { "hooks": [ { "type": "command", - "command": "me claude hook --event stop", + "command": "me claude hook --event session-end", "async": true, - "timeout": 30 + "timeout": 60 } ] } diff --git a/packages/cli/chunk.test.ts b/packages/cli/chunk.test.ts index fcfba94..f916944 100644 --- a/packages/cli/chunk.test.ts +++ b/packages/cli/chunk.test.ts @@ -2,7 +2,7 @@ * Tests for the byte-aware chunker in `chunk.ts`. */ import { describe, expect, test } from "bun:test"; -import type { MemoryCreateParams } from "@memory.build/protocol/engine"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; import { approxMemoryBytes, type BatchCreateClient, @@ -111,9 +111,19 @@ describe("batchCreateChunked", () => { /** Minimal stub client; the test supplies the per-call behavior. */ const stubClient = ( - handler: (memories: MemoryCreateParams[]) => Promise<{ ids: string[] }>, + handler: ( + memories: MemoryCreateParams[], + replaceIfMetaDiffers?: string, + ) => Promise<{ ids: string[]; updatedIds?: string[] }>, ): BatchCreateClient => ({ - memory: { batchCreate: ({ memories }) => handler(memories) }, + memory: { + batchCreate: async ({ memories, replaceIfMetaDiffers }) => { + const res = await handler(memories, replaceIfMetaDiffers); + // Old servers omit updatedIds; the helper must tolerate that, so the + // stub passes whatever the handler chose to return. + return res as { ids: string[]; updatedIds: string[] }; + }, + }, }); test("single chunk, all succeed", async () => { @@ -183,12 +193,13 @@ describe("batchCreateChunked", () => { }); test("server returns shorter ids than requested (simulating ON CONFLICT)", async () => { - // Mimics post-#64 server behavior: caller submits 3 memories, server - // inserts 2 (one was a duplicate id, dropped by ON CONFLICT). The - // helper should faithfully report the 2 inserted; classifying the - // missing one as "skipped" is the caller's job. + // Caller submits 3 memories, server inserts 2 (one was a duplicate id, + // skipped by the conditional upsert). The helper should faithfully + // report the 2 inserted; classifying the missing one as "skipped" is + // the caller's job. const client = stubClient(async (memories) => ({ ids: memories.map((m) => m.id ?? "auto").filter((id) => id !== "dup"), // server "drops" the dup id + updatedIds: [], })); const result = await batchCreateChunked(client, [ mem("a"), @@ -196,18 +207,55 @@ describe("batchCreateChunked", () => { mem("b"), ]); expect(result.insertedIds).toEqual(["a", "b"]); + expect(result.updatedIds).toEqual([]); expect(result.failedIds).toEqual([]); // no chunk failed expect(result.errors).toEqual([]); }); + test("passes replaceIfMetaDiffers through and accumulates updatedIds", async () => { + // Two chunks (big payloads); the server reports the first id of each + // chunk as updated and the rest as inserted. + const seenKeys: Array = []; + const client = stubClient(async (memories, replaceIfMetaDiffers) => { + seenKeys.push(replaceIfMetaDiffers); + const ids = memories.map((m) => m.id ?? "auto"); + return { ids: ids.slice(1), updatedIds: ids.slice(0, 1) }; + }); + const result = await batchCreateChunked( + client, + [mem("a", 700_000), mem("b", 10), mem("c", 700_000), mem("d", 10)], + { replaceIfMetaDiffers: "importer_version" }, + ); + expect(seenKeys.length).toBeGreaterThan(1); // multiple chunks + expect(new Set(seenKeys)).toEqual(new Set(["importer_version"])); + expect(result.updatedIds.length).toBe(seenKeys.length); + expect([...result.insertedIds, ...result.updatedIds].sort()).toEqual([ + "a", + "b", + "c", + "d", + ]); + }); + + test("tolerates a pre-upsert server omitting updatedIds", async () => { + const client = stubClient(async (memories) => ({ + ids: memories.map((m) => m.id ?? "auto"), + // no updatedIds field at all + })); + const result = await batchCreateChunked(client, [mem("a")]); + expect(result.insertedIds).toEqual(["a"]); + expect(result.updatedIds).toEqual([]); + }); + test("empty input never calls the server", async () => { let calls = 0; const client = stubClient(async () => { calls++; - return { ids: [] }; + return { ids: [], updatedIds: [] }; }); const result = await batchCreateChunked(client, []); expect(result.insertedIds).toEqual([]); + expect(result.updatedIds).toEqual([]); expect(result.failedIds).toEqual([]); expect(result.errors).toEqual([]); expect(calls).toBe(0); diff --git a/packages/cli/chunk.ts b/packages/cli/chunk.ts index e05dc32..f65069c 100644 --- a/packages/cli/chunk.ts +++ b/packages/cli/chunk.ts @@ -14,7 +14,7 @@ * that callers should reach for unless they need a custom budget. */ -import type { MemoryCreateParams } from "@memory.build/protocol/engine"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; /** * Hard cap on memories per `memory.batchCreate` call. Matches the protocol @@ -108,21 +108,35 @@ export function* chunkMemoriesForBatchCreate( /** * Minimal client shape `batchCreateChunked` needs. Structurally typed so - * callers can pass an `EngineClient` or a stub in tests without coupling + * callers can pass a `MemoryClient` or a stub in tests without coupling * this module to the full client surface. */ export interface BatchCreateClient { memory: { batchCreate: (params: { memories: MemoryCreateParams[]; - }) => Promise<{ ids: string[] }>; + replaceIfMetaDiffers?: string; + }) => Promise<{ ids: string[]; updatedIds: string[] }>; }; } +/** Options applied to every chunk of a `batchCreateChunked` run. */ +export interface BatchCreateChunkedOptions { + /** + * Meta key for the server's conditional replace: a memory whose explicit + * id already exists is rewritten in place when the stored row's value for + * this key differs (importers pass "importer_version" so version bumps + * re-render existing rows), else skipped. Unset: duplicates are skipped. + */ + replaceIfMetaDiffers?: string; +} + /** Result of a chunked `batchCreate` run. */ export interface BatchCreateChunkedResult { /** Ids the server confirmed inserted (across all successful chunks). */ insertedIds: string[]; + /** Existing rows rewritten in place via `replaceIfMetaDiffers`. */ + updatedIds: string[]; /** * Explicit ids submitted in chunks that errored, flattened across all * failed chunks for callers that just need a set of "ids to exclude @@ -151,26 +165,36 @@ export interface BatchCreateChunkedResult { * * Chunks are sent sequentially. A failed chunk is recorded once in * `errors` and its explicit ids are added to `failedIds`; it does not - * abort siblings. Successful chunks contribute to `insertedIds`. + * abort siblings. Successful chunks contribute to `insertedIds` and + * `updatedIds`. * - * Note: the returned `insertedIds` may be shorter than the number of - * inputs in successful chunks because the server uses - * `ON CONFLICT (id) DO NOTHING`. Use `computeSkippedIds` (or, for packs, - * `classifySkips` with `failedIds`) to classify the missing ids. + * A submitted explicit id in neither array (and not in a failed chunk) was + * skipped server-side — it already exists, at a matching meta-key value + * when `replaceIfMetaDiffers` is set. Use `computeSkippedIds` (or, for + * packs, `classifySkips` with `failedIds`) to classify the missing ids. */ export async function batchCreateChunked( client: BatchCreateClient, memories: MemoryCreateParams[], + options: BatchCreateChunkedOptions = {}, ): Promise { const insertedIds: string[] = []; + const updatedIds: string[] = []; const failedIds: string[] = []; const errors: BatchCreateChunkedResult["errors"] = []; let chunkIndex = 0; for (const chunk of chunkMemoriesForBatchCreate(memories)) { try { - const { ids } = await client.memory.batchCreate({ memories: chunk }); - insertedIds.push(...ids); + const res = await client.memory.batchCreate({ + memories: chunk, + ...(options.replaceIfMetaDiffers !== undefined + ? { replaceIfMetaDiffers: options.replaceIfMetaDiffers } + : {}), + }); + insertedIds.push(...res.ids); + // A pre-upsert server doesn't return updatedIds; treat as none updated. + updatedIds.push(...(res.updatedIds ?? [])); } catch (error) { const msg = error instanceof Error ? error.message : String(error); const ids = chunk @@ -187,5 +211,5 @@ export async function batchCreateChunked( chunkIndex++; } - return { insertedIds, failedIds, errors }; + return { insertedIds, updatedIds, failedIds, errors }; } diff --git a/packages/cli/claude/capture.test.ts b/packages/cli/claude/capture.test.ts index f8558bb..50106e1 100644 --- a/packages/cli/claude/capture.test.ts +++ b/packages/cli/claude/capture.test.ts @@ -1,281 +1,102 @@ /** - * Unit tests for Claude Code hook capture logic. + * Unit tests for Claude Code hook config resolution. + * + * Capture itself is the import path (importTranscriptFile, tested in + * packages/cli/importers). This file only covers resolveHookConfigFromEnv — + * bearer/space/server/tree-root/content-mode resolution + session fallback. */ import { describe, expect, test } from "bun:test"; -import type { EngineClient } from "@memory.build/client"; -import { - buildMeta, - captureHookEvent, - deriveProject, - extractContent, - type HookConfig, - type HookEvent, - metaTypeForEvent, - resolveHookConfigFromEnv, -} from "./capture.ts"; +import { type HookConfig, resolveHookConfigFromEnv } from "./capture.ts"; -const BASE_EVENT = { - session_id: "sess-abc", - cwd: "/tmp/myproj", - hook_event_name: "UserPromptSubmit", - transcript_path: "/tmp/transcript.jsonl", -}; - -const CONFIG: HookConfig = { - server: "https://api.example.com", - apiKey: "me.eng123.aaa.bbb", - treePrefix: "claude_code.sessions", -}; - -// ============================================================================= -// extractContent -// ============================================================================= - -describe("extractContent", () => { - test("returns prompt for user-prompt-submit", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: "hello world" }; - expect(extractContent(event, "user-prompt-submit")).toBe("hello world"); - }); - - test("returns null for empty prompt", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: "" }; - expect(extractContent(event, "user-prompt-submit")).toBeNull(); - }); - - test("returns null for whitespace-only prompt", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: " \n\t " }; - expect(extractContent(event, "user-prompt-submit")).toBeNull(); - }); - - test("returns last_assistant_message for stop", () => { - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: "final response", - }; - expect(extractContent(event, "stop")).toBe("final response"); - }); - - test("returns null for null last_assistant_message", () => { - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: null, - }; - expect(extractContent(event, "stop")).toBeNull(); - }); - - test("returns null for missing last_assistant_message", () => { - const event: HookEvent = { ...BASE_EVENT }; - expect(extractContent(event, "stop")).toBeNull(); - }); - - test("preserves internal whitespace in content", () => { - const event: HookEvent = { - ...BASE_EVENT, - prompt: "line1\n\nline2\n", - }; - expect(extractContent(event, "user-prompt-submit")).toBe( - "line1\n\nline2\n", - ); - }); -}); - -// ============================================================================= -// metaTypeForEvent -// ============================================================================= - -describe("metaTypeForEvent", () => { - test("maps user-prompt-submit to user_prompt", () => { - expect(metaTypeForEvent("user-prompt-submit")).toBe("user_prompt"); - }); - - test("maps stop to agent_response", () => { - expect(metaTypeForEvent("stop")).toBe("agent_response"); - }); -}); - -// ============================================================================= -// deriveProject -// ============================================================================= - -describe("deriveProject", () => { - test("falls back to cwd basename when git is unavailable", () => { - // /tmp/__nonexistent-dir-for-test__ has no git remote - const project = deriveProject("/tmp/myproject"); - // Can't assert exact value — may hit a git repo if /tmp is one. - // But the result should be a lowercase, sanitized single label. - expect(project).toMatch(/^[a-z0-9_]+$/); - }); - - test("handles empty cwd with 'unknown' fallback", () => { - const project = deriveProject(""); - expect(project).toMatch(/^[a-z0-9_]+$/); - }); - - test("sanitizes special characters in basename", () => { - const project = deriveProject("/tmp/my-proj.foo"); - // Result should only contain letters/digits/underscores - expect(project).toMatch(/^[a-z0-9_]+$/); - }); -}); - -// ============================================================================= -// buildMeta -// ============================================================================= - -describe("buildMeta", () => { - test("builds metadata with required fields", () => { - const event: HookEvent = { ...BASE_EVENT, prompt: "hi" }; - const meta = buildMeta(event, "user-prompt-submit", "myproject"); - - expect(meta.type).toBe("user_prompt"); - expect(meta.session_id).toBe("sess-abc"); - expect(meta.cwd).toBe("/tmp/myproj"); - expect(meta.project).toBe("myproject"); - expect(meta.source).toBe("claude-code"); - expect(meta.me_version).toBeDefined(); - expect(typeof meta.me_version).toBe("string"); - }); - - test("uses agent_response type for stop event", () => { - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: "done", - }; - const meta = buildMeta(event, "stop", "proj"); - expect(meta.type).toBe("agent_response"); - }); -}); - -// ============================================================================= -// captureHookEvent -// ============================================================================= - -/** Build a mock EngineClient that records the last memory.create call. */ -function mockClient(): { - client: EngineClient; - calls: Array>; -} { - const calls: Array> = []; - const client = { - memory: { - create: async (params: Record) => { - calls.push(params); - return { id: "01960000-0000-7000-8000-000000000000" }; - }, - }, - } as unknown as EngineClient; - return { client, calls }; -} - -describe("captureHookEvent", () => { - test("skips empty content with no API call", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { ...BASE_EVENT, prompt: " " }; - - const result = await captureHookEvent(event, "user-prompt-submit", CONFIG, { - client, - }); - - expect(result.status).toBe("skipped"); - expect(calls).toHaveLength(0); - }); - - test("captures user prompt with correct tree + meta", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { ...BASE_EVENT, prompt: "hello" }; - const now = new Date("2026-04-23T10:00:00Z"); - - const result = await captureHookEvent(event, "user-prompt-submit", CONFIG, { - client, - now: () => now, - }); - - expect(result.status).toBe("captured"); - expect(result.memoryId).toBe("01960000-0000-7000-8000-000000000000"); - expect(calls).toHaveLength(1); - const [call] = calls as [Record]; - expect(call.content).toBe("hello"); - expect(call.tree).toBe("claude_code.sessions"); - expect(call.temporal).toEqual({ start: "2026-04-23T10:00:00.000Z" }); - const meta = call.meta as Record; - expect(meta.type).toBe("user_prompt"); - expect(meta.session_id).toBe("sess-abc"); - expect(meta.source).toBe("claude-code"); - }); - - test("captures stop event with agent_response type", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { - ...BASE_EVENT, - last_assistant_message: "goodbye", - }; - - const result = await captureHookEvent(event, "stop", CONFIG, { client }); - - expect(result.status).toBe("captured"); - expect(calls).toHaveLength(1); - const [call] = calls as [Record]; - expect(call.content).toBe("goodbye"); - const meta = call.meta as Record; - expect(meta.type).toBe("agent_response"); +describe("resolveHookConfigFromEnv", () => { + test("returns null when no api_key and no session fallback", () => { + expect(resolveHookConfigFromEnv({})).toBeNull(); }); - test("uses custom treePrefix from config", async () => { - const { client, calls } = mockClient(); - const event: HookEvent = { ...BASE_EVENT, prompt: "x" }; - const cfg: HookConfig = { - ...CONFIG, - treePrefix: "my.custom.prefix", - }; - - await captureHookEvent(event, "user-prompt-submit", cfg, { client }); - - const [call] = calls as [Record]; - expect(call.tree).toBe("my.custom.prefix"); + test("returns null when space is missing", () => { + expect( + resolveHookConfigFromEnv({ + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + }), + ).toBeNull(); }); -}); - -// ============================================================================= -// resolveHookConfigFromEnv -// ============================================================================= -describe("resolveHookConfigFromEnv", () => { - test("returns null when api_key is missing", () => { - const cfg = resolveHookConfigFromEnv({}); - expect(cfg).toBeNull(); - }); - - test("returns config when api_key is present", () => { + test("resolves full config from plugin env", () => { const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.eng.aaa.bbb", + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", CLAUDE_PLUGIN_OPTION_SERVER: "https://api.example.com", - CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "my.prefix", + CLAUDE_PLUGIN_OPTION_TREE_ROOT: "share.work", + CLAUDE_PLUGIN_OPTION_CONTENT_MODE: "full_transcript", }); expect(cfg).toEqual({ - apiKey: "me.eng.aaa.bbb", + token: "me.lookupid12345678.secret", + space: "eng123def456", server: "https://api.example.com", - treePrefix: "my.prefix", - }); + treeRoot: "share.work", + fullTranscript: true, + } satisfies HookConfig); }); - test("falls back to default server and tree_prefix", () => { + test("defaults: server, tree root, content mode (default = not full)", () => { const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.eng.aaa.bbb", + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", }); expect(cfg).toEqual({ - apiKey: "me.eng.aaa.bbb", + token: "me.lookupid12345678.secret", + space: "eng123def456", server: "https://api.memory.build", - treePrefix: "claude_code.sessions", - }); + treeRoot: "share.projects", + fullTranscript: false, + } satisfies HookConfig); }); - test("treats empty string as missing (falls back to default)", () => { + test("content_mode=default → fullTranscript false", () => { const cfg = resolveHookConfigFromEnv({ - CLAUDE_PLUGIN_OPTION_API_KEY: "me.eng.aaa.bbb", - CLAUDE_PLUGIN_OPTION_SERVER: "", - CLAUDE_PLUGIN_OPTION_TREE_PREFIX: "", + CLAUDE_PLUGIN_OPTION_API_KEY: "k", + CLAUDE_PLUGIN_OPTION_SPACE: "s", + CLAUDE_PLUGIN_OPTION_CONTENT_MODE: "default", }); - expect(cfg?.server).toBe("https://api.memory.build"); - expect(cfg?.treePrefix).toBe("claude_code.sessions"); + expect(cfg?.fullTranscript).toBe(false); + }); + + test("falls back to the login session when api_key is blank", () => { + const cfg = resolveHookConfigFromEnv( + { CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456" }, + { sessionToken: "sess-token", server: "https://api.example.com" }, + ); + expect(cfg?.token).toBe("sess-token"); + expect(cfg?.server).toBe("https://api.example.com"); + }); + + test("treats an unsubstituted ${...} placeholder as blank (uses session)", () => { + const cfg = resolveHookConfigFromEnv( + { + CLAUDE_PLUGIN_OPTION_API_KEY: "${user_config.api_key}", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", + }, + { sessionToken: "sess-token" }, + ); + expect(cfg?.token).toBe("sess-token"); + }); + + test("uses the active space fallback when plugin space is unset", () => { + const cfg = resolveHookConfigFromEnv( + {}, + { sessionToken: "sess-token", activeSpace: "act123def456" }, + ); + expect(cfg?.space).toBe("act123def456"); + }); + + test("plugin api_key takes precedence over the session", () => { + const cfg = resolveHookConfigFromEnv( + { + CLAUDE_PLUGIN_OPTION_API_KEY: "me.lookupid12345678.secret", + CLAUDE_PLUGIN_OPTION_SPACE: "eng123def456", + }, + { sessionToken: "sess-token" }, + ); + expect(cfg?.token).toBe("me.lookupid12345678.secret"); }); }); diff --git a/packages/cli/claude/capture.ts b/packages/cli/claude/capture.ts index bf6f179..28ca77d 100644 --- a/packages/cli/claude/capture.ts +++ b/packages/cli/claude/capture.ts @@ -1,225 +1,109 @@ /** - * Claude Code hook event parsing and memory capture. + * Claude Code capture hook — config resolution + event shape. * - * Pure functions for event parsing, project derivation, and metadata - * construction are testable in isolation. The `captureHookEvent` entry - * point handles memory creation via EngineClient. + * Capture itself is the import path: the hook reads the session transcript and + * runs it through `importTranscriptFile` (packages/cli/importers), so live + * captures and `me import claude` produce identical memories (tree, ids, `source_*` + * metadata). This module only resolves the runtime config (bearer + space + + * tree root + content mode) and types the slice of the hook event payload we + * read. The orchestration lives in `commands/claude.ts` (`me claude hook`). */ +import { + DEFAULT_SESSIONS_NODE_NAME, + DEFAULT_TREE_ROOT, +} from "../importers/index.ts"; -import { CLIENT_VERSION } from "../../../version"; -import { createClient, type EngineClient } from "../client.ts"; - -// ============================================================================= -// Hook config (derived at runtime from CLAUDE_PLUGIN_OPTION_* env vars) -// ============================================================================= - -export interface HookConfig { - /** Memory Engine server URL. */ - server: string; - /** API key (from the plugin's sensitive userConfig). */ - apiKey: string; - /** Tree path prefix for captured memories (ltree). */ - treePrefix: string; -} - -// ============================================================================= -// Event types -// ============================================================================= - -export type HookEventName = "user-prompt-submit" | "stop"; - -export const HOOK_EVENT_NAMES: HookEventName[] = ["user-prompt-submit", "stop"]; - -/** Fields common to all Claude Code hook events. */ -interface HookEventBase { - session_id: string; - transcript_path?: string; - cwd: string; - hook_event_name?: string; -} - -export interface UserPromptSubmitEvent extends HookEventBase { - prompt: string; -} - -export interface StopEvent extends HookEventBase { - last_assistant_message?: string | null; - stop_hook_active?: boolean; -} - -export type HookEvent = UserPromptSubmitEvent | StopEvent; +export const DEFAULT_SERVER = "https://api.memory.build"; -// ============================================================================= -// Pure helpers (testable) -// ============================================================================= +/** Per-project sessions leaf, shared with `me import claude`. */ +export const SESSIONS_NODE = DEFAULT_SESSIONS_NODE_NAME; /** - * Extract the memory content from a hook event. - * - * Returns null if the event has no content to capture. + * Hook events the plugin registers. Both drive a full transcript import (Stop + * per turn; SessionEnd as a final flush) — idempotent, so re-importing is a + * no-op for already-captured messages. */ -export function extractContent( - event: HookEvent, - eventName: HookEventName, -): string | null { - if (eventName === "user-prompt-submit") { - const prompt = (event as UserPromptSubmitEvent).prompt; - if (typeof prompt !== "string") return null; - const trimmed = prompt.trim(); - return trimmed.length > 0 ? prompt : null; - } - - if (eventName === "stop") { - const msg = (event as StopEvent).last_assistant_message; - if (typeof msg !== "string") return null; - const trimmed = msg.trim(); - return trimmed.length > 0 ? msg : null; - } - - return null; +export const HOOK_EVENT_NAMES = ["stop", "session-end"] as const; +export type HookEventName = (typeof HOOK_EVENT_NAMES)[number]; + +/** The slice of a Claude Code hook event payload the capture hook reads. */ +export interface HookEvent { + session_id?: string; + cwd?: string; + /** Path to the session transcript JSONL (present on Stop / SessionEnd). */ + transcript_path?: string; + hook_event_name?: string; } -/** Map an event name to the `type` metadata value. */ -export function metaTypeForEvent(eventName: HookEventName): string { - switch (eventName) { - case "user-prompt-submit": - return "user_prompt"; - case "stop": - return "agent_response"; - } +/** Resolved hook config: where + how to write captured memories. */ +export interface HookConfig { + /** Memory Engine server URL. */ + server: string; + /** + * Bearer for the memory endpoint: the plugin's api key (sensitive userConfig) + * when set, else the user's `me login` session token. + */ + token: string; + /** Active space slug (X-Me-Space). */ + space: string; + /** Tree root; captures nest as `..agent_sessions`. */ + treeRoot: string; + /** content_mode=full_transcript → also store reasoning + tool calls/results. */ + fullTranscript: boolean; } -/** - * Normalize a raw string into a single ltree label. - * Letters, digits, and underscores only; lowercased. - */ -function sanitizeLtreeLabel(raw: string): string { - const cleaned = raw.replace(/[^a-zA-Z0-9]/g, "_").toLowerCase(); - return cleaned.length > 0 ? cleaned : "unknown"; +/** Credentials the hook falls back to when the plugin's api_key is unset. */ +export interface HookFallbackCreds { + apiKey?: string; + sessionToken?: string; + activeSpace?: string; + server?: string; } /** - * Derive a project label from a cwd. - * - * Tries `git remote get-url origin` first; falls back to the basename of - * the cwd. The result is a single ltree label (sanitized). + * Treat unset / empty / unsubstituted-placeholder values as missing. Claude Code + * may substitute an empty string (or leave the literal `${user_config.x}`) for an + * optional userConfig field the user left blank. */ -export function deriveProject(cwd: string): string { - try { - const proc = Bun.spawnSync(["git", "remote", "get-url", "origin"], { - cwd, - stdout: "pipe", - stderr: "ignore", - }); - if (proc.exitCode === 0) { - const url = new TextDecoder().decode(proc.stdout).trim(); - // Extract the last path segment, stripping .git - // Matches https://github.com/org/repo.git and git@github.com:org/repo.git - const match = url.match(/[/:]([^/:]+?)(?:\.git)?$/); - if (match?.[1]) { - return sanitizeLtreeLabel(match[1]); - } - } - } catch { - // Fall through to cwd basename - } - - const parts = cwd.split("/").filter(Boolean); - const basename = parts[parts.length - 1] ?? "unknown"; - return sanitizeLtreeLabel(basename); -} - -/** Build the metadata object for a captured memory. */ -export function buildMeta( - event: HookEvent, - eventName: HookEventName, - project: string, -): Record { - return { - type: metaTypeForEvent(eventName), - session_id: event.session_id, - project, - cwd: event.cwd, - source: "claude-code", - me_version: CLIENT_VERSION, - }; +function blank(v: string | undefined): boolean { + return !v || /^\$\{.*\}$/.test(v); } -// ============================================================================= -// Config resolution from environment -// ============================================================================= - -export const DEFAULT_SERVER = "https://api.memory.build"; -export const DEFAULT_TREE_PREFIX = "claude_code.sessions"; - /** - * Resolve the hook config from `CLAUDE_PLUGIN_OPTION_*` env vars exported - * by Claude Code for the plugin. Returns null if required values are - * missing. - * - * Claude Code delivers `sensitive: true` userConfig values (like api_key) - * through the same env var mechanism as non-sensitive ones. + * Resolve the hook config. The bearer is the plugin's `api_key` + * (`CLAUDE_PLUGIN_OPTION_API_KEY`) when set; otherwise it falls back to the + * user's `me login` session (passed in via `creds`, so this stays pure/testable). + * The space comes from the plugin config, else the caller's active space. + * Returns null when no bearer or no space is available. */ export function resolveHookConfigFromEnv( env: NodeJS.ProcessEnv = process.env, + creds: HookFallbackCreds = {}, ): HookConfig | null { - const apiKey = env.CLAUDE_PLUGIN_OPTION_API_KEY; - if (!apiKey) return null; - - return { - apiKey, - server: env.CLAUDE_PLUGIN_OPTION_SERVER || DEFAULT_SERVER, - treePrefix: env.CLAUDE_PLUGIN_OPTION_TREE_PREFIX || DEFAULT_TREE_PREFIX, - }; -} - -// ============================================================================= -// Capture entry point -// ============================================================================= - -export interface CaptureResult { - status: "captured" | "skipped"; - reason?: string; - memoryId?: string; -} - -export interface CaptureOptions { - /** Override the client (for tests). */ - client?: EngineClient; - /** Override timestamp (for deterministic tests). */ - now?: () => Date; -} - -/** - * Capture a hook event as a memory. - * - * Returns immediately if there's no content to capture. Otherwise creates - * a memory in the engine under `config.treePrefix` with metadata. - */ -export async function captureHookEvent( - event: HookEvent, - eventName: HookEventName, - config: HookConfig, - opts: CaptureOptions = {}, -): Promise { - const content = extractContent(event, eventName); - if (content === null) { - return { status: "skipped", reason: "empty content" }; - } - - const project = deriveProject(event.cwd); - const meta = buildMeta(event, eventName, project); - const now = (opts.now ?? (() => new Date()))(); - - const client = - opts.client ?? createClient({ url: config.server, apiKey: config.apiKey }); - - const result = await client.memory.create({ - content, - tree: config.treePrefix, - meta, - temporal: { start: now.toISOString() }, - }); - - return { status: "captured", memoryId: result.id }; + const pluginKey = blank(env.CLAUDE_PLUGIN_OPTION_API_KEY) + ? undefined + : env.CLAUDE_PLUGIN_OPTION_API_KEY; + // Bearer precedence mirrors `me mcp`: plugin key > ME_API_KEY > login session. + const token = pluginKey ?? creds.apiKey ?? creds.sessionToken; + if (!token) return null; + + // Space: plugin config, else the active space. Required either way. + const space = blank(env.CLAUDE_PLUGIN_OPTION_SPACE) + ? creds.activeSpace + : env.CLAUDE_PLUGIN_OPTION_SPACE; + if (!space) return null; + + const server = blank(env.CLAUDE_PLUGIN_OPTION_SERVER) + ? (creds.server ?? DEFAULT_SERVER) + : (env.CLAUDE_PLUGIN_OPTION_SERVER as string); + + const treeRoot = blank(env.CLAUDE_PLUGIN_OPTION_TREE_ROOT) + ? DEFAULT_TREE_ROOT + : (env.CLAUDE_PLUGIN_OPTION_TREE_ROOT as string); + + const fullTranscript = + (env.CLAUDE_PLUGIN_OPTION_CONTENT_MODE ?? "").toLowerCase() === + "full_transcript"; + + return { server, token, space, treeRoot, fullTranscript }; } diff --git a/packages/cli/client.ts b/packages/cli/client.ts index 268297a..6113e65 100644 --- a/packages/cli/client.ts +++ b/packages/cli/client.ts @@ -7,60 +7,60 @@ * import everything from one place. */ import { - type AccountsClient, - type AccountsClientOptions, type AuthClient, type AuthClientOptions, - createAccountsClient as baseCreateAccountsClient, createAuthClient as baseCreateAuthClient, - createClient as baseCreateClient, - type ClientOptions, - type EngineClient, + createMemoryClient as baseCreateMemoryClient, + createUserClient as baseCreateUserClient, + type MemoryClient, + type MemoryClientOptions, + type UserClient, + type UserClientOptions, } from "@memory.build/client"; import { CLIENT_VERSION } from "../../version"; /** - * Engine client factory with `clientVersion: CLIENT_VERSION` injected. + * Auth client factory. + * + * The device-flow endpoints don't go through the JSON-RPC pipeline, so they + * don't currently observe `X-Client-Version`. Re-exported here for symmetry + * so command files have a single import point. */ -export function createClient(options: ClientOptions = {}): EngineClient { - return baseCreateClient({ clientVersion: CLIENT_VERSION, ...options }); +export function createAuthClient(options: AuthClientOptions = {}): AuthClient { + return baseCreateAuthClient(options); } /** - * Accounts client factory with `clientVersion: CLIENT_VERSION` injected. + * Memory client factory (space data-plane + management) with + * `clientVersion: CLIENT_VERSION` injected. Talks to /api/v1/memory/rpc with the + * active space carried as X-Me-Space. */ -export function createAccountsClient( - options: AccountsClientOptions = {}, -): AccountsClient { - return baseCreateAccountsClient({ - clientVersion: CLIENT_VERSION, - ...options, - }); +export function createMemoryClient( + options: MemoryClientOptions = {}, +): MemoryClient { + return baseCreateMemoryClient({ clientVersion: CLIENT_VERSION, ...options }); } /** - * Auth client factory. - * - * The device-flow endpoints don't go through the JSON-RPC pipeline, so they - * don't currently observe `X-Client-Version`. Re-exported here for symmetry - * so command files have a single import point. + * User client factory (agent lifecycle + space discovery + whoami) with + * `clientVersion: CLIENT_VERSION` injected. Talks to /api/v1/user/rpc. */ -export function createAuthClient(options: AuthClientOptions = {}): AuthClient { - return baseCreateAuthClient(options); +export function createUserClient(options: UserClientOptions = {}): UserClient { + return baseCreateUserClient({ clientVersion: CLIENT_VERSION, ...options }); } // Re-export types and helpers used across the CLI. Pass-through so command // files don't need to dual-import from "@memory.build/client". export { - type AccountsClient, - type AccountsClientOptions, type AuthClient, type AuthClientOptions, type CheckServerVersionOptions, - type ClientOptions, checkServerVersion, DeviceFlowError, - type EngineClient, isRpcError, + type MemoryClient, + type MemoryClientOptions, RpcError, + type UserClient, + type UserClientOptions, } from "@memory.build/client"; diff --git a/packages/cli/commands/access.ts b/packages/cli/commands/access.ts new file mode 100644 index 0000000..001cbe0 --- /dev/null +++ b/packages/cli/commands/access.ts @@ -0,0 +1,178 @@ +/** + * me access — tree-access grants in the active space. + * + * The core model uses three additive levels: r = read, w = write, o = owner. + * Grants are keyed by (principal, tree path); an owner grant at a path lets the + * holder manage access within that subtree. + * + * - me access grant : grant or update access + * - me access rm-grant : remove a grant + * - me access list [principal] [--path

]: list grants (optionally scoped) + * + * is a UUID, or a name (user = email, agent/group = display name). + */ +import * as clack from "@clack/prompts"; +import { + accessLevelName, + parseAccessLevel, +} from "@memory.build/protocol/space"; +import { Command } from "commander"; +import { resolveCredentials } from "../credentials.ts"; +import { getOutputFormat, output, table } from "../output.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, + resolveSpacePrincipalId, +} from "../util.ts"; + +function createAccessGrantCommand(): Command { + return new Command("grant") + .description("grant or update a principal's access at a tree path") + .argument( + "", + "principal id or name (user email / agent / group)", + ) + .argument("", "tree path (empty string for the space root)") + .argument("", "access level: r (read), w (write), o (owner)") + .action( + async (principal: string, path: string, level: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const access = parseAccessLevel(level); + if (!access) { + handleError( + new Error(`Invalid level '${level}'. Use r, w, or o.`), + fmt, + ); + } + + const memory = buildMemoryClient(creds); + try { + const principalId = await resolveSpacePrincipalId( + memory, + principal, + fmt, + ); + const result = await memory.grant.set({ + principalId, + treePath: path, + access, + }); + output( + { principalId, treePath: path, access, ...result }, + fmt, + () => { + clack.log.success( + `Granted ${accessLevelName(access)} on '${path}' to ${principal}`, + ); + }, + ); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }, + ); +} + +function createAccessRmGrantCommand(): Command { + return new Command("rm-grant") + .description("remove a principal's grant at a tree path") + .argument("", "principal id or name") + .argument("", "tree path") + .action(async (principal: string, path: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const principalId = await resolveSpacePrincipalId( + memory, + principal, + fmt, + ); + const result = await memory.grant.remove({ + principalId, + treePath: path, + }); + output({ principalId, treePath: path, ...result }, fmt, () => { + if (result.removed) { + clack.log.success(`Removed grant on '${path}' from ${principal}`); + } else { + clack.log.warn("Grant not found."); + } + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAccessListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list grants in the active space") + .argument("[principal]", "filter by principal id or name") + .option("--path ", "only grants at or below this tree path") + .action(async (principal: string | undefined, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const principalId = principal + ? await resolveSpacePrincipalId(memory, principal, fmt) + : undefined; + const { grants } = await memory.grant.list({ + principalId: principalId ?? null, + treePath: opts.path ?? null, + }); + + // Map principal ids → names for display (member-accessible lookup). + const names = new Map(); + const ids = [...new Set(grants.map((g) => g.principalId))]; + if (ids.length > 0) { + const { principals } = await memory.principal.lookup({ ids }); + for (const p of principals) names.set(p.id, p.name); + } + + output({ grants }, fmt, () => { + if (grants.length === 0) { + console.log(" No grants found."); + return; + } + table( + ["principal", "tree_path", "access"], + grants.map((g) => [ + names.get(g.principalId) ?? g.principalId, + g.treePath === "" ? "(root)" : g.treePath, + accessLevelName(g.access), + ]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +export function createAccessCommand(): Command { + const access = new Command("access").description( + "manage tree-access grants in the active space", + ); + access.addCommand(createAccessGrantCommand()); + access.addCommand(createAccessRmGrantCommand()); + access.addCommand(createAccessListCommand()); + return access; +} diff --git a/packages/cli/commands/agent.ts b/packages/cli/commands/agent.ts new file mode 100644 index 0000000..cc34984 --- /dev/null +++ b/packages/cli/commands/agent.ts @@ -0,0 +1,202 @@ +/** + * me agent — manage your agents (global service accounts). + * + * Agents are owned by you and live across spaces; their lifecycle is on the + * user endpoint. Bringing an agent into the active space and minting its + * (space-bound) api key are space operations — see `me agent add` and + * `me apikey create`. + * + * - me agent list: list your agents + * - me agent create : create an agent + * - me agent rename : rename an agent + * - me agent delete : delete an agent + * - me agent add : add the agent to the active space + * - me agent groups : list the agent's groups in the active space + * + * is an agent id or name (names are unique per user). + */ +import * as clack from "@clack/prompts"; +import { Command } from "commander"; +import { resolveCredentials } from "../credentials.ts"; +import { getOutputFormat, output, table } from "../output.ts"; +import { + buildMemoryClient, + buildUserClient, + handleError, + requireSession, + requireSpace, + resolveAgentId, +} from "../util.ts"; + +function createAgentListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list your agents") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const { agents } = await user.agent.list(); + output({ agents }, fmt, () => { + if (agents.length === 0) { + console.log(" No agents. Run 'me agent create '."); + return; + } + table( + ["name", "id"], + agents.map((a) => [a.name, a.id]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentCreateCommand(): Command { + return new Command("create") + .description("create an agent") + .argument("", "agent name (unique per user)") + .action(async (name: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const { id } = await user.agent.create({ name }); + output({ id, name }, fmt, () => { + clack.log.success(`Created agent '${name}' (${id})`); + clack.log.info( + "Add it to a space with 'me agent add', then mint a key with 'me apikey create'.", + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentRenameCommand(): Command { + return new Command("rename") + .description("rename an agent") + .argument("", "agent id or name") + .argument("", "new name") + .action(async (agent: string, newName: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + const result = await user.agent.rename({ id, name: newName }); + output({ id, name: newName, ...result }, fmt, () => { + clack.log.success(`Renamed agent → '${newName}'`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentDeleteCommand(): Command { + return new Command("delete") + .alias("rm") + .description("delete an agent") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + const result = await user.agent.delete({ id }); + output({ id, ...result }, fmt, () => { + if (result.deleted) clack.log.success(`Deleted agent ${agent}`); + else clack.log.warn("Agent not found."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentAddCommand(): Command { + return new Command("add") + .description("add one of your agents to the active space") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + // Bringing your own agent into a space is self-service (no admin flag). + const result = await memory.principal.add({ principalId: id }); + output({ agentId: id, ...result }, fmt, () => { + clack.log.success(`Added agent ${agent} to the space.`); + clack.log.info("Mint a key with 'me apikey create'."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createAgentGroupsCommand(): Command { + return new Command("groups") + .description("list an agent's groups in the active space") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); + try { + const id = await resolveAgentId(user, agent, fmt); + const { groups } = await memory.group.listForMember({ memberId: id }); + output({ groups }, fmt, () => { + if (groups.length === 0) { + console.log(" Not in any groups."); + return; + } + table( + ["group", "admin", "id"], + groups.map((g) => [g.name, g.admin ? "yes" : "", g.groupId]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +export function createAgentCommand(): Command { + const agent = new Command("agent").description("manage your agents"); + agent.addCommand(createAgentListCommand()); + agent.addCommand(createAgentCreateCommand()); + agent.addCommand(createAgentRenameCommand()); + agent.addCommand(createAgentDeleteCommand()); + agent.addCommand(createAgentAddCommand()); + agent.addCommand(createAgentGroupsCommand()); + return agent; +} diff --git a/packages/cli/commands/apikey.ts b/packages/cli/commands/apikey.ts index eb83b18..f249508 100644 --- a/packages/cli/commands/apikey.ts +++ b/packages/cli/commands/apikey.ts @@ -1,187 +1,144 @@ /** - * me apikey — API key management commands. + * me apikey — manage your agents' API keys. * - * - me apikey list : List API keys for a user - * - me apikey create [name]: Create a new API key - * - me apikey show: Show the API key stored in credentials.yaml - * - me apikey revoke : Revoke an API key - * - me apikey delete : Permanently delete an API key + * Keys are agent-only (humans authenticate with a session) and global: a key + * works in any space the agent has been admitted to. The plaintext key is shown + * exactly once, by `create`. There is no revoke state — delete is the removal. + * + * - me apikey create [name] [--expires ]: mint a key (shown once) + * - me apikey list : list an agent's keys + * - me apikey get : key metadata + * - me apikey delete : delete (revoke) a key + * + * is an agent id or name; is an api-key id. */ import * as clack from "@clack/prompts"; import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { - getServerCredentials, - resolveCredentials, - resolveServer, -} from "../credentials.ts"; +import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; import { + buildUserClient, handleError, - requireEngine, requireSession, - resolveUserId, + resolveAgentId, } from "../util.ts"; -function createApiKeyListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list API keys for a user") - .argument("", "user name or ID") - .action(async (user: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const { apiKeys } = await engine.apiKey.list({ userId }); - - output({ apiKeys }, fmt, () => { - if (apiKeys.length === 0) { - console.log(" No API keys found."); - return; - } - table( - ["id", "name", "status"], - apiKeys.map((k) => [ - k.id, - k.name, - k.revokedAt ? "revoked" : "active", - ]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - function createApiKeyCreateCommand(): Command { return new Command("create") - .description("create a new API key") - .argument("", "user name or ID") + .description("mint an API key for one of your agents") + .argument("", "agent id or name") .argument("[name]", "key name (auto-generated if omitted)") .option("--expires ", "expiration timestamp (ISO 8601)") - .action(async (user: string, name: string | undefined, opts, cmd) => { + .action(async (agent: string, name: string | undefined, opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); + const user = buildUserClient(creds); const keyName = name ?? `cli-${new Date().toISOString().slice(0, 10)}`; try { - const userId = await resolveUserId(engine, user); - const result = await engine.apiKey.create({ - userId, + const agentId = await resolveAgentId(user, agent, fmt); + const result = await user.apiKey.create({ + agentId, name: keyName, - expiresAt: opts.expires ?? undefined, + expiresAt: opts.expires ?? null, }); - output(result, fmt, () => { - clack.log.success(`Created API key '${result.apiKey.name}'`); - console.log(` ID: ${result.apiKey.id}`); + clack.log.success(`Created API key '${keyName}'`); + console.log(` ID: ${result.id}`); clack.note( - result.rawKey, - "API Key (save this — it won't be shown again)", + result.key, + "API key — save it now; it won't be shown again", + ); + clack.log.info( + "Give it to the agent via ME_API_KEY or its MCP config. It works in any space the agent is a member of.", ); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } -function createApiKeyShowCommand(): Command { - return new Command("show") - .description("show the API key stored in credentials.yaml for an engine") - .option("--engine ", "engine slug (defaults to the active engine)") - .action(async (opts, cmd) => { +function createApiKeyListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list an agent's API keys") + .argument("", "agent id or name") + .action(async (agent: string, _opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = buildUserClient(creds); try { - const server = resolveServer(globalOpts.server); - const stored = getServerCredentials(server); - const engine: string | undefined = opts.engine ?? stored.active_engine; - if (!engine) { - throw new Error( - "no active engine for this server. Run 'me engine use ' or pass --engine .", - ); - } - const apiKey = stored.engines?.[engine]?.api_key; - if (!apiKey) { - throw new Error( - `no API key stored for engine '${engine}' on ${server}.`, + const agentId = await resolveAgentId(user, agent, fmt); + const { apiKeys } = await user.apiKey.list({ memberId: agentId }); + output({ apiKeys }, fmt, () => { + if (apiKeys.length === 0) { + console.log(" No API keys."); + return; + } + table( + ["id", "name", "created", "expires"], + apiKeys.map((k) => [k.id, k.name, k.createdAt, k.expiresAt ?? ""]), ); - } - - output({ server, engine, apiKey }, fmt, () => { - console.log(` Server: ${server}`); - console.log(` Engine: ${engine}`); - console.log(` API key: ${apiKey}`); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } -function createApiKeyRevokeCommand(): Command { - return new Command("revoke") - .description("revoke an API key") - .argument("", "API key ID") +function createApiKeyGetCommand(): Command { + return new Command("get") + .description("show API key metadata") + .argument("", "API key id") .action(async (id: string, _opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const user = buildUserClient(creds); try { - const result = await engine.apiKey.revoke({ id }); - - output(result, fmt, () => { - if (result.revoked) { - clack.log.success("API key revoked."); - } else { + const { apiKey } = await user.apiKey.get({ id }); + output({ apiKey }, fmt, () => { + if (!apiKey) { clack.log.warn("API key not found."); + return; } + console.log(` ID: ${apiKey.id}`); + console.log(` Name: ${apiKey.name}`); + console.log(` Agent: ${apiKey.memberId}`); + console.log(` Created: ${apiKey.createdAt}`); + console.log(` Expires: ${apiKey.expiresAt ?? "(never)"}`); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } function createApiKeyDeleteCommand(): Command { return new Command("delete") - .alias("rm") - .description("permanently delete an API key") - .argument("", "API key ID") + .aliases(["rm", "revoke"]) + .description("delete (revoke) an API key") + .argument("", "API key id") .option("-y, --yes", "skip confirmation prompt") .action(async (id: string, opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - requireEngine(creds, fmt); if (fmt === "text" && !opts.yes) { const confirmed = await clack.confirm({ - message: `Permanently delete API key ${id}?`, + message: `Delete API key ${id}? This revokes it immediately.`, + initialValue: false, }); if (clack.isCancel(confirmed) || !confirmed) { clack.cancel("Cancelled."); @@ -189,30 +146,26 @@ function createApiKeyDeleteCommand(): Command { } } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - + const user = buildUserClient(creds); try { - const result = await engine.apiKey.delete({ id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success("API key deleted."); - } else { - clack.log.warn("API key not found."); - } + const result = await user.apiKey.delete({ id }); + output({ id, ...result }, fmt, () => { + if (result.deleted) clack.log.success("API key deleted."); + else clack.log.warn("API key not found."); }); } catch (error) { - handleError(error, fmt); + handleError(error, fmt, { sessionServer: creds.server }); } }); } export function createApiKeyCommand(): Command { - const apikey = new Command("apikey").description("manage API keys"); - apikey.addCommand(createApiKeyListCommand()); + const apikey = new Command("apikey").description( + "manage your agents' API keys", + ); apikey.addCommand(createApiKeyCreateCommand()); - apikey.addCommand(createApiKeyShowCommand()); - apikey.addCommand(createApiKeyRevokeCommand()); + apikey.addCommand(createApiKeyListCommand()); + apikey.addCommand(createApiKeyGetCommand()); apikey.addCommand(createApiKeyDeleteCommand()); return apikey; } diff --git a/packages/cli/commands/claude.test.ts b/packages/cli/commands/claude.test.ts new file mode 100644 index 0000000..e323168 --- /dev/null +++ b/packages/cli/commands/claude.test.ts @@ -0,0 +1,36 @@ +/** + * Tests for `me claude` helpers. + */ +import { describe, expect, test } from "bun:test"; +import { pluginListShowsInstalled } from "./claude.ts"; + +describe("pluginListShowsInstalled", () => { + test("finds the plugin by its id in `claude plugin list --json` output", () => { + const out = JSON.stringify([ + { id: "superpowers@superpowers-marketplace", enabled: true }, + { id: "memory-engine@memory-engine", version: "0.1.0", enabled: true }, + ]); + expect(pluginListShowsInstalled(out)).toBe(true); + }); + + test("a disabled install still counts as installed", () => { + const out = JSON.stringify([ + { id: "memory-engine@memory-engine", enabled: false }, + ]); + expect(pluginListShowsInstalled(out)).toBe(true); + }); + + test("other plugins do not match", () => { + const out = JSON.stringify([ + { id: "memory-engine-fork@somewhere", enabled: true }, + ]); + expect(pluginListShowsInstalled(out)).toBe(false); + }); + + test("empty list and unparseable output count as not installed", () => { + expect(pluginListShowsInstalled("[]")).toBe(false); + expect(pluginListShowsInstalled("")).toBe(false); + expect(pluginListShowsInstalled("Installed plugins:\n ❯ …")).toBe(false); + expect(pluginListShowsInstalled('{"not": "an array"}')).toBe(false); + }); +}); diff --git a/packages/cli/commands/claude.ts b/packages/cli/commands/claude.ts index 0bd41d0..e589927 100644 --- a/packages/cli/commands/claude.ts +++ b/packages/cli/commands/claude.ts @@ -1,36 +1,68 @@ /** * me claude — Claude Code integration commands. * - * Two integration paths: + * `me claude install` has two modes: * - * 1. Full plugin (hooks + slash commands + MCP) via Claude Code's native - * plugin marketplace: + * 1. Full plugin (default) — installs the Memory Engine plugin (hooks + + * slash commands + MCP) via Claude Code's native plugin marketplace, + * driving the same commands you'd otherwise run by hand: * * claude plugin marketplace add timescale/memory-engine - * claude plugin install memory-engine@memory-engine [--scope user|project|local] - * # then, in a Claude Code session: - * /plugin # select memory-engine, Configure, fill api_key/server/tree_prefix + * claude plugin install memory-engine@memory-engine \ + * --config server=… [--config space=…] [--config api_key=…] * * Claude Code delivers the configured values to our hook (`me claude - * hook --event `) via CLAUDE_PLUGIN_OPTION_* env vars. + * hook --event `) via CLAUDE_PLUGIN_OPTION_* env vars. api_key is + * optional: left blank, the hook (and the plugin's MCP server) use your + * `me login` session. * - * 2. MCP-only via `me claude install`. Registers `me` as an MCP server - * with Claude Code (no hooks, no slash commands — just the tools). + * Pass --dev (run from inside the repo) to install the plugin from your + * local checkout — the repo's .claude-plugin/marketplace.json — instead of + * the published marketplace. The two share the marketplace name + * "memory-engine", so --dev re-points it at your working tree and + * reinstalls fresh (plugin files are copied into the cache, so a new build + * needs a reinstall). + * + * 2. MCP-only (`--mcp-only`) — registers `me` as an MCP server with Claude + * Code (no hooks, no slash commands — just the tools). */ +import { existsSync } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import * as clack from "@clack/prompts"; import { Command, InvalidArgumentError } from "commander"; import { - captureHookEvent, HOOK_EVENT_NAMES, type HookEvent, type HookEventName, resolveHookConfigFromEnv, + SESSIONS_NODE, } from "../claude/capture.ts"; +import { createMemoryClient } from "../client.ts"; +import { resolveCredentials } from "../credentials.ts"; import { claudeImporter } from "../importers/claude.ts"; +import { GIT_HISTORY_NODE_NAME } from "../importers/git.ts"; +import { + DEFAULT_SESSIONS_NODE_NAME, + DEFAULT_TREE_ROOT, + importTranscriptFile, +} from "../importers/index.ts"; +import { SlugRegistry } from "../importers/slug.ts"; import { type AgentInstallOptions, runAgentMcpInstall, } from "../mcp/agent-install.ts"; -import { buildAgentImportSubcommand } from "./import.ts"; +import { getOutputFormat } from "../output.ts"; +import { createClaudeImportCommand, runAgentImport } from "./import.ts"; +import { runGitImport } from "./import-git.ts"; +import { gitHookStatus, runGitHookInstall } from "./import-git-hook.ts"; + +/** GitHub source for `claude plugin marketplace add`. */ +const PLUGIN_MARKETPLACE_SOURCE = "timescale/memory-engine"; +/** The marketplace `name` (from .claude-plugin/marketplace.json). */ +const PLUGIN_MARKETPLACE_NAME = "memory-engine"; +/** `@` ref for `claude plugin install`. */ +const PLUGIN_REF = `memory-engine@${PLUGIN_MARKETPLACE_NAME}`; const CLAUDE_SCOPES = ["local", "user", "project"] as const; type ClaudeScope = (typeof CLAUDE_SCOPES)[number]; @@ -45,45 +77,288 @@ function parseClaudeScope(value: string): ClaudeScope { } /** - * me claude install — register me as an MCP server with Claude Code. + * me claude install — install the Memory Engine plugin for Claude Code. * - * MCP-only: leaves the full Claude Code plugin install flow alone. Use this - * if you want the `me` MCP tools available in Claude Code but don't want the - * plugin's hooks or slash commands. + * Default: the full plugin (hooks + slash commands + MCP), installed via + * Claude Code's native plugin marketplace. `--mcp-only` falls back to + * registering just the `me` MCP server (no hooks, no slash commands). */ function createClaudeInstallCommand(): Command { return new Command("install") - .description("register me as an MCP server with Claude Code") - .option("--api-key ", "API key to embed in MCP config") - .option("--server ", "server URL to embed in MCP config") + .description( + "install the Memory Engine plugin for Claude Code (hooks + slash commands + MCP)", + ) + .option( + "--mcp-only", + "register only the me MCP server (no hooks or slash commands)", + ) + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) + .option("--server ", "server URL to embed in the config") + .option( + "--space ", + "pin a space (default: resolve ME_SPACE / active space at runtime)", + ) .option( "-s, --scope ", `Claude Code config scope (${CLAUDE_SCOPES.join(", ")})`, parseClaudeScope, "user", ) + .option( + "--dev", + "install the plugin from the local checkout instead of the published marketplace (run from inside the repo)", + ) .action( async ( - opts: AgentInstallOptions & { scope: ClaudeScope }, + opts: AgentInstallOptions & { + scope: ClaudeScope; + mcpOnly?: boolean; + dev?: boolean; + }, cmd: Command, ) => { const globalOpts = cmd.optsWithGlobals(); - await runAgentMcpInstall("claude", { + const server = globalOpts.server ?? opts.server; + if (opts.mcpOnly) { + if (opts.dev) { + clack.log.warn( + "--dev has no effect with --mcp-only: the MCP server already runs your local `me` binary on PATH.", + ); + } + await runAgentMcpInstall("claude", { + apiKey: opts.apiKey, + server, + space: opts.space, + scope: opts.scope, + }); + return; + } + await runClaudePluginInstall({ apiKey: opts.apiKey, - server: globalOpts.server ?? opts.server, + server, + space: opts.space, scope: opts.scope, + dev: opts.dev, }); }, ); } +/** Run a command, capturing its exit code, stdout, and stderr. */ +async function runCommand( + cmd: string[], +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const exitCode = await proc.exited; + return { exitCode, stdout, stderr }; +} + +/** + * Walk up from `startDir` to the repo's marketplace manifest + * (`.claude-plugin/marketplace.json`), returning the directory that contains it + * — the marketplace root passed to `claude plugin marketplace add`. Used by + * `--dev` to install the plugin from the local checkout. Returns undefined when + * not run from inside the repo. + */ +function findRepoMarketplaceRoot(startDir: string): string | undefined { + let dir = resolve(startDir); + for (;;) { + if (existsSync(join(dir, ".claude-plugin", "marketplace.json"))) return dir; + const parent = dirname(dir); + if (parent === dir) return undefined; // reached filesystem root + dir = parent; + } +} + +/** + * Install the full Memory Engine plugin for Claude Code. + * + * Drives Claude Code's plugin CLI: registers the marketplace (idempotent — a + * no-op if it's already configured) and installs the plugin, passing the + * resolved server/space/api_key through `--config` (the same path as the + * interactive `/plugin` configure flow). Credential handling mirrors the + * MCP-only path: an api key requires a pinned space; otherwise the plugin + * falls back to your `me login` session at runtime. + */ +async function runClaudePluginInstall( + opts: AgentInstallOptions & { scope: ClaudeScope; dev?: boolean }, +): Promise { + if (Bun.which("claude") === null) { + clack.log.error( + "Claude Code (claude) not found on PATH. Install it first.", + ); + process.exit(1); + } + + // Resolve credentials: flags > env (ME_API_KEY / ME_SERVER / ME_SPACE) > + // stored config. + const creds = resolveCredentials(opts.server); + const apiKey = opts.apiKey ?? creds.apiKey; + const server = opts.server ?? creds.server; + if (!server) { + clack.log.error("No server URL available. Pass --server or set ME_SERVER."); + process.exit(1); + } + const space = opts.space ?? creds.activeSpace; + + if (apiKey) { + // A global key isn't space-bound, so the space must be fixed. + if (!space) { + clack.log.error( + "No space for the API key. Pass --space, set ME_SPACE, or run 'me space use ' (keys are global, so the space must be fixed).", + ); + process.exit(1); + } + } else if (!creds.sessionToken) { + clack.log.error( + "Not logged in. Run 'me login' (the plugin will use your session), or pass --api-key / set ME_API_KEY for a headless agent.", + ); + process.exit(1); + } else if (!space) { + clack.log.warn( + "No active space set — captures are skipped until you run 'me space use ' (or set ME_SPACE). Re-run with --space to pin one.", + ); + } + + // Resolve the marketplace source: the published GitHub repo, or — with --dev + // — the local checkout, so captures exercise the plugin files from your + // working tree (.mcp.json, hooks, slash commands) rather than the published + // version. + let marketplaceSource = PLUGIN_MARKETPLACE_SOURCE; + if (opts.dev) { + const root = findRepoMarketplaceRoot(process.cwd()); + if (!root) { + clack.log.error( + "--dev must be run from inside the memory-engine repo (no .claude-plugin/marketplace.json found at or above the current directory).", + ); + process.exit(1); + } + marketplaceSource = root; + } + + const spin = clack.spinner(); + + // 1. Register the marketplace. + if (opts.dev) { + // The local and published marketplaces share the name "memory-engine", so + // they can't coexist and `marketplace add` won't re-point an existing name; + // plugin install also copies files into the cache, so a fresh build needs a + // reinstall. Tear both down first (ignoring "not found" — these may be a + // no-op on a clean machine), then re-add from the local checkout so the + // install below picks up your working tree. + spin.start( + "Pointing the Memory Engine marketplace at your local checkout...", + ); + await runCommand([ + "claude", + "plugin", + "uninstall", + "-y", + "-s", + opts.scope, + PLUGIN_REF, + ]); + await runCommand([ + "claude", + "plugin", + "marketplace", + "remove", + PLUGIN_MARKETPLACE_NAME, + ]); + const add = await runCommand([ + "claude", + "plugin", + "marketplace", + "add", + "--scope", + opts.scope, + marketplaceSource, + ]); + if (add.exitCode !== 0 && !/already/i.test(add.stderr + add.stdout)) { + spin.stop("Failed to add the local marketplace"); + clack.log.error( + `claude plugin marketplace add exited with ${add.exitCode}${add.stderr ? ` — ${add.stderr.trim()}` : ""}`, + ); + process.exit(1); + } + } else { + // Idempotent: skip if already there. + spin.start("Adding the Memory Engine marketplace..."); + const list = await runCommand(["claude", "plugin", "marketplace", "list"]); + const alreadyAdded = + list.exitCode === 0 && list.stdout.includes(marketplaceSource); + if (!alreadyAdded) { + const add = await runCommand([ + "claude", + "plugin", + "marketplace", + "add", + "--scope", + opts.scope, + marketplaceSource, + ]); + if (add.exitCode !== 0 && !/already/i.test(add.stderr + add.stdout)) { + spin.stop("Failed to add the marketplace"); + clack.log.error( + `claude plugin marketplace add exited with ${add.exitCode}${add.stderr ? ` — ${add.stderr.trim()}` : ""}`, + ); + process.exit(1); + } + } + } + + // 2. Install the plugin, baking the resolved config so captures land in the + // right space. Leave tree_root / content_mode at the plugin defaults + // (reconfigure them later via `/plugin` if needed). + spin.message("Installing the memory-engine plugin..."); + const install = ["claude", "plugin", "install", "--scope", opts.scope]; + install.push("--config", `server=${server}`); + if (space) install.push("--config", `space=${space}`); + if (apiKey) install.push("--config", `api_key=${apiKey}`); + install.push(PLUGIN_REF); + + const result = await runCommand(install); + if (result.exitCode !== 0) { + if (/already/i.test(result.stderr + result.stdout)) { + spin.stop("Memory Engine plugin already installed"); + clack.log.info( + "Run '/plugin' in Claude Code to reconfigure (or '--mcp-only' for the MCP server alone).", + ); + return; + } + spin.stop("Failed to install the plugin"); + clack.log.error( + `claude plugin install exited with ${result.exitCode}${result.stderr ? ` — ${result.stderr.trim()}` : ""}`, + ); + process.exit(1); + } + + spin.stop( + opts.dev + ? "Installed the Memory Engine plugin from your local checkout" + : "Installed the Memory Engine plugin for Claude Code", + ); + clack.log.info( + "Restart Claude Code (or run '/plugin') to load the hooks + slash commands.", + ); +} + /** - * me claude hook — invoked by the Claude Code plugin to capture events as - * memories. + * me claude hook — invoked by the Claude Code plugin on Stop / SessionEnd to + * capture the session. * - * Reads the event JSON from stdin, pulls credentials + config from the - * CLAUDE_PLUGIN_OPTION_* env vars that Claude Code exports for the plugin, - * and creates a memory. + * Reads the event JSON from stdin for the `transcript_path`, resolves config + * from the CLAUDE_PLUGIN_OPTION_* env vars (falling back to the `me login` + * session when no api_key is configured), and runs the transcript through + * `importTranscriptFile` — the same parse + write as `me import claude`, incremental so + * each call only writes messages new since the last. * * Best-effort: logs failures to stderr but always exits 0 so that a hook * failure never blocks a Claude Code session. @@ -104,45 +379,53 @@ function createClaudeHookCommand(): Command { process.exit(0); } - // Resolve config from env - const config = resolveHookConfigFromEnv(); + // Resolve config: the plugin's api_key if configured, else fall back to + // the user's `me login` session (resolved from the keychain/config). + const config = resolveHookConfigFromEnv( + process.env, + resolveCredentials(), + ); if (!config) { console.error( - "[memory-engine] CLAUDE_PLUGIN_OPTION_API_KEY not set. " + - "Configure the plugin via `/plugin` in Claude Code.", + "[memory-engine] no credentials. Run `me login`, or set the plugin's " + + "api_key + space via `/plugin` in Claude Code.", ); process.exit(0); } - // Read stdin - let input: string; + // Read + parse the event JSON from stdin for the transcript path. + let event: HookEvent; try { - input = await Bun.stdin.text(); + event = JSON.parse(await Bun.stdin.text()) as HookEvent; } catch (error) { console.error( - `[memory-engine] failed to read stdin: ${error instanceof Error ? error.message : String(error)}`, + `[memory-engine] failed to read/parse event JSON: ${error instanceof Error ? error.message : String(error)}`, ); process.exit(0); } - // Parse JSON - let event: HookEvent; - try { - event = JSON.parse(input) as HookEvent; - } catch (error) { + const transcriptPath = event.transcript_path; + if (!transcriptPath) { console.error( - `[memory-engine] failed to parse event JSON: ${error instanceof Error ? error.message : String(error)}`, + `[memory-engine] ${eventName}: no transcript_path in event payload`, ); process.exit(0); } - // Capture + // Import the transcript (incremental; same path as `me import claude`). try { - const result = await captureHookEvent(event, eventName, config); - if (result.status === "skipped") { - // Silent skip — no stderr output for empty content - process.exit(0); - } + const client = createMemoryClient({ + url: config.server, + token: config.token, + space: config.space, + }); + await importTranscriptFile(client, claudeImporter, transcriptPath, { + treeRoot: config.treeRoot, + sessionsNodeName: SESSIONS_NODE, + fullTranscript: config.fullTranscript, + dryRun: false, + verbose: false, + }); } catch (error) { console.error( `[memory-engine] ${eventName} capture failed: ${error instanceof Error ? error.message : String(error)}`, @@ -153,16 +436,362 @@ function createClaudeHookCommand(): Command { }); } +/** Markers delimiting the section `me claude init` manages in a CLAUDE.md. */ +const CLAUDE_MD_START = + ""; +const CLAUDE_MD_END = ""; + +/** Dim (secondary text) ANSI, for de-emphasizing hint copy. `\x1b[22m` resets + * only the dim attribute so surrounding clack styling is left intact. */ +const DIM = "\x1b[2m"; +const DIM_OFF = "\x1b[22m"; + +/** Green checkmark (resets only the foreground color) for already-done init + * steps, matching clack's green log symbols. */ +const CHECK = "\x1b[32m✓\x1b[39m"; + +/** + * Build the managed CLAUDE.md block that tells an agent where this project's + * memories live in Memory Engine and how to search them. `projectTree` is the + * canonical (dot-separated) ltree path (e.g. `share.projects.foo`); `space` is + * the active space slug, if known. + */ +function buildClaudeMdSection(projectTree: string, space?: string): string { + const sessions = `${projectTree}.${DEFAULT_SESSIONS_NODE_NAME}`; + const gitHistory = `${projectTree}.${GIT_HISTORY_NODE_NAME}`; + const where = space ? `Memory Engine (space \`${space}\`)` : "Memory Engine"; + return [ + CLAUDE_MD_START, + "## Project memories (Memory Engine)", + "", + `Prior context for this project — including captured/imported Claude Code`, + `sessions — is stored in ${where} under the tree:`, + "", + ` ${projectTree}`, + "", + `- Captured & imported agent sessions: \`${sessions}\``, + `- Imported git commit history: \`${gitHistory}\``, + `- Search them with the \`me_memory_search\` MCP tool (set \`tree\` to`, + ` \`${projectTree}\`), or from a shell: \`me search "" --tree ${projectTree}\`.`, + "", + "Always consult these memories when exploring the codebase or starting a", + "task: search them FIRST to recall earlier decisions and context before", + "digging into the code.", + CLAUDE_MD_END, + "", + ].join("\n"); +} + +/** + * Upsert the managed Memory Engine section into the project's CLAUDE.md. + * + * Idempotent: if the marker block already exists it is replaced in place; + * otherwise the block is appended (creating the file if absent). Writes to the + * git repo root's CLAUDE.md when in a repo, else the current directory's. + */ +async function writeProjectMemoryPointer(server?: string): Promise { + const cwd = process.cwd(); + const { slug, gitRoot } = await new SlugRegistry().resolve(cwd); + const projectTree = `${DEFAULT_TREE_ROOT}.${slug}`; + const space = resolveCredentials(server).activeSpace; + const section = buildClaudeMdSection(projectTree, space); + + const claudeMdPath = join(gitRoot ?? cwd, "CLAUDE.md"); + let existing = ""; + try { + existing = await readFile(claudeMdPath, "utf8"); + } catch { + existing = ""; // no file yet → create it + } + + let next: string; + const start = existing.indexOf(CLAUDE_MD_START); + if (start !== -1) { + // Replace the existing managed block in place. + const endMarker = existing.indexOf(CLAUDE_MD_END, start); + const end = + endMarker === -1 ? existing.length : endMarker + CLAUDE_MD_END.length; + // Swallow a single trailing newline after the old block so we don't grow + // blank lines on every re-run. + const tail = existing[end] === "\n" ? end + 1 : end; + next = existing.slice(0, start) + section + existing.slice(tail); + } else if (existing.trim().length === 0) { + next = section; + } else { + // Append after the existing content with one blank line of separation. + const sep = existing.endsWith("\n") ? "\n" : "\n\n"; + next = existing + sep + section; + } + + await writeFile(claudeMdPath, next); + clack.log.success(`Recorded project memory location in ${claudeMdPath}`); +} + +/** + * me claude init — one-shot setup of Claude Code memory integration. + * + * Setup is a list of independent steps (see INIT_STEPS). In an interactive + * terminal `init` presents a multiselect of all steps (each pre-checked) so the + * user can deselect any; non-interactively it runs every step except those + * turned off by a `--skip-` flag. To add a step, append one entry to + * INIT_STEPS — it picks up both a `--skip-*` flag and a multiselect row + * automatically. + */ +interface InitStepContext { + /** Global CLI opts (carries --server, output format) for the step to use. */ + globalOpts: Record; + /** Resolved server URL, if any. */ + server?: string; +} + +/** + * Availability of an init step in this environment: offer it, hide it + * entirely (not applicable here), or report it as already done — no + * multiselect row, but a ✓ line above the prompt so the user knows it's + * covered. + */ +type StepAvailability = "available" | "hidden" | "done"; + +interface InitStep { + /** Stable id — the multiselect value and the basis of the --skip flag. */ + id: string; + /** Commander-parsed key for this step's skip flag (e.g. skipClaudeMd). */ + optionKey: string; + /** The skip flag (e.g. "--skip-claude-md"). */ + skipFlag: string; + /** Help text for the skip flag. */ + skipDescription: string; + /** Multiselect row label. */ + label: string; + /** + * Optional availability gate: "hidden" omits the step entirely; "done" + * omits it but prints `doneLabel` with a checkmark. Absent means always + * available. + */ + available?: () => Promise; + /** The ✓ line printed when `available` resolves "done". */ + doneLabel?: string; + /** Perform the step. */ + run: (ctx: InitStepContext) => Promise; +} + +/** + * Parse `claude plugin list --json` output and report whether the Memory + * Engine plugin is installed. Exported for tests. Unparseable output counts + * as not-installed — the wrong guess costs an idempotent re-install offer, + * never a missed install. + */ +export function pluginListShowsInstalled(stdout: string): boolean { + try { + const plugins = JSON.parse(stdout); + if (!Array.isArray(plugins)) return false; + return plugins.some((p) => (p as { id?: unknown }).id === PLUGIN_REF); + } catch { + return false; + } +} + +/** + * Availability of the plugin-install init step: hidden when the `claude` + * binary is absent, "done" when the plugin is already installed. + */ +async function pluginInstallAvailable(): Promise { + if (Bun.which("claude") === null) return "hidden"; + const { exitCode, stdout } = await runCommand([ + "claude", + "plugin", + "list", + "--json", + ]); + if (exitCode !== 0) return "available"; // can't tell → offer the install + return pluginListShowsInstalled(stdout) ? "done" : "available"; +} + +const INIT_STEPS: InitStep[] = [ + { + id: "plugin-install", + optionKey: "skipPluginInstall", + skipFlag: "--skip-plugin-install", + skipDescription: "do not install the Claude Code plugin", + label: "Install the Claude Code plugin (hooks + slash commands + MCP)", + // Hidden when Claude Code is absent; ✓ when the plugin is already there. + available: pluginInstallAvailable, + doneLabel: "Claude Code plugin already installed", + run: ({ server }) => runClaudePluginInstall({ server, scope: "user" }), + }, + { + id: "transcript-import", + optionKey: "skipTranscriptImport", + skipFlag: "--skip-transcript-import", + skipDescription: "do not import this project's Claude Code sessions", + label: "Import this project's Claude Code sessions", + // Init is per-project setup, so scope the backfill to sessions recorded + // in this repo (cwd at or under the repo root) — `me import claude` + // remains the machine-wide sweep. The temp-cwd filter exists to keep + // throwaway sessions out of bulk sweeps; with the scope pinned to the + // project the user is standing in, it would only veto projects that + // happen to live under a temp dir, so include them. + run: async ({ globalOpts }) => { + const { gitRoot } = await new SlugRegistry().resolve(process.cwd()); + await runAgentImport( + claudeImporter, + { project: gitRoot ?? process.cwd(), includeTempCwd: true }, + globalOpts, + ); + }, + }, + { + id: "git-import", + optionKey: "skipGitImport", + skipFlag: "--skip-git-import", + skipDescription: "do not import the repo's git commit history", + label: "Import git commit history", + run: ({ globalOpts }) => runGitImport({ skipIfNotRepo: true }, globalOpts), + }, + { + id: "git-hook", + optionKey: "skipGitHook", + skipFlag: "--skip-git-hook", + skipDescription: "do not install the git post-commit capture hook", + label: "Install a git post-commit hook (keeps git history current)", + // Hidden outside a git repo or when a committed hooks manager owns the + // hook path; ✓ when the managed block is already installed. + available: async () => { + const status = await gitHookStatus(process.cwd()); + if (status === "installed") return "done"; + return status === "installable" ? "available" : "hidden"; + }, + doneLabel: "Git post-commit hook already installed", + run: ({ globalOpts }) => + runGitHookInstall({ skipIfNotRepo: true }, globalOpts), + }, + { + id: "claude-md", + optionKey: "skipClaudeMd", + skipFlag: "--skip-claude-md", + skipDescription: + "do not write the memory pointer into the project's CLAUDE.md", + label: "Add a memory pointer to CLAUDE.md", + run: ({ server }) => writeProjectMemoryPointer(server), + }, +]; + +function createClaudeInitCommand(): Command { + const cmd = new Command("init").description( + "set up Claude Code memory integration (interactive step picker; otherwise runs all steps)", + ); + // One --skip- flag per step, so non-interactive runs can opt out. + for (const step of INIT_STEPS) { + cmd.option(step.skipFlag, step.skipDescription); + } + cmd.action(async (opts: Record, cmdRef: Command) => { + const globalOpts = cmdRef.optsWithGlobals(); + const server = + typeof globalOpts.server === "string" ? globalOpts.server : undefined; + const fmt = getOutputFormat(globalOpts); + + // Interactive (a TTY with text output): present a multiselect pre-checked + // with the baseline so the user can deselect steps. Otherwise run the + // baseline as-is. + const interactive = + fmt === "text" && + Boolean(process.stdin.isTTY) && + Boolean(process.stdout.isTTY); + + // Steps available in this environment (e.g. plugin-install hides itself + // when Claude Code is absent). Already-done steps get a ✓ line instead + // of a row, so the user knows they're covered. The probe is skipped for + // steps already opted out non-interactively, so a `--skip-` run + // never pays for that step's availability check. + const candidates: InitStep[] = []; + const doneLabels: string[] = []; + for (const step of INIT_STEPS) { + if (!interactive && opts[step.optionKey] === true) continue; + const availability = step.available + ? await step.available() + : "available"; + if (availability === "hidden") continue; + if (availability === "done") { + doneLabels.push(step.doneLabel ?? step.label); + continue; + } + candidates.push(step); + } + if (fmt === "text") { + // One block: spacing 0 keeps consecutive ✓ lines adjacent (clack's + // default spacing of 1 would put a bare guide line between them). + doneLabels.forEach((label, i) => { + clack.log.message(label, { symbol: CHECK, spacing: i === 0 ? 1 : 0 }); + }); + } + + // Baseline = every available step not turned off via its --skip-* flag. + const baseline = candidates.filter((s) => opts[s.optionKey] !== true); + + let selectedIds: string[]; + if (interactive) { + const picked = await clack.multiselect({ + message: `Setup steps to run ${DIM}(all selected by default — ↑/↓ move, space to toggle off/on, enter to confirm)${DIM_OFF}`, + options: candidates.map((s) => ({ + value: s.id, + label: s.label, + })), + initialValues: baseline.map((s) => s.id), + required: false, + }); + if (clack.isCancel(picked)) { + clack.cancel("Cancelled."); + process.exit(0); + } + selectedIds = picked; + } else { + selectedIds = baseline.map((s) => s.id); + } + + const selected = candidates.filter((s) => selectedIds.includes(s.id)); + if (selected.length === 0) { + clack.log.info("No setup steps selected — nothing to do."); + return; + } + + const ctx: InitStepContext = { globalOpts, server }; + for (const step of selected) { + // Announce the step before its own output (progress spinners etc.) + // appears, so the counters have context. Structured output modes stay + // clean for parsing. + if (fmt === "text") clack.log.step(step.label); + await step.run(ctx); + } + if (fmt === "text") printInitOutro(); + }); + return cmd; +} + +/** + * Closing guidance after init: what having project memories wired up + * actually buys the user, and how to invoke them deliberately. + */ +function printInitOutro(): void { + clack.note( + [ + "Ask Claude about this project's history or architecture — it now", + "draws on your memories (past sessions, git history) automatically,", + "and consults them when exploring the code for new features.", + "", + "You can also point Claude at them explicitly, e.g.:", + `${DIM}"Search memory engine: why did we structure the database this way?"${DIM_OFF}`, + `${DIM}"Check me memories for past work on this area before we start"${DIM_OFF}`, + `${DIM}"What do my me memories say about how deploys work here?"${DIM_OFF}`, + ].join("\n"), + "Your project now has memory", + ); +} + export function createClaudeCommand(): Command { const claude = new Command("claude").description("Claude Code integration"); claude.addCommand(createClaudeInstallCommand()); + claude.addCommand(createClaudeInitCommand()); claude.addCommand(createClaudeHookCommand()); - claude.addCommand( - buildAgentImportSubcommand( - "import Claude Code sessions from ~/.claude/projects", - claudeImporter, - true, - ), - ); + claude.addCommand(createClaudeImportCommand()); return claude; } diff --git a/packages/cli/commands/codex.ts b/packages/cli/commands/codex.ts index 51986b1..c3b3329 100644 --- a/packages/cli/commands/codex.ts +++ b/packages/cli/commands/codex.ts @@ -4,23 +4,30 @@ * - me codex install: register me as an MCP server with Codex CLI */ import { Command } from "commander"; -import { codexImporter } from "../importers/codex.ts"; import { type AgentInstallOptions, runAgentMcpInstall, } from "../mcp/agent-install.ts"; -import { buildAgentImportSubcommand } from "./import.ts"; +import { createCodexImportCommand } from "./import.ts"; function createCodexInstallCommand(): Command { return new Command("install") .description("register me as an MCP server with Codex CLI") - .option("--api-key ", "API key to embed in MCP config") + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) .option("--server ", "server URL to embed in MCP config") + .option( + "--space ", + "pin a space (default: resolve ME_SPACE / active space at runtime)", + ) .action(async (opts: AgentInstallOptions, cmd: Command) => { const globalOpts = cmd.optsWithGlobals(); await runAgentMcpInstall("codex", { apiKey: opts.apiKey, server: globalOpts.server ?? opts.server, + space: opts.space, }); }); } @@ -28,11 +35,6 @@ function createCodexInstallCommand(): Command { export function createCodexCommand(): Command { const codex = new Command("codex").description("Codex CLI integration"); codex.addCommand(createCodexInstallCommand()); - codex.addCommand( - buildAgentImportSubcommand( - "import Codex sessions from ~/.codex/sessions and archived_sessions", - codexImporter, - ), - ); + codex.addCommand(createCodexImportCommand()); return codex; } diff --git a/packages/cli/commands/engine.ts b/packages/cli/commands/engine.ts deleted file mode 100644 index de26e61..0000000 --- a/packages/cli/commands/engine.ts +++ /dev/null @@ -1,470 +0,0 @@ -/** - * me engine — engine management commands. - * - * - me engine list: List engines across all your orgs - * - me engine use [id-or-name]: Select the active engine - * - me engine create : Create a new engine in an org - * - me engine rename : Rename an engine - * - me engine delete : Permanently delete an engine - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; -import { - getEngineApiKey, - resolveCredentials, - setActiveEngine, - storeApiKey, -} from "../credentials.ts"; -import { - getOutputFormat, - type OutputFormat, - output, - table, -} from "../output.ts"; -import { handleError, requireSession, resolveOrgId } from "../util.ts"; - -// UUIDv7 pattern for argument detection -const UUIDV7_RE = - /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -/** - * Flattened engine info with org context. - */ -interface EngineInfo { - id: string; - slug: string; - name: string; - status: string; - orgId: string; - orgName: string; -} - -/** - * Fetch all engines across all the user's orgs. - */ -async function fetchAllEngines( - accounts: ReturnType, -): Promise { - const { orgs } = await accounts.org.list(); - const engines: EngineInfo[] = []; - - for (const org of orgs) { - const { engines: orgEngines } = await accounts.engine.list({ - orgId: org.id, - }); - for (const engine of orgEngines) { - engines.push({ - id: engine.id, - slug: engine.slug, - name: engine.name, - status: engine.status, - orgId: org.id, - orgName: org.name, - }); - } - } - - return engines; -} - -/** - * me engine list — list engines across all orgs. - */ -function createEngineListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list engines across all your organizations") - .action(async (_opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const engines = await fetchAllEngines(accounts); - - const data = { - engines: engines.map((e) => ({ - id: e.id, - slug: e.slug, - name: e.name, - status: e.status, - orgName: e.orgName, - active: e.slug === creds.activeEngine, - })), - }; - - output(data, fmt, () => { - if (engines.length === 0) { - console.log(" No engines found."); - return; - } - table( - ["id", "name", "slug", "org", "status"], - engines.map((e) => [ - e.id, - e.name, - e.slug, - e.orgName, - e.slug === creds.activeEngine ? `${e.status} (active)` : e.status, - ]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * Resolve an engine argument (ID, name, or interactive picker). - */ -async function resolveEngine( - engines: EngineInfo[], - arg: string | undefined, - fmt: OutputFormat, -): Promise { - if (!arg) { - // No argument — interactive picker - if (fmt !== "text") { - output( - { error: "Engine ID or name is required in non-interactive mode" }, - fmt, - () => {}, - ); - process.exit(1); - } - - if (engines.length === 0) { - clack.log.error("No engines found."); - process.exit(1); - } - - const selected = await clack.select({ - message: "Select an engine", - options: engines.map((e) => ({ - value: e.id, - label: `${e.name} — ${e.orgName}`, - hint: e.slug, - })), - }); - - if (clack.isCancel(selected)) { - clack.cancel("Cancelled."); - process.exit(0); - } - - return engines.find((e) => e.id === (selected as string)) ?? null; - } - - // Argument provided — try to match - if (UUIDV7_RE.test(arg)) { - // Looks like a UUID — match by ID - const match = engines.find((e) => e.id === arg); - if (!match) { - const msg = `No engine found with ID: ${arg}`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - return match; - } - - // Match by name - const matches = engines.filter((e) => e.name === arg); - if (matches.length === 0) { - const msg = `No engine named '${arg}' found`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - if (matches.length > 1) { - const msg = `Multiple engines named '${arg}'. Use the engine ID instead:`; - if (fmt === "text") { - clack.log.error(msg); - for (const m of matches) { - console.log(` ${m.id} — ${m.orgName}`); - } - } else { - output( - { - error: msg, - matches: matches.map((m) => ({ - id: m.id, - orgName: m.orgName, - slug: m.slug, - })), - }, - fmt, - () => {}, - ); - } - process.exit(1); - } - - return matches[0] ?? null; -} - -/** - * me engine use — select the active engine. - */ -function createEngineUseCommand(): Command { - return new Command("use") - .description("select the active engine") - .argument("[id-or-name]", "engine ID or name") - .action(async (arg: string | undefined, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const engines = await fetchAllEngines(accounts); - const engine = await resolveEngine(engines, arg, fmt); - if (!engine) { - process.exit(1); - } - - // Check if we already have an API key for this engine - const existingKey = getEngineApiKey(creds.server, engine.slug); - if (existingKey) { - // Already have a key — just switch active engine - setActiveEngine(creds.server, engine.slug); - output( - { - engine: engine.name, - slug: engine.slug, - org: engine.orgName, - switched: true, - }, - fmt, - () => { - clack.log.success( - `Switched to engine '${engine.name}' (${engine.orgName})`, - ); - }, - ); - return; - } - - // No key — call setupAccess - const spin = fmt === "text" ? clack.spinner() : null; - spin?.start("Setting up engine access..."); - - const result = await accounts.engine.setupAccess({ - engineId: engine.id, - }); - - // Store the API key and set active engine - storeApiKey(creds.server, result.engineSlug, result.rawKey); - - spin?.stop("Engine access configured."); - - output( - { - engine: result.engineName, - slug: result.engineSlug, - org: result.orgName, - userId: result.userId, - setup: true, - }, - fmt, - () => { - clack.log.success( - `Connected to engine '${result.engineName}' (${result.orgName})`, - ); - }, - ); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * me engine create — create a new engine in an org. - */ -function createEngineCreateCommand(): Command { - return new Command("create") - .description("create a new engine in an organization") - .argument("", "engine name") - .option("--org ", "organization ID") - .option("--language ", "text search language", "english") - .action(async (name: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const engine = await accounts.engine.create({ - orgId, - name, - language: opts.language, - }); - - output(engine, fmt, () => { - clack.log.success(`Created engine '${engine.name}'`); - console.log(` ID: ${engine.id}`); - console.log(` Slug: ${engine.slug}`); - console.log(` Status: ${engine.status}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * me engine rename — rename an engine. - * - * Renaming changes only the human-readable name. The engine slug - * (which backs the underlying `me_` schema and any stored API - * keys) remains unchanged, so the active engine selection and - * API-key access are unaffected. - */ -function createEngineRenameCommand(): Command { - return new Command("rename") - .description("rename an engine") - .argument("", "engine ID or name") - .argument("", "new engine name") - .action(async (idOrName: string, newName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const engines = await fetchAllEngines(accounts); - const engine = await resolveEngine(engines, idOrName, fmt); - if (!engine) { - handleError(new Error(`Engine not found: ${idOrName}`), fmt); - } - - const oldName = engine.name; - const updated = await accounts.engine.update({ - id: engine.id, - name: newName, - }); - - output(updated, fmt, () => { - clack.log.success( - `Renamed engine '${oldName}' → '${updated.name}' (${engine.orgName})`, - ); - console.log(` ID: ${updated.id}`); - console.log(` Slug: ${updated.slug}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * me engine delete — permanently delete an engine and all its data. - */ -function createEngineDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("permanently delete an engine and all its data") - .argument("", "engine ID or name") - .option("--force", "skip confirmation prompt") - .action(async (idOrName: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - // Resolve engine by ID or name - const engines = await fetchAllEngines(accounts); - const engine = await resolveEngine(engines, idOrName, fmt); - if (!engine) { - handleError(new Error(`Engine not found: ${idOrName}`), fmt); - } - - // Confirmation: require typing the engine name - if (fmt === "text" && !opts.force) { - clack.log.warn( - "This will permanently delete the engine and ALL its data (memories, users, grants).", - ); - clack.log.warn("This action cannot be undone."); - console.log(); - - const confirmation = await clack.text({ - message: `Type the engine name "${engine.name}" to confirm deletion`, - validate: (value) => { - if (value !== engine.name) { - return `Please type "${engine.name}" exactly to confirm`; - } - }, - }); - - if (clack.isCancel(confirmation)) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const result = await accounts.engine.delete({ id: engine.id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success( - `Engine '${engine.name}' has been permanently deleted.`, - ); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -/** - * Create the engine command group. - */ -export function createEngineCommand(): Command { - const engine = new Command("engine").description("manage engines"); - - engine.addCommand(createEngineListCommand()); - engine.addCommand(createEngineUseCommand()); - engine.addCommand(createEngineCreateCommand()); - engine.addCommand(createEngineRenameCommand()); - engine.addCommand(createEngineDeleteCommand()); - - return engine; -} diff --git a/packages/cli/commands/gemini.ts b/packages/cli/commands/gemini.ts index 6a51d78..c76b302 100644 --- a/packages/cli/commands/gemini.ts +++ b/packages/cli/commands/gemini.ts @@ -24,8 +24,15 @@ function parseGeminiScope(value: string): GeminiScope { function createGeminiInstallCommand(): Command { return new Command("install") .description("register me as an MCP server with Gemini CLI") - .option("--api-key ", "API key to embed in MCP config") + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) .option("--server ", "server URL to embed in MCP config") + .option( + "--space ", + "pin a space (default: resolve ME_SPACE / active space at runtime)", + ) .option( "-s, --scope ", `Gemini CLI config scope (${GEMINI_SCOPES.join(", ")})`, @@ -41,6 +48,7 @@ function createGeminiInstallCommand(): Command { await runAgentMcpInstall("gemini", { apiKey: opts.apiKey, server: globalOpts.server ?? opts.server, + space: opts.space, scope: opts.scope, }); }, diff --git a/packages/cli/commands/grant.ts b/packages/cli/commands/grant.ts deleted file mode 100644 index 1380bee..0000000 --- a/packages/cli/commands/grant.ts +++ /dev/null @@ -1,183 +0,0 @@ -/** - * me grant — tree grant management commands. - * - * - me grant create : Grant tree access - * - me grant revoke : Revoke tree access - * - me grant list [user]: List grants - * - me grant check : Check access - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createGrantCreateCommand(): Command { - return new Command("create") - .description("grant tree access to a user") - .argument("", "user name or ID") - .argument("", "tree path") - .argument("", "actions: read, create, update, delete") - .option("--with-grant-option", "allow grantee to re-grant") - .action( - async (user: string, path: string, actions: string[], opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.grant.create({ - userId, - treePath: path, - actions: actions as ("read" | "create" | "update" | "delete")[], - withGrantOption: opts.withGrantOption ?? false, - }); - - output(result, fmt, () => { - clack.log.success( - `Granted [${actions.join(", ")}] on '${path}' to ${user}`, - ); - }); - } catch (error) { - handleError(error, fmt); - } - }, - ); -} - -function createGrantRevokeCommand(): Command { - return new Command("revoke") - .description("revoke tree access from a user") - .argument("", "user name or ID") - .argument("", "tree path") - .action(async (user: string, path: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.grant.revoke({ - userId, - treePath: path, - }); - - output(result, fmt, () => { - if (result.revoked) { - clack.log.success(`Revoked grant on '${path}' from ${user}`); - } else { - clack.log.warn("Grant not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createGrantListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list grants") - .argument("[user]", "filter by user name or ID (optional)") - .action(async (user: string | undefined, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = user ? await resolveUserId(engine, user) : undefined; - const { grants } = await engine.grant.list( - userId ? { userId } : undefined, - ); - - output({ grants }, fmt, () => { - if (grants.length === 0) { - console.log(" No grants found."); - return; - } - table( - ["user", "tree_path", "actions", "grant_option"], - grants.map((g) => [ - g.userName, - g.treePath, - g.actions.join(", "), - g.withGrantOption ? "yes" : "", - ]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createGrantCheckCommand(): Command { - return new Command("check") - .description("check if a user has access to a tree path") - .argument("", "user name or ID") - .argument("", "tree path") - .argument("", "action: read, create, update, delete") - .action(async (user: string, path: string, action: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.grant.check({ - userId, - treePath: path, - action: action as "read" | "create" | "update" | "delete", - }); - - output(result, fmt, () => { - if (result.allowed) { - clack.log.success(`${action} on '${path}': allowed`); - } else { - clack.log.warn(`${action} on '${path}': denied`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createGrantCommand(): Command { - const grant = new Command("grant").description("manage tree grants"); - grant.addCommand(createGrantCreateCommand()); - grant.addCommand(createGrantRevokeCommand()); - grant.addCommand(createGrantListCommand()); - grant.addCommand(createGrantCheckCommand()); - return grant; -} diff --git a/packages/cli/commands/group.ts b/packages/cli/commands/group.ts new file mode 100644 index 0000000..9ff7702 --- /dev/null +++ b/packages/cli/commands/group.ts @@ -0,0 +1,314 @@ +/** + * me group — manage groups in the active space. + * + * Groups bundle members (users / agents) so a single tree-access grant covers + * everyone in the group. Group membership also confers space membership. + * + * - me group list: list groups + * - me group mine: list the groups you are in + * - me group create : create a group + * - me group rename : rename a group + * - me group delete : delete a group + * - me group add [--admin]: add a member (user/agent) + * - me group remove : remove a member + * - me group members : list a group's members + * + * is a group id or name; is a user/agent id or name (a UUID is + * always accepted; name resolution requires space-manager authority). + */ +import * as clack from "@clack/prompts"; +import { Command } from "commander"; +import type { MemoryClient } from "../client.ts"; +import { resolveCredentials } from "../credentials.ts"; +import { + getOutputFormat, + type OutputFormat, + output, + table, +} from "../output.ts"; +import { + buildMemoryClient, + buildUserClient, + handleError, + requireSession, + requireSpace, + resolveSpacePrincipalId, +} from "../util.ts"; + +const UUIDV7_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +/** Resolve a group id from a UUID or name (via group.list). */ +async function resolveGroupId( + memory: MemoryClient, + input: string, + fmt: OutputFormat, +): Promise { + if (UUIDV7_RE.test(input)) return input; + const { groups } = await memory.group.list(); + const lower = input.toLowerCase(); + const matches = groups.filter((g) => g.name.toLowerCase() === lower); + if (matches.length === 1 && matches[0]) return matches[0].id; + const msg = + matches.length === 0 + ? `No group named '${input}' in this space.` + : `Multiple groups named '${input}'. Use the group id instead.`; + if (fmt === "text") { + clack.log.error(msg); + if (matches.length > 1) + for (const g of matches) console.log(` ${g.name} — ${g.id}`); + } else { + output({ error: msg, matches }, fmt, () => {}); + } + process.exit(1); +} + +function createGroupListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list groups in the active space") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const { groups } = await memory.group.list(); + output({ groups }, fmt, () => { + if (groups.length === 0) { + console.log(" No groups. Run 'me group create '."); + return; + } + table( + ["name", "id"], + groups.map((g) => [g.name, g.id]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupCreateCommand(): Command { + return new Command("create") + .description("create a group") + .argument("", "group name") + .action(async (name: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const { id } = await memory.group.create({ name }); + output({ id, name }, fmt, () => { + clack.log.success(`Created group '${name}' (${id})`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupRenameCommand(): Command { + return new Command("rename") + .description("rename a group") + .argument("", "group id or name") + .argument("", "new name") + .action(async (group: string, newName: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const id = await resolveGroupId(memory, group, fmt); + const result = await memory.group.rename({ id, name: newName }); + output({ id, name: newName, ...result }, fmt, () => { + clack.log.success(`Renamed group → '${newName}'`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupDeleteCommand(): Command { + return new Command("delete") + .alias("rm") + .description("delete a group") + .argument("", "group id or name") + .action(async (group: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const id = await resolveGroupId(memory, group, fmt); + const result = await memory.group.delete({ id }); + output({ id, ...result }, fmt, () => { + if (result.deleted) clack.log.success(`Deleted group ${group}`); + else clack.log.warn("Group not found."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupAddCommand(): Command { + return new Command("add") + .description("add a member (user/agent) to a group") + .argument("", "group id or name") + .argument("", "user/agent id or name") + .option("--admin", "make them a group admin (can manage group membership)") + .action(async (group: string, member: string, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const groupId = await resolveGroupId(memory, group, fmt); + const memberId = await resolveSpacePrincipalId(memory, member, fmt); + const result = await memory.group.addMember({ + groupId, + memberId, + admin: opts.admin === true, + }); + output({ groupId, memberId, ...result }, fmt, () => { + clack.log.success( + `Added ${member} to group ${group}${opts.admin ? " as an admin" : ""}.`, + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupRemoveCommand(): Command { + return new Command("remove") + .alias("rm-member") + .description("remove a member from a group") + .argument("", "group id or name") + .argument("", "user/agent id or name") + .action(async (group: string, member: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const groupId = await resolveGroupId(memory, group, fmt); + const memberId = await resolveSpacePrincipalId(memory, member, fmt); + const result = await memory.group.removeMember({ groupId, memberId }); + output({ groupId, memberId, ...result }, fmt, () => { + if (result.removed) + clack.log.success(`Removed ${member} from ${group}`); + else clack.log.warn("Member not in group."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupMembersCommand(): Command { + return new Command("members") + .description("list a group's members") + .argument("", "group id or name") + .action(async (group: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const groupId = await resolveGroupId(memory, group, fmt); + const { members } = await memory.group.listMembers({ groupId }); + output({ members }, fmt, () => { + if (members.length === 0) { + console.log(" No members."); + return; + } + table( + ["name", "kind", "admin", "id"], + members.map((m) => [ + m.name, + m.kind, + m.admin ? "yes" : "", + m.memberId, + ]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createGroupMineCommand(): Command { + return new Command("mine") + .description("list the groups you are in (in the active space)") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const user = buildUserClient(creds); + const memory = buildMemoryClient(creds); + try { + const me = await user.whoami(); + const { groups } = await memory.group.listForMember({ + memberId: me.id, + }); + output({ groups }, fmt, () => { + if (groups.length === 0) { + console.log(" You're not in any groups in this space."); + return; + } + table( + ["group", "admin", "id"], + groups.map((g) => [g.name, g.admin ? "yes" : "", g.groupId]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +export function createGroupCommand(): Command { + const group = new Command("group").description( + "manage groups in the active space", + ); + group.addCommand(createGroupListCommand()); + group.addCommand(createGroupMineCommand()); + group.addCommand(createGroupCreateCommand()); + group.addCommand(createGroupRenameCommand()); + group.addCommand(createGroupDeleteCommand()); + group.addCommand(createGroupAddCommand()); + group.addCommand(createGroupRemoveCommand()); + group.addCommand(createGroupMembersCommand()); + return group; +} diff --git a/packages/cli/commands/import-git-hook.test.ts b/packages/cli/commands/import-git-hook.test.ts new file mode 100644 index 0000000..32810a8 --- /dev/null +++ b/packages/cli/commands/import-git-hook.test.ts @@ -0,0 +1,83 @@ +/** + * Tests for the managed post-commit hook block helpers. + */ +import { describe, expect, test } from "bun:test"; +import { + buildHookBlock, + removeHookBlock, + upsertHookScript, +} from "./import-git-hook.ts"; + +const BLOCK = buildHookBlock('"/usr/local/bin/me"'); +const START = "# >>> memory-engine"; + +describe("buildHookBlock", () => { + test("embeds the invocation, backgrounded and silenced", () => { + expect(BLOCK).toContain( + '("/usr/local/bin/me" import git >/dev/null 2>&1 &)', + ); + expect(BLOCK.startsWith(START)).toBe(true); + expect(BLOCK.endsWith("\n")).toBe(true); + }); + + test("supports a two-part source invocation", () => { + const block = buildHookBlock('"/opt/bun" "/repo/packages/cli/index.ts"'); + expect(block).toContain( + '("/opt/bun" "/repo/packages/cli/index.ts" import git >/dev/null 2>&1 &)', + ); + }); +}); + +describe("upsertHookScript", () => { + test("creates a fresh script with a shebang", () => { + const script = upsertHookScript(null, BLOCK); + expect(script.startsWith("#!/bin/sh\n")).toBe(true); + expect(script.split(START).length - 1).toBe(1); + }); + + test("treats an empty file as fresh", () => { + expect(upsertHookScript(" \n", BLOCK).startsWith("#!/bin/sh\n")).toBe( + true, + ); + }); + + test("appends once to a foreign hook, preserving it", () => { + const foreign = '#!/bin/sh\necho "their hook"\n'; + const script = upsertHookScript(foreign, BLOCK); + expect(script).toContain('echo "their hook"'); + expect(script.indexOf(START)).toBeGreaterThan(script.indexOf("their hook")); + expect(script.split(START).length - 1).toBe(1); + }); + + test("re-install replaces the managed block in place without growth", () => { + const v1 = upsertHookScript("#!/bin/sh\necho before\n", BLOCK); + const newBlock = buildHookBlock('"/new/path/me"'); + const v2 = upsertHookScript(v1, newBlock); + expect(v2.split(START).length - 1).toBe(1); + expect(v2).toContain('"/new/path/me"'); + expect(v2).not.toContain("/usr/local/bin/me"); + expect(v2).toContain("echo before"); + // Idempotent: applying the same block again changes nothing. + expect(upsertHookScript(v2, newBlock)).toBe(v2); + }); +}); + +describe("removeHookBlock", () => { + test("returns null when only the shebang would remain", () => { + const script = upsertHookScript(null, BLOCK); + expect(removeHookBlock(script)).toBeNull(); + }); + + test("preserves foreign content", () => { + const foreign = '#!/bin/sh\necho "their hook"\n'; + const script = upsertHookScript(foreign, BLOCK); + const remaining = removeHookBlock(script); + expect(remaining).toContain('echo "their hook"'); + expect(remaining).not.toContain(START); + }); + + test("is a no-op on a script without the block", () => { + const foreign = '#!/bin/sh\necho "their hook"\n'; + expect(removeHookBlock(foreign)).toBe(foreign); + }); +}); diff --git a/packages/cli/commands/import-git-hook.ts b/packages/cli/commands/import-git-hook.ts new file mode 100644 index 0000000..48cbf92 --- /dev/null +++ b/packages/cli/commands/import-git-hook.ts @@ -0,0 +1,268 @@ +/** + * `me import git-hook` — install a managed git post-commit hook that keeps a + * repo's git-history memories current. + * + * The hook runs `me import git` in the background after every commit: + * best-effort, asynchronous, silent — it never blocks or fails a commit, and + * the embedded invocation is absolute so GUI git clients (no shell PATH) + * work. Because the import is high-water incremental, ANY fire catches up the + * entire backlog (including commits that arrived via pull/rebase), so a + * single post-commit hook suffices — no post-merge/post-rewrite matrix. + * + * The hook lives in the repo's effective hooks directory as a + * marker-delimited managed block (created, replaced in place, or appended to + * a foreign hook — the same upsert discipline as the CLAUDE.md pointer). + * Repos using a committed hooks manager (`core.hooksPath`, e.g. husky) are + * refused with instructions instead of writing into committed files. + */ +import { execFile } from "node:child_process"; +import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { basename, isAbsolute, join } from "node:path"; +import { promisify } from "node:util"; +import * as clack from "@clack/prompts"; +import { Command } from "commander"; +import { getOutputFormat, output } from "../output.ts"; +import { handleError } from "../util.ts"; + +const execFileAsync = promisify(execFile); + +/** Markers delimiting the managed block inside the hook script. */ +const HOOK_START = "# >>> memory-engine (managed by `me import git-hook`) >>>"; +const HOOK_END = "# <<< memory-engine <<<"; + +const SHEBANG = "#!/bin/sh"; + +/** Quote a path for /bin/sh (double quotes; escapes embedded `"` and `\`). */ +function shQuote(path: string): string { + return `"${path.replace(/([\\"$`])/g, "\\$1")}"`; +} + +/** + * The absolute invocation embedded into the hook, resolved from how this + * process is running: the compiled `me` binary, a source run (`bun + * packages/cli/index.ts` — dev and tests), or `me` on PATH. + */ +export function resolveMeInvocation(): string { + if (basename(process.execPath) === "me") return shQuote(process.execPath); + const entry = process.argv[1]; + if (entry && /\.(ts|js)$/.test(entry) && isAbsolute(entry)) { + return `${shQuote(process.execPath)} ${shQuote(entry)}`; + } + const onPath = Bun.which("me"); + if (onPath) return shQuote(onPath); + throw new Error( + "Cannot resolve the `me` binary to embed in the hook — install it on PATH first.", + ); +} + +/** The managed block (ends with a newline). */ +export function buildHookBlock(invocation: string): string { + return [ + HOOK_START, + "# Best-effort and asynchronous: never blocks or fails the commit.", + `(${invocation} import git >/dev/null 2>&1 &)`, + HOOK_END, + "", + ].join("\n"); +} + +/** + * Upsert the managed block into an existing hook script (null = no file). + * Fresh file → shebang + block; markers present → replaced in place; + * foreign hook → block appended (a foreign script that exits early never + * reaches it — documented limitation). + */ +export function upsertHookScript( + existing: string | null, + block: string, +): string { + if (existing === null || existing.trim().length === 0) { + return `${SHEBANG}\n${block}`; + } + const start = existing.indexOf(HOOK_START); + if (start !== -1) { + const endMarker = existing.indexOf(HOOK_END, start); + const end = + endMarker === -1 ? existing.length : endMarker + HOOK_END.length; + // Swallow a single trailing newline so re-installs don't grow the file. + const tail = existing[end] === "\n" ? end + 1 : end; + return existing.slice(0, start) + block + existing.slice(tail); + } + const sep = existing.endsWith("\n") ? "\n" : "\n\n"; + return existing + sep + block; +} + +/** + * Remove the managed block. Returns the remaining script, or null when + * nothing but the shebang would remain (caller deletes the file). + */ +export function removeHookBlock(existing: string): string | null { + const start = existing.indexOf(HOOK_START); + if (start === -1) return existing; + const endMarker = existing.indexOf(HOOK_END, start); + const end = endMarker === -1 ? existing.length : endMarker + HOOK_END.length; + const tail = existing[end] === "\n" ? end + 1 : end; + const remaining = existing.slice(0, start) + existing.slice(tail); + const meaningful = remaining + .split("\n") + .filter((l) => l.trim().length > 0 && l.trim() !== SHEBANG); + return meaningful.length === 0 ? null : remaining; +} + +async function git(repo: string, args: string[]): Promise { + try { + const { stdout } = await execFileAsync("git", ["-C", repo, ...args], { + timeout: 5000, + encoding: "utf8", + }); + return stdout.trim(); + } catch { + return null; + } +} + +/** Status of the git hook for `cwd`, driving the `me claude init` step. */ +export type GitHookStatus = + | "installable" // in a repo, no hooks manager, block not yet present + | "not-applicable" // not a git repo, or core.hooksPath owns the hook path + | "installed"; // the managed block is already there + +export async function gitHookStatus(cwd: string): Promise { + const root = await git(cwd, ["rev-parse", "--show-toplevel"]); + if (!root) return "not-applicable"; + if (await git(root, ["config", "core.hooksPath"])) return "not-applicable"; + const hooksFile = await resolveHooksFile(root); + try { + const existing = await readFile(hooksFile, "utf8"); + return existing.includes(HOOK_START) ? "installed" : "installable"; + } catch { + return "installable"; // no hook file yet + } +} + +/** The effective post-commit path (worktree-aware via --git-path). */ +async function resolveHooksFile(root: string): Promise { + const hooksDir = + (await git(root, ["rev-parse", "--git-path", "hooks"])) ?? ""; + const abs = isAbsolute(hooksDir) ? hooksDir : join(root, hooksDir); + return join(abs, "post-commit"); +} + +/** Options for one install/remove run. */ +export interface GitHookOptions { + repo?: string; + remove?: boolean; + /** Soft-skip when the target isn't a git repo (used by `me claude init`). */ + skipIfNotRepo?: boolean; +} + +/** + * Install (or remove) the managed post-commit hook. Exported so + * `me claude init` can run it as a setup step. Purely local — no server + * auth required. + */ +export async function runGitHookInstall( + opts: GitHookOptions, + globalOpts: Record, +): Promise { + const fmt = getOutputFormat(globalOpts); + const repoPath = opts.repo ?? process.cwd(); + + const root = await git(repoPath, ["rev-parse", "--show-toplevel"]); + if (!root) { + if (opts.skipIfNotRepo) { + if (fmt === "text") { + clack.log.info( + `${repoPath} is not a git repository — skipping git hook install`, + ); + } + return; + } + handleError(new Error(`${repoPath} is not a git repository`), fmt); + } + + const hooksFile = await resolveHooksFile(root); + + if (opts.remove) { + let existing: string; + try { + existing = await readFile(hooksFile, "utf8"); + } catch { + output({ hooksFile, action: "absent" }, fmt, () => + clack.log.info(`No hook installed at ${hooksFile}`), + ); + return; + } + const remaining = removeHookBlock(existing); + if (remaining === existing) { + output({ hooksFile, action: "absent" }, fmt, () => + clack.log.info(`No managed block found in ${hooksFile}`), + ); + return; + } + if (remaining === null) await rm(hooksFile); + else await writeFile(hooksFile, remaining); + output({ hooksFile, action: "removed" }, fmt, () => + clack.log.success(`Removed the memory-engine hook from ${hooksFile}`), + ); + return; + } + + // A committed hooks manager (husky, lefthook, …) owns the hook path — + // don't write into committed files; tell the user what to add instead. + const hooksPath = await git(root, ["config", "core.hooksPath"]); + if (hooksPath) { + handleError( + new Error( + `This repo routes hooks through core.hooksPath (${hooksPath}) — a committed hooks manager likely owns it.\n` + + `Add this line to its post-commit hook instead:\n` + + ` me import git >/dev/null 2>&1 &`, + ), + fmt, + ); + } + + let existing: string | null = null; + try { + existing = await readFile(hooksFile, "utf8"); + } catch { + // no hook yet + } + const updated = existing !== null && existing.includes(HOOK_START); + const next = upsertHookScript( + existing, + buildHookBlock(resolveMeInvocation()), + ); + await mkdir(join(hooksFile, ".."), { recursive: true }); + await writeFile(hooksFile, next); + await chmod(hooksFile, 0o755); + + output({ hooksFile, action: updated ? "updated" : "installed" }, fmt, () => { + clack.log.success( + `${updated ? "Updated" : "Installed"} the post-commit hook at ${hooksFile}`, + ); + console.log( + " Each commit now triggers a background `me import git` (incremental,", + ); + console.log( + " silent, never blocks the commit). Remove with: me import git-hook --remove", + ); + }); +} + +/** `me import git-hook` subcommand factory. */ +export function createGitHookCommand(): Command { + return new Command("git-hook") + .description( + "install a git post-commit hook that keeps git history memories current", + ) + .argument("[repo]", "path inside the repo (default: cwd)") + .option("--remove", "remove the managed hook block") + .action(async (repoArg: string | undefined, opts, cmdRef) => { + const globalOpts = cmdRef.optsWithGlobals(); + await runGitHookInstall( + { repo: repoArg, remove: opts.remove === true }, + globalOpts, + ); + }); +} diff --git a/packages/cli/commands/import-git.test.ts b/packages/cli/commands/import-git.test.ts new file mode 100644 index 0000000..894eb57 --- /dev/null +++ b/packages/cli/commands/import-git.test.ts @@ -0,0 +1,62 @@ +/** + * Tests for `me import git` option assembly. + */ +import { describe, expect, test } from "bun:test"; +import { buildGitImportOptions } from "./import-git.ts"; + +describe("buildGitImportOptions", () => { + test("applies defaults", () => { + const opts = buildGitImportOptions({}); + expect(opts).toEqual({ + repo: undefined, + branch: undefined, + since: undefined, + until: undefined, + maxCount: undefined, + full: false, + merges: true, + fileList: true, + treeRoot: "share.projects", + dryRun: false, + verbose: false, + skipIfNotRepo: false, + }); + }); + + test("maps flags through", () => { + const opts = buildGitImportOptions( + { + branch: "main", + since: "2 weeks ago", + until: "2026-01-01", + maxCount: 100, + full: true, + merges: false, + fileList: false, + treeRoot: "~/work", + dryRun: true, + verbose: true, + skipIfNotRepo: true, + }, + "/some/repo", + ); + expect(opts.repo).toBe("/some/repo"); + expect(opts.branch).toBe("main"); + expect(opts.since).toBe("2 weeks ago"); + expect(opts.until).toBe("2026-01-01"); + expect(opts.maxCount).toBe(100); + expect(opts.full).toBe(true); + expect(opts.merges).toBe(false); + expect(opts.fileList).toBe(false); + expect(opts.treeRoot).toBe("~/work"); + expect(opts.dryRun).toBe(true); + expect(opts.verbose).toBe(true); + expect(opts.skipIfNotRepo).toBe(true); + }); + + test("rejects an invalid --tree-root", () => { + expect(() => buildGitImportOptions({ treeRoot: "bad path!" })).toThrow( + /Invalid --tree-root/, + ); + }); +}); diff --git a/packages/cli/commands/import-git.ts b/packages/cli/commands/import-git.ts new file mode 100644 index 0000000..b5495d8 --- /dev/null +++ b/packages/cli/commands/import-git.ts @@ -0,0 +1,356 @@ +/** + * `me import git` — import a repo's commit history as memories. + * + * One memory per commit (message + capped changed-file list) under + * `..git_history`, with the commit date as the + * memory's temporal and a deterministic id keyed by `(tree, sha)` — so + * re-runs are idempotent (existing commits become server-side skips). + * + * Re-runs are also incremental: the newest already-imported commit is looked + * up server-side (one search) and, when it is an ancestor of the target rev, + * only `..` is walked. Any doubt (force-push, other branch, + * explicit bounds) falls back to the full walk, which deterministic ids make + * safe. `--full` forces the full walk. + */ +import { resolve } from "node:path"; +import * as clack from "@clack/prompts"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; +import { Command, InvalidArgumentError } from "commander"; +import { batchCreateChunked } from "../chunk.ts"; +import type { MemoryClient } from "../client.ts"; +import { resolveCredentials } from "../credentials.ts"; +import { + buildCommitMemory, + GIT_HISTORY_NODE_NAME, + isAncestor, + mergeSkipReason, + walkGitLog, +} from "../importers/git.ts"; +import { + createProgressReporter, + DEFAULT_TREE_ROOT, + dedupByMemoryId, +} from "../importers/index.ts"; +import { SlugRegistry } from "../importers/slug.ts"; +import { getOutputFormat, output } from "../output.ts"; +import { + buildMemoryClient, + handleError, + requireMemoryAuth, + requireSpace, +} from "../util.ts"; +import { VALID_TREE_ROOT_RE } from "./import.ts"; +import { computeSkippedIds } from "./memory-import.ts"; + +/** Parsed options for one git import run. */ +export interface GitImportOptions { + /** Repo path (any directory inside the repo). Default: cwd. */ + repo?: string; + /** Rev to walk (branch, tag, sha). Default: HEAD. */ + branch?: string; + /** `git log --since` bound (git accepts ISO or approxidate). */ + since?: string; + /** `git log --until` bound. */ + until?: string; + /** Cap on walked commits. */ + maxCount?: number; + /** Force the full walk (skip the incremental high-water lookup). */ + full?: boolean; + /** False (via --no-merges) drops all merge commits. */ + merges?: boolean; + /** False (via --no-file-list) omits the changed-file list from content. */ + fileList?: boolean; + /** Tree root under which `.git_history` is placed. */ + treeRoot?: string; + /** Report without writing. */ + dryRun?: boolean; + /** Per-commit progress output. */ + verbose?: boolean; + /** + * Soft-skip (info, success) when the target isn't a git repo — used by + * `me claude init`, which runs in arbitrary directories. + */ + skipIfNotRepo?: boolean; +} + +/** Validate raw Commander opts into a typed option set. */ +export function buildGitImportOptions( + opts: Record, + repoArg?: string, +): GitImportOptions { + const treeRoot = + typeof opts.treeRoot === "string" ? opts.treeRoot : DEFAULT_TREE_ROOT; + if (!VALID_TREE_ROOT_RE.test(treeRoot)) { + throw new Error( + `Invalid --tree-root: '${treeRoot}'. Use ltree labels ([A-Za-z0-9_-]) separated by '.' or '/', with an optional leading '~' for your home.`, + ); + } + return { + repo: repoArg, + branch: typeof opts.branch === "string" ? opts.branch : undefined, + since: typeof opts.since === "string" ? opts.since : undefined, + until: typeof opts.until === "string" ? opts.until : undefined, + maxCount: typeof opts.maxCount === "number" ? opts.maxCount : undefined, + full: opts.full === true, + merges: opts.merges !== false, + fileList: opts.fileList !== false, + treeRoot, + dryRun: opts.dryRun === true, + verbose: opts.verbose === true, + skipIfNotRepo: opts.skipIfNotRepo === true, + }; +} + +/** Structured result of one run (also the --json/--yaml output shape). */ +interface GitImportResult { + repo: string; + remote?: string; + tree: string; + rev: string; + /** The incremental range actually walked, when one was used. */ + range?: string; + dryRun: boolean; + commitsWalked: number; + inserted: number; + /** Already present server-side (idempotent re-import). */ + skipped: number; + /** Merge commits dropped by the boilerplate rule. */ + skippedMerges: number; + failed: number; + errors: Array<{ sha: string; error: string }>; +} + +/** + * Newest already-imported commit sha under `tree`, or null. Unranked search + * returns newest-first by id, and git ids encode the commit date — so one + * `limit: 1` search yields the high-water commit. + */ +async function searchHighWaterSha( + engine: MemoryClient, + tree: string, +): Promise { + const res = await engine.memory.search({ + tree, + meta: { type: "git_commit" }, + limit: 1, + }); + const sha = res.results[0]?.meta.sha; + return typeof sha === "string" && /^[0-9a-f]{40}$/.test(sha) ? sha : null; +} + +/** + * Run one git history import end-to-end and render the outcome. Exported so + * `me claude init` can run it as a setup step, reusing the same + * auth/option/render path as the standalone `me import git`. + */ +export async function runGitImport( + rawOpts: Record, + globalOpts: Record, + repoArg?: string, +): Promise { + const creds = resolveCredentials( + typeof globalOpts.server === "string" ? globalOpts.server : undefined, + ); + const fmt = getOutputFormat(globalOpts); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); + + let opts: GitImportOptions; + try { + opts = buildGitImportOptions(rawOpts, repoArg); + } catch (error) { + handleError(error, fmt); + } + + const repoPath = resolve(opts.repo ?? process.cwd()); + const { slug, gitRoot, gitRemote } = await new SlugRegistry().resolve( + repoPath, + ); + if (!gitRoot) { + if (opts.skipIfNotRepo) { + if (fmt === "text") { + clack.log.info( + `${repoPath} is not a git repository — skipping git history import`, + ); + } + return; + } + handleError(new Error(`${repoPath} is not a git repository`), fmt); + } + + const tree = `${opts.treeRoot}.${slug}.${GIT_HISTORY_NODE_NAME}`; + const rev = opts.branch ?? "HEAD"; + const engine = buildMemoryClient(creds); + + // Incremental fast path: only when nothing narrows the walk explicitly. + const explicitBounds = + opts.full || + opts.since !== undefined || + opts.until !== undefined || + opts.maxCount !== undefined; + let range: string | undefined; + if (!explicitBounds) { + try { + const highWater = await searchHighWaterSha(engine, tree); + if (highWater && (await isAncestor(gitRoot, highWater, rev))) { + range = `${highWater}..${rev}`; + } + } catch (error) { + handleError(error, fmt); + } + } + + const progress = + fmt === "text" ? createProgressReporter(process.stderr) : undefined; + progress?.start(); + + const importedAt = new Date().toISOString(); + const planned: Array<{ memoryId: string; payload: MemoryCreateParams }> = []; + let commitsWalked = 0; + let skippedMerges = 0; + let failed = 0; + const errors: Array<{ sha: string; error: string }> = []; + + try { + for await (const commit of walkGitLog(gitRoot, { + rev, + range, + since: opts.since, + until: opts.until, + maxCount: opts.maxCount, + noMerges: opts.merges === false, + })) { + commitsWalked++; + progress?.process(`${commit.sha.slice(0, 8)} ${commit.subject}`); + if (mergeSkipReason(commit) !== null) { + skippedMerges++; + continue; + } + const built = buildCommitMemory(commit, { + tree, + projectSlug: slug, + gitRemote, + fileList: opts.fileList !== false, + importedAt, + }); + if ("error" in built) { + failed++; + errors.push({ sha: commit.sha, error: built.error }); + continue; + } + planned.push({ memoryId: built.id as string, payload: built }); + if (opts.verbose && fmt === "text") { + const line = ` ${commit.sha.slice(0, 8)} ${commit.subject}`; + if (progress) progress.log(line); + else console.log(line); + } + } + } catch (error) { + progress?.stop(); + handleError(error, fmt); + } + + const { unique } = dedupByMemoryId(planned); + + let inserted = 0; + let skipped = 0; + if (opts.dryRun) { + inserted = unique.length; + } else if (unique.length > 0) { + const submitted = unique.map((p) => p.memoryId); + const result = await batchCreateChunked( + engine, + unique.map((p) => p.payload), + ); + inserted = result.insertedIds.length; + const failedSet = new Set(result.failedIds); + skipped = computeSkippedIds(submitted, result.insertedIds).filter( + (id) => !failedSet.has(id), + ).length; + for (const e of result.errors) { + failed += e.itemCount; + errors.push({ sha: `chunk ${e.chunkIndex}`, error: e.error }); + } + } + progress?.stop(); + + const structured: GitImportResult = { + repo: gitRoot, + remote: gitRemote, + tree, + rev, + range, + dryRun: opts.dryRun === true, + commitsWalked, + inserted, + skipped, + skippedMerges, + failed, + errors, + }; + + output(structured, fmt, () => { + const verb = opts.dryRun ? "Would import" : "Imported"; + clack.log.success( + `${verb} ${inserted} new, ${skipped} already present, ${failed} failed ` + + `commits into ${tree}`, + ); + if (range) console.log(` Incremental walk: ${range}`); + console.log(` Walked ${commitsWalked} commits from ${gitRoot} (${rev})`); + if (skippedMerges > 0) { + console.log(` Skipped ${skippedMerges} boilerplate merge commit(s)`); + } + for (const e of errors) { + console.log(` ✗ ${e.sha}: ${e.error}`); + } + }); + + if (failed > 0 && inserted === 0) process.exit(2); + if (failed > 0) process.exit(1); +} + +/** Parse `--max-count` into a positive integer. */ +function parseMaxCount(value: string): number { + const n = Number.parseInt(value, 10); + if (!Number.isInteger(n) || n <= 0) { + throw new InvalidArgumentError("must be a positive integer"); + } + return n; +} + +/** `me import git` subcommand factory. */ +export function createGitImportCommand(): Command { + return new Command("git") + .description("import a repo's git commit history as memories") + .argument("[repo]", "path inside the repo to import (default: cwd)") + .option("--branch ", "branch/tag/rev to walk (default: HEAD)") + .option( + "--since ", + "only commits at/after this date (any format git accepts)", + ) + .option("--until ", "only commits at/before this date") + .option( + "--max-count ", + "import at most this many recent commits", + parseMaxCount, + ) + .option( + "--full", + "walk the full history (skip the incremental high-water lookup)", + ) + .option("--no-merges", "drop all merge commits") + .option("--no-file-list", "omit the changed-file list from commit memories") + .option( + "--tree-root ", + `tree root under which '.${GIT_HISTORY_NODE_NAME}' is placed (default: ${DEFAULT_TREE_ROOT})`, + ) + .option( + "--dry-run", + "parse and report what would be imported without writing", + ) + .option("-v, --verbose", "per-commit progress output") + .action(async (repoArg: string | undefined, opts, cmdRef) => { + const globalOpts = cmdRef.optsWithGlobals(); + await runGitImport(opts, globalOpts, repoArg); + }); +} diff --git a/packages/cli/commands/import-group.ts b/packages/cli/commands/import-group.ts new file mode 100644 index 0000000..c5db9d4 --- /dev/null +++ b/packages/cli/commands/import-group.ts @@ -0,0 +1,43 @@ +/** + * `me import` — the umbrella group for getting data into Memory Engine. + * + * One subcommand per source: + * + * me import memories file records (md/yaml/json/ndjson) + * me import claude Claude Code sessions + * me import codex Codex sessions + * me import opencode OpenCode sessions + * me import git [repo] git commit history + * + * There is deliberately no bare default: `me import ` does not parse. + * The pre-group spellings stay registered as aliases built from the same + * factories — `me memory import` (⇒ memories) and `me import` + * (⇒ claude/codex/opencode) — so adding a source here is one subcommand, + * never a new top-level command group. + */ +import { Command } from "commander"; +import { + createClaudeImportCommand, + createCodexImportCommand, + createOpenCodeImportCommand, +} from "./import.ts"; +import { createGitImportCommand } from "./import-git.ts"; +import { createGitHookCommand } from "./import-git-hook.ts"; +import { createMemoryImportCommand } from "./memory-import.ts"; + +export function createImportCommand(): Command { + const imp = new Command("import").description( + "import memories, agent sessions, and git history", + ); + imp.addCommand(createMemoryImportCommand("memories")); + imp.addCommand(createClaudeImportCommand("claude")); + imp.addCommand(createCodexImportCommand("codex")); + imp.addCommand(createOpenCodeImportCommand("opencode")); + imp.addCommand(createGitImportCommand()); + imp.addCommand(createGitHookCommand()); + imp.addHelpText( + "after", + "\nTo import memory files (the old `me import `), use: me import memories ", + ); + return imp; +} diff --git a/packages/cli/commands/import.test.ts b/packages/cli/commands/import.test.ts index 75dbc12..b0c5862 100644 --- a/packages/cli/commands/import.test.ts +++ b/packages/cli/commands/import.test.ts @@ -5,7 +5,7 @@ describe("buildOptions", () => { test("defaults imported session node name to agent_sessions", () => { const config = buildOptions({}); - expect(config.write.treeRoot).toBe("projects"); + expect(config.write.treeRoot).toBe("share.projects"); expect(config.write.sessionsNodeName).toBe("agent_sessions"); }); @@ -20,4 +20,19 @@ describe("buildOptions", () => { "Invalid --sessions-node-name: 'agent-sessions'. Must match [a-z0-9_]+", ); }); + + test("accepts a ~ (home) tree root and other lenient forms", () => { + expect(buildOptions({ treeRoot: "~" }).write.treeRoot).toBe("~"); + expect(buildOptions({ treeRoot: "~.work" }).write.treeRoot).toBe("~.work"); + expect(buildOptions({ treeRoot: "~/work" }).write.treeRoot).toBe("~/work"); + expect(buildOptions({ treeRoot: "share.projects" }).write.treeRoot).toBe( + "share.projects", + ); + }); + + test("rejects a tree root with illegal characters", () => { + expect(() => buildOptions({ treeRoot: "bad space" })).toThrow( + "Invalid --tree-root", + ); + }); }); diff --git a/packages/cli/commands/import.ts b/packages/cli/commands/import.ts index 3a788b7..e32ced8 100644 --- a/packages/cli/commands/import.ts +++ b/packages/cli/commands/import.ts @@ -1,12 +1,13 @@ /** - * Shared helpers for the per-agent `import` subcommands. + * Shared helpers for the agent-session import subcommands. * - * Each agent command group (`me claude`, `me codex`, `me opencode`) adds its - * own `import` subcommand via `buildAgentImportSubcommand`. Each source-native - * message becomes one memory, stored under - * `..`. + * Each agent importer is exposed twice: canonically under the import group + * (`me import claude|codex|opencode`) and as an alias under its agent command + * group (`me claude import`, …) — both built from the same per-tool factory + * (`createClaudeImportCommand`, …). Each source-native message becomes one + * memory, stored under `..`. * - * Shared flags across every `import` subcommand: + * Shared flags across every agent import subcommand: * --source

override default source directory * --project only import sessions with this cwd (or a child) * --since only sessions started at/after this timestamp @@ -24,22 +25,35 @@ */ import * as clack from "@clack/prompts"; import { Command } from "commander"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; +import { claudeImporter } from "../importers/claude.ts"; +import { codexImporter } from "../importers/codex.ts"; import { createProgressReporter, + DEFAULT_SESSIONS_NODE_NAME, + DEFAULT_TREE_ROOT, type Importer, type ImportResult, runImport, type WriteOptions, } from "../importers/index.ts"; +import { opencodeImporter } from "../importers/opencode.ts"; import type { ImporterOptions } from "../importers/types.ts"; import { getOutputFormat, output } from "../output.ts"; -import { handleError, requireEngine, requireSession } from "../util.ts"; +import { + buildMemoryClient, + handleError, + requireMemoryAuth, + requireSpace, +} from "../util.ts"; -const DEFAULT_TREE_ROOT = "projects"; -const DEFAULT_SESSIONS_NODE_NAME = "agent_sessions"; -const VALID_TREE_ROOT_RE = /^[a-z0-9_]+(\.[a-z0-9_]+)*$/; +// Default capture layout (share.projects..agent_sessions) lives in the +// importers module so `me import ` and the Claude Code hook share one source. +// Lenient user-facing tree-path input (matches the protocol's treePathSchema): +// labels [A-Za-z0-9_-], `.` or `/` separators, optional leading `~` (home). The +// server normalizes + authoritatively validates; this is a fast pre-check so a +// `--tree-root ~/work` or `~` lands in the caller's home instead of being rejected. +export const VALID_TREE_ROOT_RE = /^[A-Za-z0-9_~./-]+$/; const VALID_TREE_LABEL_RE = /^[a-z0-9_]+$/; /** Build a Commander option set shared by every subcommand. */ @@ -111,7 +125,7 @@ export function buildOptions(opts: Record): { : DEFAULT_SESSIONS_NODE_NAME; if (!VALID_TREE_ROOT_RE.test(treeRoot)) { throw new Error( - `Invalid --tree-root: '${treeRoot}'. Must match [a-z0-9_]+(\\.[a-z0-9_]+)*`, + `Invalid --tree-root: '${treeRoot}'. Use ltree labels ([A-Za-z0-9_-]) separated by '.' or '/', with an optional leading '~' for your home.`, ); } if (!VALID_TREE_LABEL_RE.test(sessionsNodeName)) { @@ -150,8 +164,13 @@ export function buildOptions(opts: Record): { /** * Run one importer end-to-end and render the outcome in the selected format. + * + * Exported so higher-level commands (e.g. `me claude init`) can run an import + * as one step among several, reusing the exact same auth/option/render path as + * the standalone `import` subcommand. `opts` is the parsed import-flag set (pass + * `{}` for defaults); `globalOpts` carries `--server` / output format. */ -async function runAndRender( +export async function runAgentImport( importer: Importer, opts: Record, globalOpts: Record, @@ -160,8 +179,8 @@ async function runAndRender( typeof globalOpts.server === "string" ? globalOpts.server : undefined, ); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); let config: ReturnType; try { @@ -170,7 +189,7 @@ async function runAndRender( handleError(error, fmt); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const engine = buildMemoryClient(creds); if (fmt === "text" && config.write.verbose) { const sourceNote = config.importer.source ?? importer.defaultSource; @@ -297,20 +316,54 @@ function renderResult( } /** - * Build an `import` subcommand bound to a specific importer. Each agent - * command group (`me claude`, `me codex`, `me opencode`) calls this to add - * its own `import` subcommand. + * Build a subcommand bound to a specific importer. Each importer is + * registered twice: under the `me import` group as `me import ` (its + * canonical spelling) and under the agent's command group as the + * `me import` alias — hence the `name` parameter. */ -export function buildAgentImportSubcommand( +function buildAgentImportSubcommand( description: string, importer: Importer, includeSidechainsFlag = false, + name = "import", ): Command { - const cmd = new Command("import").description(description); + const cmd = new Command(name).description(description); addCommonOptions(cmd, includeSidechainsFlag); cmd.action(async (opts, cmdRef) => { const globalOpts = cmdRef.optsWithGlobals(); - await runAndRender(importer, opts, globalOpts); + await runAgentImport(importer, opts, globalOpts); }); return cmd; } + +/** + * Per-tool import subcommand factories. Each owns its importer wiring + + * description in one place so both registrations (`me import ` and the + * `me import` alias) stay identical. + */ +export function createClaudeImportCommand(name = "import"): Command { + return buildAgentImportSubcommand( + "import Claude Code sessions from ~/.claude/projects", + claudeImporter, + true, + name, + ); +} + +export function createCodexImportCommand(name = "import"): Command { + return buildAgentImportSubcommand( + "import Codex sessions from ~/.codex/sessions and archived_sessions", + codexImporter, + false, + name, + ); +} + +export function createOpenCodeImportCommand(name = "import"): Command { + return buildAgentImportSubcommand( + "import OpenCode sessions from ~/.local/share/opencode/storage", + opencodeImporter, + false, + name, + ); +} diff --git a/packages/cli/commands/invitation.ts b/packages/cli/commands/invitation.ts deleted file mode 100644 index 68a69aa..0000000 --- a/packages/cli/commands/invitation.ts +++ /dev/null @@ -1,199 +0,0 @@ -/** - * me invitation — invitation management commands. - * - * - me invitation create : Invite someone to an organization - * - me invitation list [org-id]: List pending invitations - * - me invitation accept : Accept an invitation - * - me invitation revoke : Revoke an invitation - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { handleError, requireSession, resolveOrgId } from "../util.ts"; - -// ============================================================================= -// Invitation Commands -// ============================================================================= - -function createInvitationCreateCommand(): Command { - return new Command("create") - .description("invite someone to an organization") - .argument("", "email address to invite") - .argument("", "role: owner, admin, or member") - .option("--org ", "organization name, slug, or ID") - .option("--expires ", "expiration in days (1-30, default 7)", "7") - .action(async (email: string, role: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const expiresInDays = Number.parseInt(opts.expires, 10); - - const result = await accounts.invitation.create({ - orgId, - email, - role: role as "owner" | "admin" | "member", - expiresInDays, - }); - - output(result, fmt, () => { - clack.log.success(`Invitation sent to ${result.email}`); - console.log(` ID: ${result.id}`); - console.log(` Role: ${result.role}`); - console.log(` Expires: ${result.expiresAt}`); - clack.note( - result.token, - "Invitation token (share this with the invitee)", - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createInvitationListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list pending invitations") - .argument("[org]", "organization name, slug, or ID") - .option("--org ", "organization name, slug, or ID") - .action(async (positionalOrgId: string | undefined, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId( - accounts, - fmt, - opts.org, - positionalOrgId, - ); - const { invitations } = await accounts.invitation.list({ orgId }); - - output({ invitations }, fmt, () => { - if (invitations.length === 0) { - console.log(" No pending invitations."); - return; - } - table( - ["id", "email", "role", "expires"], - invitations.map((inv) => [ - inv.id, - inv.email, - inv.role, - inv.expiresAt, - ]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createInvitationAcceptCommand(): Command { - return new Command("accept") - .description("accept an invitation") - .argument("", "invitation token") - .action(async (token: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const result = await accounts.invitation.accept({ token }); - - // Resolve org name for a friendlier message - let orgName: string | undefined; - if (result.accepted) { - try { - const org = await accounts.org.get({ id: result.orgId }); - orgName = org.name; - } catch { - // Fall back to ID if org lookup fails - } - } - - output(result, fmt, () => { - if (result.accepted) { - const label = orgName ? `'${orgName}'` : result.orgId; - clack.log.success(`Invitation accepted! Joined ${label}.`); - } else { - clack.log.warn("Invitation could not be accepted."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createInvitationRevokeCommand(): Command { - return new Command("revoke") - .description("revoke a pending invitation") - .argument("", "invitation ID") - .action(async (id: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const result = await accounts.invitation.revoke({ id }); - - output(result, fmt, () => { - if (result.revoked) { - clack.log.success("Invitation revoked."); - } else { - clack.log.warn("Invitation not found or already used."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -// ============================================================================= -// Command Group -// ============================================================================= - -export function createInvitationCommand(): Command { - const invitation = new Command("invitation").description( - "manage invitations", - ); - invitation.addCommand(createInvitationCreateCommand()); - invitation.addCommand(createInvitationListCommand()); - invitation.addCommand(createInvitationAcceptCommand()); - invitation.addCommand(createInvitationRevokeCommand()); - return invitation; -} diff --git a/packages/cli/commands/login.ts b/packages/cli/commands/login.ts index ff62418..1b2c6ec 100644 --- a/packages/cli/commands/login.ts +++ b/packages/cli/commands/login.ts @@ -1,31 +1,34 @@ /** - * me login — authenticate via OAuth device flow. + * me login [space] — authenticate via OAuth device flow, then pick the active space. * - * 1. User picks a provider (Google/GitHub) - * 2. CLI starts device flow, gets user code + verification URL - * 3. Opens browser (or tells user to visit URL manually) - * 4. Polls until user completes auth - * 5. Stores session token in credentials file - * 6. Fetches and displays identity + * 1. Compatibility check (fail fast before the browser round-trip) + * 2. Start device flow, show user code + URL, open browser + * 3. Poll until the user authorizes → session token + * 4. Store the session token for the server + * 5. Fetch identity (whoami) and the caller's spaces + * 6. Select the active space (the X-Me-Space the rest of the CLI is scoped to): + * - a [space] argument (slug or name) is honored if it matches + * - otherwise auto-select when the user has exactly one space + * - multiple → prompt (text) / report (json); zero → suggest `me space create` */ import * as clack from "@clack/prompts"; import type { OAuthProvider } from "@memory.build/protocol/auth/device-flow"; +import type { MemberSpaceResponse } from "@memory.build/protocol/user"; import { Command } from "commander"; import { CLIENT_VERSION, MIN_SERVER_VERSION } from "../../../version"; import { checkServerVersion, - createAccountsClient, createAuthClient, + createUserClient, DeviceFlowError, RpcError, } from "../client.ts"; import { - getEngineApiKey, resolveServer, - storeApiKey, + setActiveSpace, storeSessionToken, } from "../credentials.ts"; -import { getOutputFormat, output } from "../output.ts"; +import { getOutputFormat, type OutputFormat, output } from "../output.ts"; /** * Attempt to open a URL in the user's default browser. @@ -48,26 +51,37 @@ async function openBrowser(url: string): Promise { } } +/** + * Match a [space] argument against the caller's spaces, by slug (exact) or name + * (case-insensitive). Returns the match, or null when nothing/ambiguous matches. + */ +function matchSpace( + spaces: MemberSpaceResponse[], + input: string, +): MemberSpaceResponse | null { + const bySlug = spaces.find((s) => s.slug === input); + if (bySlug) return bySlug; + const lower = input.toLowerCase(); + const byName = spaces.filter((s) => s.name.toLowerCase() === lower); + return byName.length === 1 ? (byName[0] ?? null) : null; +} + export function createLoginCommand(): Command { return new Command("login") - .description("authenticate with Memory Engine via OAuth") - .action(async (_opts, cmd) => { + .description("authenticate with Memory Engine and select the active space") + .argument("[space]", "space to activate after login (slug or name)") + .action(async (spaceArg: string | undefined, _opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const server = resolveServer(globalOpts.server); const fmt = getOutputFormat(globalOpts); const auth = createAuthClient({ url: server }); - // --- Provider selection --- if (fmt === "text") { clack.intro("me login"); } - // --- Compatibility check --- - // Verify that this CLI and the server agree on a compatible version - // before sending the user through the OAuth round-trip. Failing here - // is much friendlier than failing after they've authorized in their - // browser. + // --- Compatibility check (before the OAuth round-trip) --- try { await checkServerVersion({ url: server, @@ -75,12 +89,7 @@ export function createLoginCommand(): Command { minServerVersion: MIN_SERVER_VERSION, }); } catch (error) { - const msg = - error instanceof RpcError - ? error.message - : error instanceof Error - ? error.message - : String(error); + const msg = error instanceof Error ? error.message : String(error); if (fmt === "text") { clack.log.error(msg); clack.outro("Login failed."); @@ -99,7 +108,7 @@ export function createLoginCommand(): Command { let flow: Awaited>; try { - flow = await auth.startDeviceFlow(provider as OAuthProvider); + flow = await auth.startDeviceFlow(provider); } catch (error) { spin?.stop("Failed to start device flow."); const msg = error instanceof Error ? error.message : String(error); @@ -114,17 +123,15 @@ export function createLoginCommand(): Command { spin?.stop("Device flow started."); - // --- Display code and open browser --- if (fmt === "text") { clack.note( `Code: ${flow.userCode}\nURL: ${flow.verificationUri}`, "Enter this code in your browser", ); } - await openBrowser(flow.verificationUri); - // --- Poll for token --- + // --- Poll for the session token --- spin?.start("Waiting for authorization..."); try { @@ -132,102 +139,41 @@ export function createLoginCommand(): Command { interval: flow.interval, expiresIn: flow.expiresIn, }); - spin?.stop("Authorized!"); - // Store session token storeSessionToken(server, result.sessionToken); - // Fetch identity via accounts client - const accounts = createAccountsClient({ + const user = createUserClient({ url: server, - sessionToken: result.sessionToken, + token: result.sessionToken, }); - const identity = await accounts.me.get(); - - // Auto-select engine if exactly one exists - let engineInfo: { - name: string; - slug: string; - orgName: string; - } | null = null; - let engineCount = 0; - - try { - const { orgs } = await accounts.org.list(); - const allEngines: Array<{ - id: string; - slug: string; - name: string; - orgName: string; - }> = []; - for (const org of orgs) { - const { engines } = await accounts.engine.list({ - orgId: org.id, - }); - for (const e of engines) { - if (e.status === "active") { - allEngines.push({ - id: e.id, - slug: e.slug, - name: e.name, - orgName: org.name, - }); - } - } - } - engineCount = allEngines.length; - - if (allEngines.length === 1 && allEngines[0]) { - const engine = allEngines[0]; - // Check if we already have a key for this engine - const existingKey = getEngineApiKey(server, engine.slug); - if (existingKey) { - // Already have a key — just ensure it's active - const { setActiveEngine } = await import("../credentials.ts"); - setActiveEngine(server, engine.slug); - engineInfo = { - name: engine.name, - slug: engine.slug, - orgName: engine.orgName, - }; - } else { - // Bootstrap access - const setupResult = await accounts.engine.setupAccess({ - engineId: engine.id, - }); - storeApiKey(server, setupResult.engineSlug, setupResult.rawKey); - engineInfo = { - name: setupResult.engineName, - slug: setupResult.engineSlug, - orgName: setupResult.orgName, - }; - } - } - } catch { - // Engine auto-select is best-effort — don't fail login - } + const identity = await user.whoami(); + const { spaces } = await user.space.list(); + + const active = await selectSpace(server, spaces, spaceArg, fmt); - output({ server, identity, engine: engineInfo }, fmt, () => { + output({ server, identity, space: active }, fmt, () => { clack.log.success( `Logged in as ${identity.name} (${identity.email})`, ); clack.log.info(`Server: ${server}`); - if (engineInfo) { - clack.log.info( - `Engine: ${engineInfo.name} (${engineInfo.orgName})`, - ); - } else if (engineCount > 1) { - clack.log.info( - "Multiple engines found. Run 'me engine use' to select one.", + if (active) { + clack.log.info(`Space: ${active.name} (${active.slug})`); + clack.note( + "Run 'me claude init' at the root of a software development\nproject to set up Claude Code memory for it.", + "Next step", ); + } else if (spaces.length === 0) { + clack.log.info("No spaces yet. Run 'me space create '."); + } else { + clack.log.info("Run 'me space use ' to select a space."); } clack.outro("Done!"); }); } catch (error) { spin?.stop("Authorization failed."); const msg = - error instanceof DeviceFlowError + error instanceof DeviceFlowError || error instanceof RpcError ? error.message : error instanceof Error ? error.message @@ -242,3 +188,52 @@ export function createLoginCommand(): Command { } }); } + +/** + * Resolve and persist the active space after login. Returns the selected space, + * or null when none could be selected (and leaves the active space unchanged). + */ +async function selectSpace( + server: string, + spaces: MemberSpaceResponse[], + spaceArg: string | undefined, + fmt: OutputFormat, +): Promise { + // Explicit argument wins. + if (spaceArg) { + const match = matchSpace(spaces, spaceArg); + if (!match) { + const msg = `No space matching '${spaceArg}'.`; + if (fmt === "text") { + clack.log.error(msg); + for (const s of spaces) console.log(` ${s.name} (${s.slug})`); + } + // Don't abort the whole login — the session is already stored. + return null; + } + setActiveSpace(server, match.slug); + return match; + } + + // Exactly one space → auto-select. + if (spaces.length === 1 && spaces[0]) { + setActiveSpace(server, spaces[0].slug); + return spaces[0]; + } + + // Multiple spaces in an interactive session → prompt. + if (spaces.length > 1 && fmt === "text") { + const choice = await clack.select({ + message: "Select the active space", + options: spaces.map((s) => ({ + value: s.slug, + label: `${s.name} (${s.slug})`, + })), + }); + if (clack.isCancel(choice)) return null; + setActiveSpace(server, choice as string); + return spaces.find((s) => s.slug === choice) ?? null; + } + + return null; +} diff --git a/packages/cli/commands/mcp.test.ts b/packages/cli/commands/mcp.test.ts new file mode 100644 index 0000000..9100805 --- /dev/null +++ b/packages/cli/commands/mcp.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from "bun:test"; +import { isLegacyApiKey } from "./mcp.ts"; + +// Guards the CLI's copy of the legacy-key detector (duplicated from +// @memory.build/engine/core to avoid an engine dependency). Keep in sync with +// the engine version's tests. +describe("isLegacyApiKey", () => { + const legacy = `me.abc123def456.lookupid12345678.${"s".repeat(32)}`; + + test("true for a 4-part legacy (space-scoped) key", () => { + expect(isLegacyApiKey(legacy)).toBe(true); + }); + + test("false for a current 3-part key", () => { + expect(isLegacyApiKey(`me.lookupid12345678.${"s".repeat(32)}`)).toBe(false); + }); + + test("false for an opaque session-like token", () => { + expect(isLegacyApiKey("a".repeat(43))).toBe(false); + }); + + test("false for a 4-part token with a malformed slug", () => { + expect( + isLegacyApiKey(`me.BADSLUG78901.lookupid12345678.${"s".repeat(32)}`), + ).toBe(false); + }); +}); diff --git a/packages/cli/commands/mcp.ts b/packages/cli/commands/mcp.ts index df76f50..a14e0b9 100644 --- a/packages/cli/commands/mcp.ts +++ b/packages/cli/commands/mcp.ts @@ -1,54 +1,95 @@ /** * me mcp — run the MCP server over stdio. * - * Does NOT use the credentials file. API key must be provided - * via --api-key or ME_API_KEY env var. + * Authenticates to a space with either a human session (from `me login`) or an + * agent api key, and targets the active space (the X-Me-Space). Resolution: + * - token: --api-key > ME_API_KEY > stored session token + * - space: --space > ME_SPACE > stored active space + * + * The common case is a logged-in human: `me mcp` just works against the active + * space. Agents pass ME_API_KEY (keys are global, so a space must be given via + * --space / ME_SPACE — the installers bake it in). * * MCP registration with individual AI tools lives in per-agent commands: * me opencode install, me gemini install, me codex install * Claude Code uses the Memory Engine plugin instead of a CLI installer. */ import { Command } from "commander"; +import { resolveCredentials } from "../credentials.ts"; import { runMcpServer } from "../mcp/server.ts"; -const DEFAULT_SERVER = "https://api.memory.build"; +/** + * True if the token is a legacy 4-part api key (`me...`), + * the retired space-scoped format that no longer authenticates. Duplicated from + * `@memory.build/engine/core`'s `isLegacyApiKey` so the CLI doesn't depend on the + * engine package; the legacy format is frozen, so this won't drift. + */ +export function isLegacyApiKey(token: string): boolean { + const parts = token.split("."); + return ( + parts.length === 4 && + parts[0] === "me" && + /^[a-z0-9]{12}$/.test(parts[1] ?? "") && + /^[A-Za-z0-9_-]{16}$/.test(parts[2] ?? "") && + (parts[3]?.length ?? 0) === 32 + ); +} /** - * me mcp — run the MCP server over stdio. - * - * Does NOT use the credentials file. API key must be provided - * via --api-key or ME_API_KEY env var. + * Treat unset / empty / unsubstituted-placeholder flag values as missing. The + * Claude Code plugin's .mcp.json substitutes `${user_config.api_key}` statically; + * when api_key is left blank that arrives as `""` (or the literal placeholder), + * which must fall through to the session, not be used as a token. */ +function blankFlag(v: unknown): string | undefined { + if (typeof v !== "string" || v === "" || /^\$\{.*\}$/.test(v)) + return undefined; + return v; +} + function createMcpRunAction() { return async (_opts: Record, cmd: Command) => { const opts = cmd.optsWithGlobals(); + const creds = resolveCredentials(opts.server as string | undefined); - // Resolve API key: --api-key > ME_API_KEY - const apiKey = - (opts.apiKey as string | undefined) ?? process.env.ME_API_KEY; - if (!apiKey) { + // Token: --api-key > ME_API_KEY (creds.apiKey) > stored session token. + const token = blankFlag(opts.apiKey) ?? creds.apiKey ?? creds.sessionToken; + if (!token) { console.error( - "Error: API key required. Provide via --api-key or ME_API_KEY env var.", + "Error: no credentials. Run 'me login', or pass --api-key / set ME_API_KEY.", ); process.exit(1); } - // Resolve server: --server > ME_SERVER > default - const server = - (opts.server as string | undefined) ?? - process.env.ME_SERVER ?? - DEFAULT_SERVER; + // Fail fast on a retired space-scoped key rather than starting the server and + // failing on the first tool call with a server-side error. + if (isLegacyApiKey(token)) { + console.error( + "Error: this API key uses the old space-scoped format (me...) and no longer works. Recreate it with 'me apikey create ', then update ME_API_KEY or your MCP config.", + ); + process.exit(1); + } - await runMcpServer({ apiKey, server }); + // Space: --space > ME_SPACE / stored active space. + const space = blankFlag(opts.space) ?? creds.activeSpace; + if (!space) { + console.error( + "Error: no active space. Run 'me space use ', or pass --space / set ME_SPACE.", + ); + process.exit(1); + } + + await runMcpServer({ server: creds.server, token, space }); }; } -/** - * Create the MCP command. - */ export function createMcpCommand(): Command { return new Command("mcp") .description("run MCP server over stdio") - .option("--api-key ", "API key for engine authentication") + .option("--api-key ", "agent api key (else uses the stored session)") + .option( + "--space ", + "active space (else ME_SPACE / stored active space)", + ) .action(createMcpRunAction()); } diff --git a/packages/cli/commands/memory-edit.ts b/packages/cli/commands/memory-edit.ts index 4915d18..ab8046e 100644 --- a/packages/cli/commands/memory-edit.ts +++ b/packages/cli/commands/memory-edit.ts @@ -10,7 +10,7 @@ import { unlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { stringify as yamlStringify } from "yaml"; -import type { EngineClient } from "../client.ts"; +import type { MemoryClient } from "../client.ts"; import { parseMarkdown } from "../parsers/markdown.ts"; interface ParsedMemory { @@ -115,7 +115,7 @@ function hasChanges( * Edit a memory interactively. */ export async function editMemory( - engine: EngineClient, + engine: MemoryClient, id: string, ): Promise { const original = await engine.memory.get({ id }); diff --git a/packages/cli/commands/memory-import.test.ts b/packages/cli/commands/memory-import.test.ts index 74e78ba..b317ab1 100644 --- a/packages/cli/commands/memory-import.test.ts +++ b/packages/cli/commands/memory-import.test.ts @@ -1,5 +1,5 @@ /** - * Tests for `me memory import` helpers. + * Tests for `me import memories` (alias `me memory import`) helpers. * * The skip-detection helper exists because `engine.memory.batchCreate` * silently drops conflicting ids (post-#64). Memory import — unlike pack diff --git a/packages/cli/commands/memory-import.ts b/packages/cli/commands/memory-import.ts index 4b1f935..8894ebd 100644 --- a/packages/cli/commands/memory-import.ts +++ b/packages/cli/commands/memory-import.ts @@ -9,9 +9,9 @@ import { existsSync, statSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import * as clack from "@clack/prompts"; +import { SHARE_NAMESPACE } from "@memory.build/protocol"; import { Command } from "commander"; import { batchCreateChunked } from "../chunk.ts"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output } from "../output.ts"; import { @@ -19,7 +19,7 @@ import { type ParsedMemory, parseContent, } from "../parsers/index.ts"; -import { requireEngine, requireSession } from "../util.ts"; +import { buildMemoryClient, requireMemoryAuth, requireSpace } from "../util.ts"; /** * Collect files from a path. If directory, requires --recursive. @@ -89,8 +89,8 @@ export function computeSkippedIds( return explicitIds.filter((id) => !inserted.has(id)); } -export function createMemoryImportCommand(): Command { - return new Command("import") +export function createMemoryImportCommand(name = "import"): Command { + return new Command(name) .description("import memories from files or stdin") .argument("[files...]", "files to import (use - for stdin)") .option("--format ", "override format detection (md|yaml|json)") @@ -102,8 +102,8 @@ export function createMemoryImportCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); // Validate format option if (opts.format && !["md", "yaml", "json"].includes(opts.format)) { @@ -265,15 +265,16 @@ export function createMemoryImportCommand(): Command { } // Actual import - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const engine = buildMemoryClient(creds); let skippedIds: string[] = []; const createParams = allMemories.map(({ memory: mem }) => ({ content: mem.content, + // tree is required on the wire; records without one default to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), })); diff --git a/packages/cli/commands/memory.ts b/packages/cli/commands/memory.ts index 59fc5f3..d5a59e9 100644 --- a/packages/cli/commands/memory.ts +++ b/packages/cli/commands/memory.ts @@ -18,10 +18,14 @@ import { join } from "node:path"; import * as clack from "@clack/prompts"; import { Command } from "commander"; import { stringify as yamlStringify } from "yaml"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; -import { handleError, requireEngine, requireSession } from "../util.ts"; +import { + buildMemoryClient, + handleError, + requireMemoryAuth, + requireSpace, +} from "../util.ts"; import { editMemory } from "./memory-edit.ts"; import { createMemoryImportCommand } from "./memory-import.ts"; import { renderTree } from "./memory-tree.ts"; @@ -96,15 +100,18 @@ function createMemoryCreateCommand(): Command { .description("create a memory") .argument("[content]", "memory content") .option("--content ", "memory content (alternative to positional)") - .option("--tree ", "tree path") + .option( + "--tree ", + "tree path ('share' for shared, '~' for private home)", + ) .option("--meta ", "metadata as JSON") .option("--temporal ", "temporal range (start[,end])") .action(async (positionalContent: string | undefined, opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); // Resolve content: positional > --content flag > stdin let content = positionalContent ?? opts.content; @@ -127,16 +134,27 @@ function createMemoryCreateCommand(): Command { process.exit(1); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + if (!opts.tree) { + if (fmt === "text") { + clack.log.error( + "No tree path provided. Pass --tree ('share' for shared memories, '~' for your private home).", + ); + } else { + output({ error: "No tree path provided" }, fmt, () => {}); + } + process.exit(1); + } + + const client = buildMemoryClient(creds); try { const params: Record = { content }; - if (opts.tree) params.tree = opts.tree; + params.tree = opts.tree; if (opts.meta) params.meta = parseMeta(opts.meta); if (opts.temporal) params.temporal = parseTemporal(opts.temporal); - const memory = await engine.memory.create( - params as Parameters[0], + const memory = await client.memory.create( + params as Parameters[0], ); output(memory, fmt, () => { @@ -158,13 +176,13 @@ function createMemoryGetCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { - const memory = await engine.memory.get({ id }); + const memory = await client.memory.get({ id }); // --json / --yaml: structured output if (fmt !== "text") { @@ -226,13 +244,16 @@ function createMemorySearchCommand(): Command { .option("--temporal-within ", "memory must be within (start,end)") .option("--weight-semantic ", "semantic weight (0-1)") .option("--weight-fulltext ", "fulltext weight (0-1)") - .option("--order-by ", "sort direction (asc|desc)") + .option( + "--order-by ", + "filter-only search: order by recency, desc (default, newest first) | asc", + ) .action(async (query: string | undefined, opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); // Resolve search text. A positional query runs hybrid search if (query && opts.semantic && opts.fulltext) { @@ -304,7 +325,7 @@ function createMemorySearchCommand(): Command { weights.fulltext = Number.parseFloat(opts.weightFulltext); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { const params: Record = { @@ -323,8 +344,8 @@ function createMemorySearchCommand(): Command { params.semanticThreshold = Number.parseFloat(opts.semanticThreshold); if (opts.orderBy) params.orderBy = opts.orderBy; - const result = await engine.memory.search( - params as Parameters[0], + const result = await client.memory.search( + params as Parameters[0], ); output(result, fmt, () => { @@ -367,8 +388,8 @@ function createMemoryUpdateCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); // Resolve content let content = opts.content; @@ -387,7 +408,7 @@ function createMemoryUpdateCommand(): Command { process.exit(1); } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { const params: Record = { id }; @@ -396,8 +417,8 @@ function createMemoryUpdateCommand(): Command { if (opts.meta) params.meta = parseMeta(opts.meta); if (opts.temporal) params.temporal = parseTemporal(opts.temporal); - const memory = await engine.memory.update( - params as Parameters[0], + const memory = await client.memory.update( + params as Parameters[0], ); output(memory, fmt, () => { @@ -420,15 +441,15 @@ function createMemoryDeleteCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { if (UUIDV7_RE.test(idOrTree)) { // Single memory delete - const result = await engine.memory.delete({ id: idOrTree }); + const result = await client.memory.delete({ id: idOrTree }); output(result, fmt, () => { if (result.deleted) { clack.log.success(`Deleted memory ${idOrTree}`); @@ -438,7 +459,7 @@ function createMemoryDeleteCommand(): Command { }); } else { // Tree delete — always dry-run first - const preview = await engine.memory.deleteTree({ + const preview = await client.memory.deleteTree({ tree: idOrTree, dryRun: true, }); @@ -473,7 +494,7 @@ function createMemoryDeleteCommand(): Command { } } - const result = await engine.memory.deleteTree({ + const result = await client.memory.deleteTree({ tree: idOrTree, dryRun: false, }); @@ -497,13 +518,13 @@ function createMemoryEditCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { - await editMemory(engine, id); + await editMemory(client, id); } catch (error) { handleError(error, fmt); } @@ -519,18 +540,18 @@ function createMemoryTreeCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { const params: Record = {}; if (filter) params.tree = filter; if (opts.levels) params.levels = Number.parseInt(opts.levels, 10); - const result = await engine.memory.tree( - params as Parameters[0], + const result = await client.memory.tree( + params as Parameters[0], ); output(result, fmt, () => { @@ -554,14 +575,14 @@ function createMemoryMoveCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { // Always dry-run first to show preview - const preview = await engine.memory.move({ + const preview = await client.memory.move({ source: src, destination: dst, dryRun: true, @@ -597,7 +618,7 @@ function createMemoryMoveCommand(): Command { } } - const result = await engine.memory.move({ + const result = await client.memory.move({ source: src, destination: dst, }); @@ -650,8 +671,8 @@ function createMemoryExportCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); const format = opts.format as "json" | "yaml" | "md"; if (!["json", "yaml", "md"].includes(format)) { @@ -694,11 +715,11 @@ function createMemoryExportCommand(): Command { }; } - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { - const result = await engine.memory.search( - searchParams as Parameters[0], + const result = await client.memory.search( + searchParams as Parameters[0], ); const memories = result.results.map((r: Record) => @@ -871,17 +892,41 @@ function renderMarkdownAnsi(content: string): string { // Command Group // ============================================================================= +/** + * Build a fresh set of the memory subcommands. A Commander command can only be + * attached to one parent, so callers that want them in two places (the `memory` + * group and the top-level aliases) each call this for their own instances. + */ +function memorySubcommands(): Command[] { + return [ + createMemoryCreateCommand(), + createMemoryGetCommand(), + createMemorySearchCommand(), + createMemoryUpdateCommand(), + createMemoryDeleteCommand(), + createMemoryEditCommand(), + createMemoryTreeCommand(), + createMemoryMoveCommand(), + createMemoryImportCommand(), + createMemoryExportCommand(), + ]; +} + export function createMemoryCommand(): Command { const memory = new Command("memory").description("manage memories"); - memory.addCommand(createMemoryCreateCommand()); - memory.addCommand(createMemoryGetCommand()); - memory.addCommand(createMemorySearchCommand()); - memory.addCommand(createMemoryUpdateCommand()); - memory.addCommand(createMemoryDeleteCommand()); - memory.addCommand(createMemoryEditCommand()); - memory.addCommand(createMemoryTreeCommand()); - memory.addCommand(createMemoryMoveCommand()); - memory.addCommand(createMemoryImportCommand()); - memory.addCommand(createMemoryExportCommand()); + for (const c of memorySubcommands()) memory.addCommand(c); return memory; } + +/** + * The memory subcommands as top-level aliases (`me search`, `me create`, …) so + * the `memory` word is optional for the common data-plane operations. + * + * `import` is excluded: the top-level `import` name belongs to the + * `me import` source group (see commands/import-group.ts), where the file + * importer lives as `me import memories`. `me memory import` remains its + * alias. + */ +export function createMemoryAliasCommands(): Command[] { + return memorySubcommands().filter((c) => c.name() !== "import"); +} diff --git a/packages/cli/commands/opencode.ts b/packages/cli/commands/opencode.ts index 308fd71..b31a64a 100644 --- a/packages/cli/commands/opencode.ts +++ b/packages/cli/commands/opencode.ts @@ -4,23 +4,30 @@ * - me opencode install: register me as an MCP server with OpenCode */ import { Command } from "commander"; -import { opencodeImporter } from "../importers/opencode.ts"; import { type AgentInstallOptions, runAgentMcpInstall, } from "../mcp/agent-install.ts"; -import { buildAgentImportSubcommand } from "./import.ts"; +import { createOpenCodeImportCommand } from "./import.ts"; function createOpenCodeInstallCommand(): Command { return new Command("install") .description("register me as an MCP server with OpenCode") - .option("--api-key ", "API key to embed in MCP config") + .option( + "--api-key ", + "API key for a headless agent (default: use your login session at runtime)", + ) .option("--server ", "server URL to embed in MCP config") + .option( + "--space ", + "pin a space (default: resolve ME_SPACE / active space at runtime)", + ) .action(async (opts: AgentInstallOptions, cmd: Command) => { const globalOpts = cmd.optsWithGlobals(); await runAgentMcpInstall("opencode", { apiKey: opts.apiKey, server: globalOpts.server ?? opts.server, + space: opts.space, }); }); } @@ -28,11 +35,6 @@ function createOpenCodeInstallCommand(): Command { export function createOpenCodeCommand(): Command { const opencode = new Command("opencode").description("OpenCode integration"); opencode.addCommand(createOpenCodeInstallCommand()); - opencode.addCommand( - buildAgentImportSubcommand( - "import OpenCode sessions from ~/.local/share/opencode/storage", - opencodeImporter, - ), - ); + opencode.addCommand(createOpenCodeImportCommand()); return opencode; } diff --git a/packages/cli/commands/org.ts b/packages/cli/commands/org.ts deleted file mode 100644 index 2cae998..0000000 --- a/packages/cli/commands/org.ts +++ /dev/null @@ -1,334 +0,0 @@ -/** - * me org — organization management commands. - * - * - me org list: List your organizations - * - me org create : Create an organization - * - me org rename : Rename an organization - * - me org delete : Delete an organization - * - me org member list [org]: List members - * - me org member add : Add a member - * - me org member remove : Remove a member - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireSession, - resolveIdentityId, - resolveMember, - resolveOrg, - resolveOrgId, -} from "../util.ts"; - -// ============================================================================= -// Org Commands -// ============================================================================= - -function createOrgListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list your organizations") - .action(async (_opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const { orgs } = await accounts.org.list(); - - output({ orgs }, fmt, () => { - if (orgs.length === 0) { - console.log(" No organizations found."); - return; - } - table( - ["id", "name", "slug"], - orgs.map((org) => [org.id, org.name, org.slug]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgCreateCommand(): Command { - return new Command("create") - .description("create an organization") - .argument("", "organization name") - .action(async (name: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const org = await accounts.org.create({ name }); - - output(org, fmt, () => { - clack.log.success(`Created organization '${org.name}'`); - console.log(` ID: ${org.id}`); - console.log(` Slug: ${org.slug}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgRenameCommand(): Command { - return new Command("rename") - .description("rename an organization") - .argument("", "organization name, slug, or ID") - .argument("", "new organization name") - .action(async (nameOrId: string, newName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const org = await resolveOrg(accounts, fmt, undefined, nameOrId); - const oldName = org.name; - const updated = await accounts.org.update({ - id: org.id, - name: newName, - }); - - output(updated, fmt, () => { - clack.log.success( - `Renamed organization '${oldName}' → '${updated.name}'`, - ); - console.log(` ID: ${updated.id}`); - console.log(` Slug: ${updated.slug}`); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("delete an organization") - .argument("", "organization name, slug, or ID") - .option("-y, --yes", "skip confirmation prompt") - .action(async (nameOrId: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const org = await resolveOrg(accounts, fmt, undefined, nameOrId); - - // Confirm in text mode unless --yes - if (fmt === "text" && !opts.yes) { - const confirmed = await clack.confirm({ - message: `Delete organization '${org.name}'? This cannot be undone.`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const result = await accounts.org.delete({ id: org.id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success(`Organization '${org.name}' deleted.`); - } else { - clack.log.warn("Organization not found."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -// ============================================================================= -// Org Member Commands -// ============================================================================= - -function createOrgMemberListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list organization members") - .argument("[org]", "organization name, slug, or ID") - .option("--org ", "organization name, slug, or ID") - .action(async (positionalOrgId: string | undefined, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId( - accounts, - fmt, - opts.org, - positionalOrgId, - ); - const { members } = await accounts.org.member.list({ orgId }); - - output({ members }, fmt, () => { - if (members.length === 0) { - console.log(" No members found."); - return; - } - table( - ["name", "email", "role", "joined"], - members.map((m) => [m.name, m.email, m.role, m.createdAt]), - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgMemberAddCommand(): Command { - return new Command("add") - .description("add a member to an organization") - .argument("", "email address or identity ID") - .argument("", "role: owner, admin, or member") - .option("--org ", "organization name, slug, or ID") - .action(async (emailOrId: string, role: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const identityId = await resolveIdentityId(accounts, fmt, emailOrId); - const member = await accounts.org.member.add({ - orgId, - identityId, - role: role as "owner" | "admin" | "member", - }); - - output(member, fmt, () => { - clack.log.success( - `Added ${member.name} (${member.email}) as ${member.role}`, - ); - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -function createOrgMemberRemoveCommand(): Command { - return new Command("remove") - .description("remove a member from an organization") - .argument("", "member name, email, or identity ID") - .option("--org ", "organization name, slug, or ID") - .option("-y, --yes", "skip confirmation prompt") - .action(async (nameEmailOrId: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - - const accounts = createAccountsClient({ - url: creds.server, - sessionToken: creds.sessionToken, - }); - - try { - const orgId = await resolveOrgId(accounts, fmt, opts.org); - const member = await resolveMember(accounts, fmt, orgId, nameEmailOrId); - - // Confirm in text mode unless --yes - if (fmt === "text" && !opts.yes) { - const label = member.email - ? `${member.name} (${member.email})` - : member.name; - const confirmed = await clack.confirm({ - message: `Remove ${label}?`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const result = await accounts.org.member.remove({ - orgId, - identityId: member.identityId, - }); - - output(result, fmt, () => { - if (result.removed) { - clack.log.success(`Removed ${member.name}.`); - } else { - clack.log.warn("Member not found."); - } - }); - } catch (error) { - handleError(error, fmt, { sessionServer: creds.server }); - } - }); -} - -// ============================================================================= -// Command Group -// ============================================================================= - -function createOrgMemberCommand(): Command { - const member = new Command("member").description( - "manage organization members", - ); - member.addCommand(createOrgMemberListCommand()); - member.addCommand(createOrgMemberAddCommand()); - member.addCommand(createOrgMemberRemoveCommand()); - return member; -} - -export function createOrgCommand(): Command { - const org = new Command("org").description("manage organizations"); - org.addCommand(createOrgListCommand()); - org.addCommand(createOrgCreateCommand()); - org.addCommand(createOrgRenameCommand()); - org.addCommand(createOrgDeleteCommand()); - org.addCommand(createOrgMemberCommand()); - return org; -} diff --git a/packages/cli/commands/owner.ts b/packages/cli/commands/owner.ts deleted file mode 100644 index ba9773c..0000000 --- a/packages/cli/commands/owner.ts +++ /dev/null @@ -1,148 +0,0 @@ -/** - * me owner — tree ownership management commands. - * - * - me owner set : Set tree path owner - * - me owner remove : Remove tree path owner - * - me owner get : Get tree path owner - * - me owner list [user]: List ownership records - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createOwnerSetCommand(): Command { - return new Command("set") - .description("set tree path owner") - .argument("", "tree path") - .argument("", "user name or ID") - .action(async (path: string, user: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const result = await engine.owner.set({ userId, treePath: path }); - - output(result, fmt, () => { - clack.log.success(`Set owner of '${path}' to ${user}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createOwnerRemoveCommand(): Command { - return new Command("remove") - .description("remove tree path owner") - .argument("", "tree path") - .action(async (path: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const result = await engine.owner.remove({ treePath: path }); - - output(result, fmt, () => { - if (result.removed) { - clack.log.success(`Removed owner of '${path}'`); - } else { - clack.log.warn(`No owner found for '${path}'`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createOwnerGetCommand(): Command { - return new Command("get") - .description("get tree path owner") - .argument("", "tree path") - .action(async (path: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const owner = await engine.owner.get({ treePath: path }); - - output(owner, fmt, () => { - console.log(` Path: ${owner.treePath}`); - console.log(` Owner: ${owner.userName}`); - console.log(` Set by: ${owner.createdByName ?? "(unknown)"}`); - console.log(` Created: ${owner.createdAt}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createOwnerListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list ownership records") - .argument("[user]", "filter by user name or ID (optional)") - .action(async (user: string | undefined, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = user ? await resolveUserId(engine, user) : undefined; - const { owners } = await engine.owner.list( - userId ? { userId } : undefined, - ); - - output({ owners }, fmt, () => { - if (owners.length === 0) { - console.log(" No ownership records found."); - return; - } - table( - ["tree_path", "owner"], - owners.map((o) => [o.treePath, o.userName]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createOwnerCommand(): Command { - const owner = new Command("owner").description("manage tree ownership"); - owner.addCommand(createOwnerSetCommand()); - owner.addCommand(createOwnerRemoveCommand()); - owner.addCommand(createOwnerGetCommand()); - owner.addCommand(createOwnerListCommand()); - return owner; -} diff --git a/packages/cli/commands/pack.ts b/packages/cli/commands/pack.ts index 0deddfb..ac4a553 100644 --- a/packages/cli/commands/pack.ts +++ b/packages/cli/commands/pack.ts @@ -2,19 +2,23 @@ * me pack — memory pack management commands. * * - me pack validate : Validate a pack file locally - * - me pack install : Install a memory pack into the active engine - * - me pack list: List installed packs in the active engine + * - me pack install : Install a memory pack into the active space + * - me pack list: List installed packs in the active space */ import { readFileSync } from "node:fs"; import * as clack from "@clack/prompts"; import { Command } from "commander"; import { batchCreateChunked } from "../chunk.ts"; -import { createClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output, table } from "../output.ts"; import { parsePack, validatePackConstraints } from "../parsers/pack.ts"; -import { handleError, requireEngine, requireSession } from "../util.ts"; +import { + buildMemoryClient, + handleError, + requireMemoryAuth, + requireSpace, +} from "../util.ts"; // ============================================================================= // Validate @@ -91,7 +95,7 @@ function createPackValidateCommand(): Command { function createPackInstallCommand(): Command { return new Command("install") - .description("install a memory pack into the active engine") + .description("install a memory pack into the active space") .argument("", "pack YAML file to install") .option("--dry-run", "preview what would happen without making changes") .option("-y, --yes", "skip confirmation for stale memory deletion") @@ -99,8 +103,8 @@ function createPackInstallCommand(): Command { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); try { // Step 1: Read and validate @@ -126,14 +130,11 @@ function createPackInstallCommand(): Command { const packName = envelope.name; const packVersion = envelope.version; - // Step 2: Connect to engine - const engine = createClient({ - url: creds.server, - apiKey: creds.apiKey, - }); + // Step 2: connect to the active space + const client = buildMemoryClient(creds); // Step 3: Search for existing memories with same pack name - const existing = await engine.memory.search({ + const existing = await client.memory.search({ meta: { pack: { name: packName } }, limit: 1000, }); @@ -215,7 +216,7 @@ function createPackInstallCommand(): Command { ); for (const mem of stale) { - await engine.memory.delete({ id: mem.id }); + await client.memory.delete({ id: mem.id }); } spin?.stop( @@ -247,7 +248,7 @@ function createPackInstallCommand(): Command { insertedIds, failedIds, errors: chunkErrors, - } = await batchCreateChunked(engine, createParams); + } = await batchCreateChunked(client, createParams); spin?.stop("Done"); @@ -360,19 +361,19 @@ function createPackInstallCommand(): Command { function createPackListCommand(): Command { return new Command("list") .alias("ls") - .description("list installed packs in the active engine") + .description("list installed packs in the active space") .action(async (_opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); + requireMemoryAuth(creds, fmt); + requireSpace(creds, fmt); - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); + const client = buildMemoryClient(creds); try { // Search for all memories with meta.pack - const result = await engine.memory.search({ + const result = await client.memory.search({ meta: { pack: {} }, limit: 1000, }); @@ -424,7 +425,7 @@ function createPackListCommand(): Command { // ============================================================================= /** - * `engine.memory.batchCreate` uses `ON CONFLICT (id) DO NOTHING` server-side, + * `client.memory.batchCreate` uses `ON CONFLICT (id) DO NOTHING` server-side, * so the returned `ids` array can be shorter than the request when conflicts * occur. For pack install, ids that didn't land fall into three buckets: * diff --git a/packages/cli/commands/role.ts b/packages/cli/commands/role.ts deleted file mode 100644 index 17e3654..0000000 --- a/packages/cli/commands/role.ts +++ /dev/null @@ -1,282 +0,0 @@ -/** - * me role — role management commands. - * - * - me role create : Create a role - * - me role delete : Delete a role (alias: rm) - * - me role list: List all roles - * - me role add-member : Add user to role (by ID or name) - * - me role remove-member : Remove user from role (by ID or name) - * - me role members : List role members (by ID or name) - * - me role list-for : List roles a user belongs to (by ID or name) - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createRoleCreateCommand(): Command { - return new Command("create") - .description("create a role") - .argument("", "role name") - .option("--identity-id ", "link to an accounts identity") - .action(async (name: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const role = await engine.role.create({ - name, - identityId: opts.identityId ?? undefined, - }); - - output(role, fmt, () => { - clack.log.success(`Created role '${role.name}'`); - console.log(` ID: ${role.id}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("delete a role") - .argument("", "role ID or name") - .option("-y, --yes", "skip confirmation prompt") - .action(async (idOrName: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - if (fmt === "text" && !opts.yes) { - const confirmed = await clack.confirm({ - message: `Delete role ${idOrName}? This removes all grants and memberships. This cannot be undone.`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const result = await engine.user.delete({ id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success("Role deleted."); - } else { - clack.log.warn("Role not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list all roles") - .action(async (_opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - // Roles are users with canLogin=false - const { users: roles } = await engine.user.list({ canLogin: false }); - - output({ roles }, fmt, () => { - if (roles.length === 0) { - console.log(" No roles found."); - return; - } - table( - ["id", "name"], - roles.map((r) => [r.id, r.name]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleAddMemberCommand(): Command { - return new Command("add-member") - .description("add a user to a role") - .argument("", "role ID or name") - .argument("", "member ID or name") - .option("--with-admin-option", "allow member to manage this role") - .action(async (role: string, member: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const [roleId, memberId] = await Promise.all([ - resolveUserId(engine, role), - resolveUserId(engine, member), - ]); - - const result = await engine.role.addMember({ - roleId, - memberId, - withAdminOption: opts.withAdminOption ?? false, - }); - - output(result, fmt, () => { - if (result.added) { - clack.log.success(`Added ${member} to role ${role}`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleRemoveMemberCommand(): Command { - return new Command("remove-member") - .description("remove a user from a role") - .argument("", "role ID or name") - .argument("", "member ID or name") - .action(async (role: string, member: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const [roleId, memberId] = await Promise.all([ - resolveUserId(engine, role), - resolveUserId(engine, member), - ]); - - const result = await engine.role.removeMember({ roleId, memberId }); - - output(result, fmt, () => { - if (result.removed) { - clack.log.success(`Removed ${member} from role ${role}`); - } else { - clack.log.warn("Membership not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleMembersCommand(): Command { - return new Command("members") - .description("list members of a role") - .argument("", "role ID or name") - .action(async (role: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const roleId = await resolveUserId(engine, role); - const { members } = await engine.role.listMembers({ roleId }); - - output({ members }, fmt, () => { - if (members.length === 0) { - console.log(" No members found."); - return; - } - table( - ["member_id", "name", "admin"], - members.map((m) => [ - m.memberId, - m.memberName, - m.withAdminOption ? "yes" : "", - ]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createRoleListForCommand(): Command { - return new Command("list-for") - .description("list roles a user belongs to") - .argument("", "user ID or name") - .action(async (user: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const userId = await resolveUserId(engine, user); - const { roles } = await engine.role.listForUser({ userId }); - - output({ roles }, fmt, () => { - if (roles.length === 0) { - console.log(" No roles found."); - return; - } - table( - ["id", "name", "admin"], - roles.map((r) => [r.id, r.name, r.withAdminOption ? "yes" : ""]), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createRoleCommand(): Command { - const role = new Command("role").description("manage roles"); - role.addCommand(createRoleCreateCommand()); - role.addCommand(createRoleDeleteCommand()); - role.addCommand(createRoleListCommand()); - role.addCommand(createRoleAddMemberCommand()); - role.addCommand(createRoleRemoveMemberCommand()); - role.addCommand(createRoleMembersCommand()); - role.addCommand(createRoleListForCommand()); - return role; -} diff --git a/packages/cli/commands/serve.ts b/packages/cli/commands/serve.ts index 9580c81..0573808 100644 --- a/packages/cli/commands/serve.ts +++ b/packages/cli/commands/serve.ts @@ -3,7 +3,8 @@ * * Launches a local HTTP server that: * - serves the embedded Vite-built React app - * - proxies POST /rpc to the configured engine, injecting the stored API key + * - proxies POST /rpc to the space memory endpoint, injecting the session token + * and the active space (X-Me-Space) * * Usage: * me serve [--port ] [--host ] [--no-open] @@ -16,7 +17,7 @@ import { Command } from "commander"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output } from "../output.ts"; import { findAvailablePort, startHttpServer } from "../serve/http-server.ts"; -import { requireEngine } from "../util.ts"; +import { requireSession, requireSpace } from "../util.ts"; const DEFAULT_PORT = 3000; const DEFAULT_HOST = "127.0.0.1"; @@ -40,7 +41,8 @@ export function createServeCommand(): Command { const fmt = getOutputFormat(globalOpts); const creds = resolveCredentials(globalOpts.server); - requireEngine(creds, fmt); + requireSession(creds, fmt); + requireSpace(creds, fmt); const host: string = opts.host ?? DEFAULT_HOST; const explicitPortFlag = opts.port !== undefined; @@ -74,8 +76,8 @@ export function createServeCommand(): Command { try { running = startHttpServer({ server: creds.server, - apiKey: creds.apiKey, - engineSlug: creds.activeEngine ?? "", + token: creds.sessionToken, + space: creds.activeSpace, host, port, }); @@ -95,9 +97,7 @@ export function createServeCommand(): Command { if (fmt === "text") { clack.log.success(`Memory Engine UI running at ${running.url}`); console.log(` Remote server: ${creds.server}`); - if (creds.activeEngine) { - console.log(` Active engine: ${creds.activeEngine}`); - } + console.log(` Active space: ${creds.activeSpace}`); console.log(" Press Ctrl+C to stop."); } else { output( @@ -106,7 +106,7 @@ export function createServeCommand(): Command { host, port: port, server: creds.server, - engine: creds.activeEngine, + space: creds.activeSpace, }, fmt, () => {}, diff --git a/packages/cli/commands/space.ts b/packages/cli/commands/space.ts new file mode 100644 index 0000000..d5752f4 --- /dev/null +++ b/packages/cli/commands/space.ts @@ -0,0 +1,455 @@ +/** + * me space — manage the spaces you belong to and the active space. + * + * - me space list: list your spaces (marks the active one) + * - me space use : set the active space (the X-Me-Space) + * - me space create : create a space and make it active + * - me space rename : rename a space's display label + * - me space delete : delete a space and all its data + * - me space invite [--admin] [--share ]: invite by email (adds + * an existing user now, else a pending invite redeemed at their first login) + * - me space invite list: list pending invitations + * - me space invite revoke : revoke a pending invitation + * + * accepts a slug (exact) or a name (case-insensitive). The slug is the + * immutable 12-char routing key; the name is the renamable display label. + */ +import * as clack from "@clack/prompts"; +import { + type AccessLevel, + accessLevelName, + parseAccessLevel, +} from "@memory.build/protocol/space"; +import type { MemberSpaceResponse } from "@memory.build/protocol/user"; +import { Command } from "commander"; +import { createUserClient } from "../client.ts"; +import { + clearActiveSpace, + resolveCredentials, + setActiveSpace, +} from "../credentials.ts"; +import { + getOutputFormat, + type OutputFormat, + output, + table, +} from "../output.ts"; +import { + buildMemoryClient, + handleError, + requireSession, + requireSpace, +} from "../util.ts"; + +/** + * Resolve a argument against the caller's spaces by slug (exact) or + * name (case-insensitive). With no argument, prompts in text mode. Exits on a + * miss / ambiguity / non-interactive-without-arg. + */ +async function resolveSpaceArg( + spaces: MemberSpaceResponse[], + arg: string | undefined, + fmt: OutputFormat, +): Promise { + if (!arg) { + if (fmt !== "text") { + output({ error: "A space slug or name is required" }, fmt, () => {}); + process.exit(1); + } + if (spaces.length === 0) { + clack.log.error("You don't belong to any spaces."); + process.exit(1); + } + const selected = await clack.select({ + message: "Select a space", + options: spaces.map((s) => ({ + value: s.slug, + label: s.name, + hint: s.slug, + })), + }); + if (clack.isCancel(selected)) { + clack.cancel("Cancelled."); + process.exit(0); + } + const picked = spaces.find((s) => s.slug === selected); + if (picked) return picked; + process.exit(1); + } + + const bySlug = spaces.find((s) => s.slug === arg); + if (bySlug) return bySlug; + + const lower = arg.toLowerCase(); + const byName = spaces.filter((s) => s.name.toLowerCase() === lower); + if (byName.length === 1 && byName[0]) return byName[0]; + + if (byName.length === 0) { + const msg = `No space matching '${arg}'.`; + if (fmt === "text") { + clack.log.error(msg); + for (const s of spaces) console.log(` ${s.name} (${s.slug})`); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); + } + + const msg = `Multiple spaces named '${arg}'. Use the slug instead:`; + if (fmt === "text") { + clack.log.error(msg); + for (const s of byName) console.log(` ${s.name} — ${s.slug}`); + } else { + output({ error: msg, matches: byName }, fmt, () => {}); + } + process.exit(1); +} + +function createSpaceListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list the spaces you belong to") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + output( + { + spaces: spaces.map((s) => ({ + ...s, + active: s.slug === creds.activeSpace, + })), + }, + fmt, + () => { + if (spaces.length === 0) { + console.log(" No spaces. Run 'me space create '."); + return; + } + table( + ["name", "slug", "admin", "active"], + spaces.map((s) => [ + s.name, + s.slug, + s.admin ? "yes" : "", + s.slug === creds.activeSpace ? "active" : "", + ]), + ); + }, + ); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceUseCommand(): Command { + return new Command("use") + .description("set the active space") + .argument("[space]", "space slug or name") + .action(async (arg: string | undefined, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + const space = await resolveSpaceArg(spaces, arg, fmt); + setActiveSpace(creds.server, space.slug); + output({ space, switched: true }, fmt, () => { + clack.log.success(`Active space: ${space.name} (${space.slug})`); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceCreateCommand(): Command { + return new Command("create") + .description("create a new space and make it active") + .argument("", "space display name") + .action(async (name: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const created = await user.space.create({ name }); + // A new space's creator is its admin + owner@root — make it active. + setActiveSpace(creds.server, created.slug); + output({ ...created, name, active: true }, fmt, () => { + clack.log.success(`Created space '${name}' (${created.slug})`); + clack.log.info("It is now your active space."); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceRenameCommand(): Command { + return new Command("rename") + .description("rename a space's display label (the slug is immutable)") + .argument("", "space slug or name") + .argument("", "new display name") + .action(async (arg: string, newName: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + const space = await resolveSpaceArg(spaces, arg, fmt); + const oldName = space.name; + const result = await user.space.rename({ + slug: space.slug, + name: newName, + }); + output({ slug: space.slug, name: newName, ...result }, fmt, () => { + clack.log.success( + `Renamed space '${oldName}' → '${newName}' (${space.slug})`, + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceDeleteCommand(): Command { + return new Command("delete") + .alias("rm") + .description("permanently delete a space and all its data") + .argument("", "space slug or name") + .option("--force", "skip confirmation prompt") + .action(async (arg: string, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + + const user = createUserClient({ + url: creds.server, + token: creds.sessionToken, + }); + + try { + const { spaces } = await user.space.list(); + const space = await resolveSpaceArg(spaces, arg, fmt); + + if (fmt === "text" && !opts.force) { + clack.log.warn( + "This permanently deletes the space and ALL its data (memories, grants, groups).", + ); + clack.log.warn("This action cannot be undone."); + const confirmation = await clack.text({ + message: `Type the space name "${space.name}" to confirm deletion`, + validate: (value) => + value !== space.name + ? `Please type "${space.name}" exactly to confirm` + : undefined, + }); + if (clack.isCancel(confirmation)) { + clack.cancel("Cancelled."); + process.exit(0); + } + } + + const result = await user.space.delete({ slug: space.slug }); + // If we just deleted the active space, drop the stale pointer. + if (result.deleted && creds.activeSpace === space.slug) { + clearActiveSpace(creds.server); + } + output({ slug: space.slug, ...result }, fmt, () => { + if (result.deleted) { + clack.log.success(`Space '${space.name}' has been deleted.`); + if (creds.activeSpace === space.slug) { + clack.log.info("Run 'me space use ' to pick another."); + } + } + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +/** + * Map a `--share` value to the nullable access level: "none" → null (no share + * grant), otherwise read/write/owner via the shared parser. Exits on bad input. + */ +function parseShareLevel(value: string, fmt: OutputFormat): AccessLevel | null { + if (value.trim().toLowerCase() === "none") return null; + const level = parseAccessLevel(value); + if (level !== null) return level; + const msg = `Invalid --share value '${value}'. Use none, read, write, or owner.`; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); +} + +/** Display label for a stored share-access level (null → "none"). */ +function shareLabel(level: AccessLevel | null): string { + return level === null ? "none" : accessLevelName(level); +} + +function createSpaceInviteListCommand(): Command { + return new Command("list") + .alias("ls") + .description("list pending invitations for the active space") + .action(async (_opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const { invitations } = await memory.invite.list(); + output({ invitations }, fmt, () => { + if (invitations.length === 0) { + console.log(" No pending invitations."); + return; + } + table( + ["email", "admin", "share", "invited by", "created"], + invitations.map((i) => [ + i.email, + i.admin ? "yes" : "", + shareLabel(i.shareAccess), + i.invitedByName ?? "", + i.createdAt, + ]), + ); + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceInviteRevokeCommand(): Command { + return new Command("revoke") + .description("revoke a pending invitation by email") + .argument("", "the invitee's email") + .action(async (email: string, _opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + const memory = buildMemoryClient(creds); + try { + const result = await memory.invite.revoke({ email }); + output({ email, ...result }, fmt, () => { + if (result.revoked) { + clack.log.success(`Revoked the invitation for ${email}.`); + } else { + clack.log.warn(`No pending invitation for ${email}.`); + } + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); +} + +function createSpaceInviteCommand(): Command { + const invite = new Command("invite") + .description("invite a user to the active space by email") + .argument("[email]", "the invitee's email (omit when using a subcommand)") + .option("--admin", "make the user a space admin") + .option( + "--share ", + "shared-root access to grant: none | read | write | owner", + "read", + ) + .action(async (email: string | undefined, opts, cmd) => { + const globalOpts = cmd.optsWithGlobals(); + const creds = resolveCredentials(globalOpts.server); + const fmt = getOutputFormat(globalOpts); + requireSession(creds, fmt); + requireSpace(creds, fmt); + + if (!email) { + const msg = + "An email is required: me space invite [--admin] [--share ]"; + if (fmt === "text") { + clack.log.error(msg); + } else { + output({ error: msg }, fmt, () => {}); + } + process.exit(1); + } + + const shareAccess = parseShareLevel(opts.share, fmt); + const memory = buildMemoryClient(creds); + try { + const result = await memory.invite.create({ + email, + admin: opts.admin === true, + shareAccess, + }); + output({ email, ...result }, fmt, () => { + if (result.applied) { + clack.log.success( + `Added ${email} to the space${opts.admin ? " as an admin" : ""}.`, + ); + } else { + clack.log.success( + `Invited ${email} — they'll join when they next sign in.`, + ); + } + }); + } catch (error) { + handleError(error, fmt, { sessionServer: creds.server }); + } + }); + invite.addCommand(createSpaceInviteListCommand()); + invite.addCommand(createSpaceInviteRevokeCommand()); + return invite; +} + +export function createSpaceCommand(): Command { + const space = new Command("space").description("manage spaces"); + space.addCommand(createSpaceListCommand()); + space.addCommand(createSpaceUseCommand()); + space.addCommand(createSpaceCreateCommand()); + space.addCommand(createSpaceRenameCommand()); + space.addCommand(createSpaceDeleteCommand()); + space.addCommand(createSpaceInviteCommand()); + return space; +} diff --git a/packages/cli/commands/user.ts b/packages/cli/commands/user.ts deleted file mode 100644 index dca80e3..0000000 --- a/packages/cli/commands/user.ts +++ /dev/null @@ -1,220 +0,0 @@ -/** - * me user — engine user management commands. - * - * - me user list: List users in the active engine - * - me user create : Create an engine user - * - me user get : Get user by ID or name - * - me user delete : Delete a user (by ID or name) - * - me user rename : Rename a user (by ID or name) - */ -import * as clack from "@clack/prompts"; -import { Command } from "commander"; -import { createClient } from "../client.ts"; -import { resolveCredentials } from "../credentials.ts"; -import { getOutputFormat, output, table } from "../output.ts"; -import { - handleError, - requireEngine, - requireSession, - resolveUserId, -} from "../util.ts"; - -function createUserListCommand(): Command { - return new Command("list") - .alias("ls") - .description("list users in the active engine") - .option("--login-only", "only show users that can login") - .action(async (opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const { users } = await engine.user.list( - opts.loginOnly ? { canLogin: true } : undefined, - ); - - output({ users }, fmt, () => { - if (users.length === 0) { - console.log(" No users found."); - return; - } - table( - ["id", "name", "flags"], - users.map((u) => { - const flags = [ - u.superuser ? "superuser" : "", - u.createrole ? "createrole" : "", - !u.canLogin ? "role" : "", - ] - .filter(Boolean) - .join(", "); - return [u.id, u.name, flags]; - }), - ); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserCreateCommand(): Command { - return new Command("create") - .description("create an engine user") - .argument("", "user name") - .option("--superuser", "grant superuser privileges") - .option("--createrole", "can create other users/roles") - .option("--no-login", "create as role (cannot authenticate)") - .option("--identity-id ", "link to an accounts identity") - .action(async (name: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const user = await engine.user.create({ - name, - superuser: opts.superuser ?? false, - createrole: opts.createrole ?? false, - canLogin: opts.login !== false, - identityId: opts.identityId ?? undefined, - }); - - output(user, fmt, () => { - clack.log.success(`Created user '${user.name}'`); - console.log(` ID: ${user.id}`); - console.log(` Superuser: ${user.superuser}`); - console.log(` Can Login: ${user.canLogin}`); - if (user.identityId) { - console.log(` Identity: ${user.identityId}`); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserGetCommand(): Command { - return new Command("get") - .description("get a user by ID or name") - .argument("", "user ID (UUIDv7) or name") - .action(async (idOrName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const user = await engine.user.get({ id }); - - output(user, fmt, () => { - console.log(` Name: ${user.name}`); - console.log(` ID: ${user.id}`); - console.log(` Superuser: ${user.superuser}`); - console.log(` Createrole: ${user.createrole}`); - console.log(` Can Login: ${user.canLogin}`); - console.log(` Identity: ${user.identityId ?? "(none)"}`); - console.log(` Created: ${user.createdAt}`); - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserDeleteCommand(): Command { - return new Command("delete") - .alias("rm") - .description("delete a user") - .argument("", "user ID or name") - .option("-y, --yes", "skip confirmation prompt") - .action(async (idOrName: string, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - if (fmt === "text" && !opts.yes) { - const confirmed = await clack.confirm({ - message: `Delete user ${idOrName}? This cannot be undone.`, - }); - if (clack.isCancel(confirmed) || !confirmed) { - clack.cancel("Cancelled."); - process.exit(0); - } - } - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const result = await engine.user.delete({ id }); - - output(result, fmt, () => { - if (result.deleted) { - clack.log.success("User deleted."); - } else { - clack.log.warn("User not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -function createUserRenameCommand(): Command { - return new Command("rename") - .description("rename a user") - .argument("", "user ID or name") - .argument("", "new name") - .action(async (idOrName: string, newName: string, _opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const creds = resolveCredentials(globalOpts.server); - const fmt = getOutputFormat(globalOpts); - requireSession(creds, fmt); - requireEngine(creds, fmt); - - const engine = createClient({ url: creds.server, apiKey: creds.apiKey }); - - try { - const id = await resolveUserId(engine, idOrName); - const result = await engine.user.rename({ id, name: newName }); - - output(result, fmt, () => { - if (result.renamed) { - clack.log.success(`User renamed to '${newName}'.`); - } else { - clack.log.warn("User not found."); - } - }); - } catch (error) { - handleError(error, fmt); - } - }); -} - -export function createUserCommand(): Command { - const user = new Command("user").description("manage engine users"); - user.addCommand(createUserListCommand()); - user.addCommand(createUserCreateCommand()); - user.addCommand(createUserGetCommand()); - user.addCommand(createUserDeleteCommand()); - user.addCommand(createUserRenameCommand()); - return user; -} diff --git a/packages/cli/commands/whoami.ts b/packages/cli/commands/whoami.ts index a00b54d..8230ca9 100644 --- a/packages/cli/commands/whoami.ts +++ b/packages/cli/commands/whoami.ts @@ -1,51 +1,48 @@ /** - * me whoami — show current identity and active engine. + * me whoami — show the current identity, server, and active space. */ import { Command } from "commander"; -import { createAccountsClient } from "../client.ts"; +import { createUserClient } from "../client.ts"; import { resolveCredentials } from "../credentials.ts"; import { getOutputFormat, output } from "../output.ts"; import { handleError, requireSession } from "../util.ts"; export function createWhoamiCommand(): Command { return new Command("whoami") - .description("show current identity and active engine") + .description("show current identity, server, and active space") .action(async (_opts, cmd) => { const globalOpts = cmd.optsWithGlobals(); const creds = resolveCredentials(globalOpts.server); const fmt = getOutputFormat(globalOpts); requireSession(creds, fmt); - const accounts = createAccountsClient({ + const user = createUserClient({ url: creds.server, - sessionToken: creds.sessionToken, + token: creds.sessionToken, }); try { - const identity = await accounts.me.get(); + const identity = await user.whoami(); - const data: Record = { - server: creds.server, - identity: { - id: identity.id, - name: identity.name, - email: identity.email, + output( + { + server: creds.server, + identity, + activeSpace: creds.activeSpace ?? null, }, - activeEngine: creds.activeEngine ?? null, - hasApiKey: !!creds.apiKey, - }; - - output(data, fmt, () => { - console.log(` Name: ${identity.name}`); - console.log(` Email: ${identity.email}`); - console.log(` ID: ${identity.id}`); - console.log(` Server: ${creds.server}`); - if (creds.activeEngine) { - console.log(` Engine: ${creds.activeEngine}`); - } else { - console.log(" Engine: (none — run 'me engine use' to select)"); - } - }); + fmt, + () => { + console.log(` Name: ${identity.name}`); + console.log(` Email: ${identity.email}`); + console.log(` ID: ${identity.id}`); + console.log(` Server: ${creds.server}`); + if (creds.activeSpace) { + console.log(` Space: ${creds.activeSpace}`); + } else { + console.log(" Space: (none — run 'me space use ')"); + } + }, + ); } catch (error) { handleError(error, fmt, { sessionServer: creds.server }); } diff --git a/packages/cli/credentials.test.ts b/packages/cli/credentials.test.ts new file mode 100644 index 0000000..1a19e3a --- /dev/null +++ b/packages/cli/credentials.test.ts @@ -0,0 +1,153 @@ +/** + * Credential storage tests — the file-fallback path. + * + * Forces the 0600-file fallback (ME_NO_KEYCHAIN) and an isolated XDG config dir + * so the behavior is deterministic across platforms. The OS keychain backend is + * exercised separately in keychain.test.ts. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as creds from "./credentials.ts"; +import { resetKeychainForTests } from "./keychain.ts"; + +const SERVER = "https://api.example.com"; +const TOKEN_ENVS = ["ME_SESSION_TOKEN", "ME_SPACE", "ME_SERVER", "ME_API_KEY"]; +// Every env key these tests touch — snapshotted and restored so the ambient +// environment (and other test files in the same process) is left untouched. +const ENV_KEYS = [...TOKEN_ENVS, "XDG_CONFIG_HOME", "ME_NO_KEYCHAIN"]; + +let configDir: string; +let savedEnv: Record; + +beforeEach(() => { + savedEnv = {}; + for (const k of ENV_KEYS) savedEnv[k] = process.env[k]; + + configDir = mkdtempSync(join(tmpdir(), "me-creds-")); + process.env.XDG_CONFIG_HOME = configDir; + process.env.ME_NO_KEYCHAIN = "1"; // force the file fallback + for (const k of TOKEN_ENVS) delete process.env[k]; + resetKeychainForTests(); +}); + +afterEach(() => { + rmSync(configDir, { recursive: true, force: true }); + for (const k of ENV_KEYS) { + const v = savedEnv[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + resetKeychainForTests(); +}); + +test("store + resolve a session token (file fallback)", () => { + creds.storeSessionToken(SERVER, "tok-123"); + const r = creds.resolveCredentials(SERVER); + expect(r.server).toBe(SERVER); + expect(r.sessionToken).toBe("tok-123"); + // fallback stores the token in the secrets file (no keychain) + expect(creds.getServerSecrets(SERVER).session_token).toBe("tok-123"); +}); + +test("the credentials file is written 0600", () => { + creds.storeSessionToken(SERVER, "tok"); + const file = join(configDir, "me", "credentials.yaml"); + expect(existsSync(file)).toBe(true); + // low 9 permission bits = rw------- (0o600) + expect(statSync(file).mode & 0o777).toBe(0o600); + // sanity: the token is actually in the file in fallback mode + expect(readFileSync(file, "utf-8")).toContain("tok"); +}); + +test("clearSessionToken removes the token", () => { + creds.storeSessionToken(SERVER, "tok-123"); + creds.clearSessionToken(SERVER); + expect(creds.resolveCredentials(SERVER).sessionToken).toBeUndefined(); +}); + +test("ME_SESSION_TOKEN env overrides the stored token", () => { + creds.storeSessionToken(SERVER, "stored"); + process.env.ME_SESSION_TOKEN = "from-env"; + expect(creds.resolveCredentials(SERVER).sessionToken).toBe("from-env"); +}); + +test("active space: set / resolve / clear; ME_SPACE wins", () => { + creds.setActiveSpace(SERVER, "abc123def456"); + expect(creds.resolveCredentials(SERVER).activeSpace).toBe("abc123def456"); + + process.env.ME_SPACE = "envspace0001"; + expect(creds.resolveCredentials(SERVER).activeSpace).toBe("envspace0001"); + delete process.env.ME_SPACE; + + creds.clearActiveSpace(SERVER); + expect(creds.resolveCredentials(SERVER).activeSpace).toBeUndefined(); +}); + +test("logout clears the secret but keeps the active space", () => { + creds.storeSessionToken(SERVER, "tok"); + creds.setActiveSpace(SERVER, "abc123def456"); + creds.clearServerCredentials(SERVER); // logout + const r = creds.resolveCredentials(SERVER); + expect(r.sessionToken).toBeUndefined(); + expect(r.activeSpace).toBe("abc123def456"); // non-secret config survives logout +}); + +test("secrets and config live in separate files", () => { + creds.storeSessionToken(SERVER, "tok-sep"); + creds.setActiveSpace(SERVER, "abc123def456"); + const configFile = readFileSync( + join(configDir, "me", "config.yaml"), + "utf-8", + ); + const credsFile = readFileSync( + join(configDir, "me", "credentials.yaml"), + "utf-8", + ); + // config.yaml has the active space (non-secret), not the token + expect(configFile).toContain("abc123def456"); + expect(configFile).not.toContain("tok-sep"); + // credentials.yaml has the token (fallback), not the active space + expect(credsFile).toContain("tok-sep"); + expect(credsFile).not.toContain("abc123def456"); +}); + +test("migrates a legacy credentials.yaml (token + active_space + default)", () => { + // a pre-split credentials.yaml that bundled everything together + const dir = join(configDir, "me"); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + writeFileSync( + join(dir, "credentials.yaml"), + [ + `default_server: ${SERVER}`, + "servers:", + ` ${SERVER}:`, + " session_token: legacy-tok", + " active_space: legacyspace1", + ].join("\n"), + { mode: 0o600 }, + ); + + // reading resolves all three, migrating the non-secret bits out + const r = creds.resolveCredentials(); + expect(r.server).toBe(SERVER); + expect(r.sessionToken).toBe("legacy-tok"); + expect(r.activeSpace).toBe("legacyspace1"); + + // config.yaml now exists with the non-secret bits; credentials.yaml is + // secret-only (no active_space left behind) + const configFile = readFileSync(join(dir, "config.yaml"), "utf-8"); + expect(configFile).toContain("legacyspace1"); + const credsFile = readFileSync(join(dir, "credentials.yaml"), "utf-8"); + expect(credsFile).toContain("legacy-tok"); + expect(credsFile).not.toContain("legacyspace1"); +}); diff --git a/packages/cli/credentials.ts b/packages/cli/credentials.ts index a45916a..cef8e13 100644 --- a/packages/cli/credentials.ts +++ b/packages/cli/credentials.ts @@ -1,89 +1,93 @@ /** - * Credential storage — multi-server, multi-engine credential management. + * Credential + config storage — multi-server. * - * Stores session tokens and per-engine API keys in - * $XDG_CONFIG_HOME/me/credentials.yaml (default: ~/.config/me/). + * Two files under $XDG_CONFIG_HOME/me (default ~/.config/me): + * - config.yaml — non-secret: the default server + each server's active + * space (the X-Me-Space). + * - credentials.yaml — 0600, secrets only: the session-token fallback, used + * when no OS keychain is available (see ./keychain.ts); + * empty / absent on hosts with a keychain. * - * File format: + * The session token (the one secret) prefers the OS keychain; the file is the + * fallback. Api keys are never stored — agents get their key via `ME_API_KEY` + * (or their MCP config); `apiKey.create` prints it once. + * + * config.yaml: * ```yaml * default_server: https://api.memory.build * servers: * https://api.memory.build: - * session_token: "..." - * active_engine: "abc123defg45" - * engines: - * abc123defg45: - * api_key: "me.abc123defg45.xxxx.yyyy" - * xyz789qwer12: - * api_key: "me.xyz789qwer12.xxxx.yyyy" + * active_space: abc123def456 * ``` + * credentials.yaml (0600): + * ```yaml + * servers: + * https://api.memory.build: + * session_token: "..." # only when there's no keychain + * ``` + * + * A pre-split credentials.yaml (which once held default_server + active_space + * next to the token) is migrated to this layout on first read. */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { parse, stringify } from "yaml"; +import { keychainDelete, keychainGet, keychainSet } from "./keychain.ts"; // ============================================================================= -// Constants +// Constants & types // ============================================================================= export const DEFAULT_SERVER = "https://api.memory.build"; -// ============================================================================= -// Types -// ============================================================================= +/** Per-server non-secret config. */ +export interface ServerConfig { + /** Active space slug (the X-Me-Space). */ + active_space?: string; +} -/** - * Per-engine credential entry. - */ -export interface EngineCredentials { - api_key: string; +/** config.yaml structure. */ +export interface ConfigFile { + default_server: string; + servers: Record; } -/** - * Per-server credential entry. - */ -export interface ServerCredentials { +/** Per-server secrets — the keychain-free fallback. */ +export interface ServerSecrets { session_token?: string; - active_engine?: string; - engines?: Record; } -/** - * Full credentials file structure. - */ +/** credentials.yaml structure (secrets only). */ export interface CredentialsFile { - default_server: string; - servers: Record; + servers: Record; } -/** - * Resolved credentials for a specific server. - */ +/** Resolved credentials for a specific server. */ export interface ResolvedCredentials { server: string; sessionToken?: string; + /** Agent api key — ME_API_KEY only; never persisted. */ apiKey?: string; - activeEngine?: string; + /** Active space slug (the X-Me-Space) — ME_SPACE env > stored active_space. */ + activeSpace?: string; } // ============================================================================= // Path Helpers // ============================================================================= -/** - * Get the config directory path. - * Respects $XDG_CONFIG_HOME, defaults to ~/.config/me. - */ +/** Config directory — respects $XDG_CONFIG_HOME, defaults to ~/.config/me. */ function getConfigDir(): string { const xdg = process.env.XDG_CONFIG_HOME; const base = xdg || join(homedir(), ".config"); return join(base, "me"); } -/** - * Get the credentials file path. - */ +function getConfigPath(): string { + return join(getConfigDir(), "config.yaml"); +} + function getCredentialsPath(): string { return join(getConfigDir(), "credentials.yaml"); } @@ -103,17 +107,14 @@ export function normalizeOrigin(url: string): string { } try { const parsed = new URL(url); - // Remove default ports if ( (parsed.protocol === "https:" && parsed.port === "443") || (parsed.protocol === "http:" && parsed.port === "80") ) { parsed.port = ""; } - // Return origin (scheme + host + port, no trailing slash) return parsed.origin; } catch { - // If URL parsing fails, return as-is with trailing slash stripped return url.replace(/\/+$/, ""); } } @@ -122,168 +123,207 @@ export function normalizeOrigin(url: string): string { // Read / Write // ============================================================================= -/** - * Read the credentials file. Returns empty structure if file doesn't exist. - */ -export function readCredentials(): CredentialsFile { - const path = getCredentialsPath(); - if (!existsSync(path)) { - return { - default_server: DEFAULT_SERVER, - servers: {}, - }; - } +function ensureDir(): void { + const dir = getConfigDir(); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); +} +/** Read config.yaml (non-secret). Empty structure if absent / unparseable. */ +function readConfig(): ConfigFile { + migrateLegacyIfNeeded(); + const path = getConfigPath(); + if (!existsSync(path)) return { default_server: DEFAULT_SERVER, servers: {} }; try { - const content = readFileSync(path, "utf-8"); - const data = parse(content) as Partial | null; + const data = parse( + readFileSync(path, "utf-8"), + ) as Partial | null; return { default_server: data?.default_server ?? DEFAULT_SERVER, servers: data?.servers ?? {}, }; } catch { - return { - default_server: DEFAULT_SERVER, - servers: {}, - }; + return { default_server: DEFAULT_SERVER, servers: {} }; } } -/** - * Write the credentials file atomically with secure permissions. - * Creates the config directory if it doesn't exist. - */ -export function writeCredentials(creds: CredentialsFile): void { - const dir = getConfigDir(); - const path = getCredentialsPath(); +/** Write config.yaml. Non-secret, but the dir is 0700 (owner-only). */ +function writeConfig(config: ConfigFile): void { + ensureDir(); + writeFileSync(getConfigPath(), stringify(config, { lineWidth: 0 })); +} - // Create config directory with 0700 (owner-only) - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true, mode: 0o700 }); +/** Read credentials.yaml (secrets). Empty structure if absent / unparseable. */ +function readSecrets(): CredentialsFile { + migrateLegacyIfNeeded(); + const path = getCredentialsPath(); + if (!existsSync(path)) return { servers: {} }; + try { + const data = parse( + readFileSync(path, "utf-8"), + ) as Partial | null; + return { servers: data?.servers ?? {} }; + } catch { + return { servers: {} }; } - - const content = stringify(creds, { lineWidth: 0 }); - - // Write with 0600 (owner read/write only) - writeFileSync(path, content, { mode: 0o600 }); } -// ============================================================================= -// Server Credential Operations -// ============================================================================= - -/** - * Get credentials for a specific server. - */ -export function getServerCredentials(server: string): ServerCredentials { - const creds = readCredentials(); - const origin = normalizeOrigin(server); - return creds.servers[origin] ?? {}; +/** Write credentials.yaml with 0600 (owner read/write only). */ +function writeSecrets(secrets: CredentialsFile): void { + ensureDir(); + writeFileSync(getCredentialsPath(), stringify(secrets, { lineWidth: 0 }), { + mode: 0o600, + }); } /** - * Store a session token for a server. - * Also sets this server as the default. + * One-time split of a pre-split credentials.yaml — which used to hold + * default_server + per-server active_space alongside the token — into config.yaml + * (non-secret) + a secret-only credentials.yaml. A no-op once config.yaml exists, + * and when there's nothing legacy to move. */ -export function storeSessionToken(server: string, token: string): void { - const creds = readCredentials(); - const origin = normalizeOrigin(server); - - if (!creds.servers[origin]) { - creds.servers[origin] = {}; +function migrateLegacyIfNeeded(): void { + if (existsSync(getConfigPath()) || !existsSync(getCredentialsPath())) return; + + let legacy: { + default_server?: unknown; + servers?: Record< + string, + { session_token?: unknown; active_space?: unknown } + >; + } | null; + try { + legacy = parse(readFileSync(getCredentialsPath(), "utf-8")); + } catch { + return; } - creds.servers[origin].session_token = token; - creds.default_server = origin; + if (!legacy || typeof legacy !== "object") return; + + const config: ConfigFile = { + default_server: + typeof legacy.default_server === "string" + ? legacy.default_server + : DEFAULT_SERVER, + servers: {}, + }; + const secrets: CredentialsFile = { servers: {} }; + let sawLegacy = typeof legacy.default_server === "string"; + for (const [origin, entry] of Object.entries(legacy.servers ?? {})) { + if (typeof entry?.active_space === "string") { + config.servers[origin] = { active_space: entry.active_space }; + sawLegacy = true; + } + if (typeof entry?.session_token === "string") { + secrets.servers[origin] = { session_token: entry.session_token }; + } + } + if (!sawLegacy) return; // already secret-only — nothing to migrate - writeCredentials(creds); + writeConfig(config); + writeSecrets(secrets); } -/** - * Store an API key for an engine on a server. - * Also sets the engine as active. - */ -export function storeApiKey( - server: string, - engineSlug: string, - apiKey: string, -): void { - const creds = readCredentials(); - const origin = normalizeOrigin(server); +// ============================================================================= +// Per-server accessors +// ============================================================================= - if (!creds.servers[origin]) { - creds.servers[origin] = {}; - } - if (!creds.servers[origin].engines) { - creds.servers[origin].engines = {}; - } - creds.servers[origin].engines[engineSlug] = { api_key: apiKey }; - creds.servers[origin].active_engine = engineSlug; +/** Non-secret config for a server (active space). */ +export function getServerConfig(server: string): ServerConfig { + return readConfig().servers[normalizeOrigin(server)] ?? {}; +} - writeCredentials(creds); +/** Secrets for a server (the keychain-free session-token fallback). */ +export function getServerSecrets(server: string): ServerSecrets { + return readSecrets().servers[normalizeOrigin(server)] ?? {}; } +// ============================================================================= +// Session token +// ============================================================================= + /** - * Set the active engine for a server (without modifying API keys). + * Store a session token for a server, and record it as the default server. + * Prefers the OS keychain; only when that's unavailable does the token land in + * the 0600 credentials file (and any stale file copy is dropped once the + * keychain has it). The default server is non-secret config (config.yaml). */ -export function setActiveEngine(server: string, engineSlug: string): void { - const creds = readCredentials(); +export function storeSessionToken(server: string, token: string): void { const origin = normalizeOrigin(server); - if (!creds.servers[origin]) { - creds.servers[origin] = {}; + const secrets = readSecrets(); + if (keychainSet(origin, token)) { + if (secrets.servers[origin]) { + delete secrets.servers[origin]; // keychain is the source of truth + writeSecrets(secrets); + } + } else { + secrets.servers[origin] = { session_token: token }; + writeSecrets(secrets); } - creds.servers[origin].active_engine = engineSlug; - writeCredentials(creds); + const config = readConfig(); + config.default_server = origin; + writeConfig(config); } /** - * Get the API key for a specific engine on a server. - */ -export function getEngineApiKey( - server: string, - engineSlug: string, -): string | undefined { - const stored = getServerCredentials(server); - return stored.engines?.[engineSlug]?.api_key; -} - -/** - * Clear just the session token for a server, leaving any stored engines and - * API keys in place. Used after the server tells us the session is expired so - * the next command surfaces "Not logged in" instead of repeating the 401. - * - * No-op if no credentials are stored for the server, or if the token came + * Clear a server's session token from both the keychain and the file. Keeps + * non-secret config (active space, default server). No-op for a token that came * from $ME_SESSION_TOKEN (we can't unset an env var the user controls). */ export function clearSessionToken(server: string): void { - const creds = readCredentials(); const origin = normalizeOrigin(server); + keychainDelete(origin); - const entry = creds.servers[origin]; - if (!entry?.session_token) { - return; + const secrets = readSecrets(); + if (secrets.servers[origin]) { + delete secrets.servers[origin]; + writeSecrets(secrets); } - delete entry.session_token; - - writeCredentials(creds); } /** - * Clear all credentials for a server. + * Log out of a server: clear its session secret (keychain + file) but keep the + * non-secret config (active space, default server) so a re-login resumes where + * you left off. */ export function clearServerCredentials(server: string): void { - const creds = readCredentials(); - const origin = normalizeOrigin(server); + clearSessionToken(server); +} - delete creds.servers[origin]; +// ============================================================================= +// Active space (config) +// ============================================================================= - // If we just cleared the default server, reset to default - if (creds.default_server === origin) { - creds.default_server = DEFAULT_SERVER; - } +/** Set the active space (the X-Me-Space) for a server. */ +export function setActiveSpace(server: string, spaceSlug: string): void { + const config = readConfig(); + const origin = normalizeOrigin(server); + if (!config.servers[origin]) config.servers[origin] = {}; + config.servers[origin].active_space = spaceSlug; + writeConfig(config); +} - writeCredentials(creds); +/** Clear the active space for a server (e.g. after deleting it). No-op if unset. */ +export function clearActiveSpace(server: string): void { + const config = readConfig(); + const origin = normalizeOrigin(server); + const entry = config.servers[origin]; + if (!entry?.active_space) return; + delete entry.active_space; + writeConfig(config); +} + +/** + * Resolve the active space slug for a server. + * Priority: --space flag > ME_SPACE env > stored active_space. + */ +export function resolveSpace( + server: string, + flagValue?: string, +): string | undefined { + if (flagValue) return flagValue; + if (process.env.ME_SPACE) return process.env.ME_SPACE; + return getServerConfig(server).active_space; } // ============================================================================= @@ -292,37 +332,36 @@ export function clearServerCredentials(server: string): void { /** * Resolve the active server URL. - * - * Priority: --server flag > ME_SERVER env > default_server in creds > DEFAULT_SERVER + * Priority: --server flag > ME_SERVER env > default_server (config) > DEFAULT_SERVER */ export function resolveServer(flagValue?: string): string { if (flagValue) return normalizeOrigin(flagValue); if (process.env.ME_SERVER) return normalizeOrigin(process.env.ME_SERVER); - - const creds = readCredentials(); - return creds.default_server; + return readConfig().default_server; } /** - * Resolve all credentials for the active server. - * - * For each credential type, env vars take priority over the stored file. - * API key is resolved from the active engine's stored key. + * Resolve all credentials for the active server. The session token + * (ME_SESSION_TOKEN env > file > keychain) authenticates humans; the active + * space (ME_SPACE env > config) is the X-Me-Space. An agent api key is never + * persisted — it only ever comes from ME_API_KEY. */ export function resolveCredentials(serverFlag?: string): ResolvedCredentials { const server = resolveServer(serverFlag); - const stored = getServerCredentials(server); - - // Resolve API key: env var > active engine's stored key - const activeEngine = stored.active_engine; - const storedApiKey = activeEngine - ? stored.engines?.[activeEngine]?.api_key - : undefined; + const origin = normalizeOrigin(server); + const config = getServerConfig(server); + const secrets = getServerSecrets(server); return { server, - sessionToken: process.env.ME_SESSION_TOKEN ?? stored.session_token, - apiKey: process.env.ME_API_KEY ?? storedApiKey, - activeEngine, + // env wins; then the file (keychain-free fallback); then the keychain. The + // token lives in exactly one of file/keychain, so checking the file first + // avoids a keychain lookup on hosts that use the file fallback. + sessionToken: + process.env.ME_SESSION_TOKEN ?? + secrets.session_token ?? + keychainGet(origin), + apiKey: process.env.ME_API_KEY, + activeSpace: process.env.ME_SPACE ?? config.active_space, }; } diff --git a/packages/cli/importers/claude.test.ts b/packages/cli/importers/claude.test.ts index 554d5eb..15bcf2c 100644 --- a/packages/cli/importers/claude.test.ts +++ b/packages/cli/importers/claude.test.ts @@ -9,6 +9,7 @@ import { describe, expect, test } from "bun:test"; import { join } from "node:path"; import { claudeImporter, + encodeProjectDir, sanitizeUserText, unwrapSdkReplayBundle, } from "./claude.ts"; @@ -59,6 +60,50 @@ function userTextsOf(messages: ConversationMessage[]): string[] { ); } +describe("encodeProjectDir", () => { + test("encodes a cwd the way Claude Code names project dirs", () => { + expect(encodeProjectDir("/Users/test/project")).toBe("-Users-test-project"); + // Trailing slashes are ignored; all non-alphanumerics become dashes. + expect(encodeProjectDir("/Users/x/my.app/")).toBe("-Users-x-my-app"); + expect(encodeProjectDir("/Users/x/my_app")).toBe("-Users-x-my-app"); + }); +}); + +describe("claude importer project-dir pruning", () => { + test("a matching --project keeps the project's directory", async () => { + const { sessions, stats } = await collect( + baseOptions({ projectFilter: "/Users/test/project" }), + ); + expect(stats.totalFiles).toBeGreaterThan(0); + expect(sessions.length).toBeGreaterThan(0); + }); + + test("an ancestor --project keeps descendant project directories", async () => { + const { sessions } = await collect( + baseOptions({ projectFilter: "/Users/test" }), + ); + expect(sessions.length).toBeGreaterThan(0); + }); + + test("a foreign --project never scans other projects' files", async () => { + const { sessions, stats } = await collect( + baseOptions({ projectFilter: "/Users/other/project" }), + ); + // Pruned at the directory level: zero files scanned, not scanned-then-skipped. + expect(stats.totalFiles).toBe(0); + expect(sessions).toEqual([]); + }); + + test("an encoded-prefix collision that is not a path ancestor is pruned", async () => { + // "/Users/test/proj" is a string prefix of the project but not an + // ancestor directory — the `${encoded}-` boundary must reject it. + const { stats } = await collect( + baseOptions({ projectFilter: "/Users/test/proj" }), + ); + expect(stats.totalFiles).toBe(0); + }); +}); + describe("claude importer", () => { test("skips sidechains by default", async () => { const { sessions, stats } = await collect(baseOptions()); diff --git a/packages/cli/importers/claude.ts b/packages/cli/importers/claude.ts index 6658dca..eddf944 100644 --- a/packages/cli/importers/claude.ts +++ b/packages/cli/importers/claude.ts @@ -14,7 +14,7 @@ */ import { promises as fs } from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { filterBySessionShape, recordSkip } from "./filters.ts"; import type { Importer } from "./index.ts"; import type { ProgressReporter } from "./progress.ts"; @@ -77,6 +77,8 @@ export const claudeImporter: Importer = { tool: "claude", defaultSource: DEFAULT_SOURCE, discoverSessions, + // Single-file parse for the live capture hook (importTranscriptFile). + parseFile: parseSessionFile, }; async function* discoverSessions( @@ -93,6 +95,18 @@ async function* discoverSessions( return; } + // With a --project filter, skip other projects' directories outright: the + // directory name encodes the session cwd, so most of the machine's + // transcripts never need to be opened. The encoding is lossy, so this only + // prunes — kept files still pass the exact per-session cwd filter below. + if (options.projectFilter) { + const encoded = encodeProjectDir(options.projectFilter); + projectDirs = projectDirs.filter((dir) => { + const name = basename(dir); + return name === encoded || name.startsWith(`${encoded}-`); + }); + } + for (const projectDir of projectDirs) { const files = await listJsonlFiles(projectDir); for (const file of files) { @@ -145,6 +159,16 @@ function countUserMessages(messages: ConversationMessage[]): number { } /** List immediate subdirectories of `path`. */ +/** + * Encode a cwd the way Claude Code names its per-project transcript + * directories: every non-alphanumeric character becomes `-` (e.g. + * /Users/x/my.app → -Users-x-my-app). Lossy (a literal `-` and a `/` encode + * identically), so matches are candidates, never authoritative. + */ +export function encodeProjectDir(cwd: string): string { + return cwd.replace(/\/+$/, "").replace(/[^a-zA-Z0-9]/g, "-"); +} + async function listSubdirs(path: string): Promise { const entries = await fs.readdir(path, { withFileTypes: true }); return entries diff --git a/packages/cli/importers/git.test.ts b/packages/cli/importers/git.test.ts new file mode 100644 index 0000000..e2cea99 --- /dev/null +++ b/packages/cli/importers/git.test.ts @@ -0,0 +1,287 @@ +/** + * Tests for the git history importer: the streaming `git log` parser and + * the per-commit memory builder. Fixture strings mirror the byte layout + * git actually emits for + * `--numstat --pretty=format:%x01%H%x00…%B%x00` (verified empirically): + * + * \x01\0\0\0\0\0\0\0\n\n\n + * + * Merge commits emit no numstat lines; a root commit has an empty parents + * field; the final record ends at EOF without trailing separators. + */ +import { describe, expect, test } from "bun:test"; +import { + BODY_BYTES_CAP, + buildCommitMemory, + type CommitMemoryContext, + FILE_LIST_CAP, + type GitCommit, + GitLogParser, + mergeSkipReason, +} from "./git.ts"; + +const SHA_A = "a".repeat(40); +const SHA_B = "b".repeat(40); +const SHA_C = "c".repeat(40); + +/** Build one raw log record in the observed wire layout. */ +function rec(opts: { + sha?: string; + authorName?: string; + authorEmail?: string; + authorDate?: string; + commitDate?: string; + parents?: string; + body?: string; + numstat?: string[]; +}): string { + const fields = [ + opts.sha ?? SHA_A, + opts.authorName ?? "Ada", + opts.authorEmail ?? "ada@example.com", + opts.authorDate ?? "2026-01-02T03:04:05+02:00", + opts.commitDate ?? "2026-01-02T03:04:06+02:00", + opts.parents ?? SHA_B, + opts.body ?? "subject line\n", + ].join("\x00"); + const tail = + opts.numstat && opts.numstat.length > 0 + ? `\n${opts.numstat.join("\n")}\n\n` + : "\n"; + return `\x01${fields}\x00${tail}`; +} + +/** Parse a full log text in one push + end. */ +function parseAll(text: string): GitCommit[] { + const parser = new GitLogParser(); + return [...parser.push(text), ...parser.end()]; +} + +describe("GitLogParser", () => { + test("parses a single commit with files", () => { + const commits = parseAll( + rec({ + body: "fix: a thing\n\nlonger explanation\nover two lines\n", + numstat: ["10\t2\tsrc/a.ts", "0\t5\tsrc/b.ts"], + }), + ); + expect(commits).toHaveLength(1); + const c = commits[0]; + expect(c?.sha).toBe(SHA_A); + expect(c?.authorName).toBe("Ada"); + expect(c?.authorEmail).toBe("ada@example.com"); + expect(c?.authorDate).toBe("2026-01-02T03:04:05+02:00"); + expect(c?.commitDate).toBe("2026-01-02T03:04:06+02:00"); + expect(c?.parents).toEqual([SHA_B]); + expect(c?.subject).toBe("fix: a thing"); + expect(c?.body).toBe("longer explanation\nover two lines"); + expect(c?.files).toEqual([ + { path: "src/a.ts", insertions: 10, deletions: 2 }, + { path: "src/b.ts", insertions: 0, deletions: 5 }, + ]); + }); + + test("parses multiple records, including a final record at EOF", () => { + const commits = parseAll( + rec({ sha: SHA_A, numstat: ["1\t0\ta.txt"] }) + + rec({ sha: SHA_B, parents: "", body: "first\n" }), + ); + expect(commits.map((c) => c.sha)).toEqual([SHA_A, SHA_B]); + // Root commit: empty parents field → no parents. + expect(commits[1]?.parents).toEqual([]); + }); + + test("merge commits carry two parents and no files", () => { + const commits = parseAll( + rec({ parents: `${SHA_B} ${SHA_C}`, body: "Merge branch 'x'\n" }), + ); + expect(commits[0]?.parents).toEqual([SHA_B, SHA_C]); + expect(commits[0]?.files).toEqual([]); + }); + + test("handles rename and binary numstat lines", () => { + const commits = parseAll( + rec({ + numstat: ["3\t1\tsrc/{old => new}/mod.ts", "-\t-\tassets/logo.png"], + }), + ); + expect(commits[0]?.files).toEqual([ + { path: "src/{old => new}/mod.ts", insertions: 3, deletions: 1 }, + { path: "assets/logo.png", insertions: null, deletions: null }, + ]); + }); + + test("a \\x01 inside a message body does not start a new record", () => { + const commits = parseAll( + rec({ sha: SHA_A, body: "subject\n\nweird \x01 control char\n" }) + + rec({ sha: SHA_B }), + ); + expect(commits).toHaveLength(2); + expect(commits[0]?.body).toBe("weird \x01 control char"); + }); + + test("reassembles records split across arbitrary chunk boundaries", () => { + const full = + rec({ sha: SHA_A, numstat: ["1\t0\ta.txt"] }) + + rec({ sha: SHA_B, body: "two\n" }) + + rec({ sha: SHA_C, parents: "", body: "three\n" }); + // Split mid-header, mid-field, and mid-numstat to stress the buffer. + for (const chunkSize of [1, 7, 41, 64]) { + const parser = new GitLogParser(); + const commits: GitCommit[] = []; + for (let i = 0; i < full.length; i += chunkSize) { + commits.push(...parser.push(full.slice(i, i + chunkSize))); + } + commits.push(...parser.end()); + expect(commits.map((c) => c.sha)).toEqual([SHA_A, SHA_B, SHA_C]); + } + }); + + test("empty input yields nothing", () => { + const parser = new GitLogParser(); + expect(parser.push("")).toEqual([]); + expect(parser.end()).toEqual([]); + }); + + test("throws on a malformed record", () => { + expect(() => parseAll("\x01not-a-sha\x00rest")).toThrow(/malformed/); + }); +}); + +/** A parsed commit for builder tests. */ +function commit(overrides: Partial = {}): GitCommit { + return { + sha: SHA_A, + authorName: "Ada", + authorEmail: "ada@example.com", + authorDate: "2026-01-02T03:04:05+02:00", + commitDate: "2026-01-02T03:04:06+02:00", + parents: [SHA_B], + subject: "fix: a thing", + body: "details here", + files: [{ path: "src/a.ts", insertions: 10, deletions: 2 }], + ...overrides, + }; +} + +function ctx( + overrides: Partial = {}, +): CommitMemoryContext { + return { + tree: "share.projects.demo.git_history", + projectSlug: "demo", + gitRemote: "git@github.com:org/demo.git", + fileList: true, + importedAt: "2026-06-10T00:00:00.000Z", + ...overrides, + }; +} + +describe("buildCommitMemory", () => { + test("builds content, meta, temporal, and a deterministic id", () => { + const built = buildCommitMemory(commit(), ctx()); + if ("error" in built) throw new Error(built.error); + expect(built.content).toBe( + "fix: a thing\n\ndetails here\n\nFiles:\n src/a.ts (+10 -2)", + ); + expect(built.tree).toBe("share.projects.demo.git_history"); + // Commit date, normalized to UTC. + expect(built.temporal).toEqual({ start: "2026-01-02T01:04:06.000Z" }); + expect(built.meta).toEqual({ + type: "git_commit", + sha: SHA_A, + source_project_slug: "demo", + source_git_repo: "git@github.com:org/demo.git", + author_name: "Ada", + author_email: "ada@example.com", + author_date: "2026-01-02T03:04:05+02:00", + commit_date: "2026-01-02T03:04:06+02:00", + files_changed: 1, + insertions: 10, + deletions: 2, + imported_at: "2026-06-10T00:00:00.000Z", + importer_version: "1", + }); + + // Deterministic: same inputs → same id; different tree → different id. + const again = buildCommitMemory(commit(), ctx()); + if ("error" in again) throw new Error(again.error); + expect(again.id).toBe(built.id); + const moved = buildCommitMemory(commit(), ctx({ tree: "share.other" })); + if ("error" in moved) throw new Error(moved.error); + expect(moved.id).not.toBe(built.id); + }); + + test("omits remote and merge marker when absent, sets them when present", () => { + const plain = buildCommitMemory(commit(), ctx({ gitRemote: undefined })); + if ("error" in plain) throw new Error(plain.error); + expect(plain.meta).not.toContainKey("source_git_repo"); + expect(plain.meta).not.toContainKey("is_merge"); + + const merge = buildCommitMemory(commit({ parents: [SHA_B, SHA_C] }), ctx()); + if ("error" in merge) throw new Error(merge.error); + expect(merge.meta?.is_merge).toBe(true); + }); + + test("renders binary files and caps the file list", () => { + const files = Array.from({ length: FILE_LIST_CAP + 3 }, (_, i) => ({ + path: `f${i}.ts`, + insertions: 1, + deletions: 0, + })); + files[0] = { path: "img.png", insertions: null, deletions: null } as never; + const built = buildCommitMemory(commit({ files }), ctx()); + if ("error" in built) throw new Error(built.error); + expect(built.content).toContain(" img.png (binary)"); + expect(built.content).toContain(` … and 3 more files`); + // Binary files don't contribute to line counts. + expect(built.meta?.insertions).toBe(FILE_LIST_CAP + 2); + expect(built.meta?.files_changed).toBe(FILE_LIST_CAP + 3); + }); + + test("omits the file list when disabled", () => { + const built = buildCommitMemory(commit(), ctx({ fileList: false })); + if ("error" in built) throw new Error(built.error); + expect(built.content).toBe("fix: a thing\n\ndetails here"); + }); + + test("truncates oversized bodies on a byte budget", () => { + const built = buildCommitMemory( + commit({ body: "x".repeat(BODY_BYTES_CAP + 1000), files: [] }), + ctx(), + ); + if ("error" in built) throw new Error(built.error); + expect(built.content).toContain("…[truncated]"); + expect(Buffer.byteLength(built.content, "utf8")).toBeLessThan( + BODY_BYTES_CAP + 200, + ); + }); + + test("reports an error for an unparsable commit date", () => { + const built = buildCommitMemory( + commit({ commitDate: "not-a-date" }), + ctx(), + ); + expect(built).toEqual({ error: "invalid commit date: not-a-date" }); + }); +}); + +describe("mergeSkipReason", () => { + test("non-merge commits are never skipped", () => { + expect(mergeSkipReason(commit({ body: "" }))).toBeNull(); + }); + + test("body-less merges are boilerplate", () => { + expect(mergeSkipReason(commit({ parents: [SHA_B, SHA_C], body: "" }))).toBe( + "merge_boilerplate", + ); + }); + + test("merges with a body (PR merges) are kept", () => { + expect( + mergeSkipReason( + commit({ parents: [SHA_B, SHA_C], body: "Fix auth refresh (#42)" }), + ), + ).toBeNull(); + }); +}); diff --git a/packages/cli/importers/git.ts b/packages/cli/importers/git.ts new file mode 100644 index 0000000..c17ab85 --- /dev/null +++ b/packages/cli/importers/git.ts @@ -0,0 +1,372 @@ +/** + * Git history importer — walks `git log` and turns each commit into one + * memory under `..git_history`. + * + * Identity: a deterministic UUIDv7 keyed by `git::` with the + * commit date as the timestamp half, so re-imports collide server-side + * (`ON CONFLICT (id) DO NOTHING`) and become no-op skips — no cursor or + * client-side state. Incremental runs (see commands/import-git.ts) only + * narrow the walk; correctness never depends on them. + * + * The walk is one streamed `git log` invocation with NUL-separated fields: + * + * %x01 %H %x00 %an %x00 %ae %x00 %aI %x00 %cI %x00 %P %x00 %B %x00 + * + * followed by `--numstat` lines until the next record. Git forbids NUL in + * commit messages, so the field splits are unambiguous; records are anchored + * by `\x01` + 40-hex sha + NUL so a `\x01` inside a message body can't start + * a record. Output is streamed through an incremental parser, so repos of + * any size walk in constant memory. + */ + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; +import { deterministicUuidV7 } from "./uuid.ts"; + +const execFileAsync = promisify(execFile); + +/** Per-project tree node holding imported commits (next to agent_sessions). */ +export const GIT_HISTORY_NODE_NAME = "git_history"; + +/** + * Version tag stored in `meta.importer_version`. Reserved for a future + * re-render path (cf. IMPORTER_VERSION in importers/index.ts); for now a + * bump only marks newly-written records. + */ +export const GIT_IMPORTER_VERSION = "1"; + +/** Max file lines rendered into a commit memory's `Files:` block. */ +export const FILE_LIST_CAP = 50; + +/** Max body bytes rendered into a commit memory before truncation. */ +export const BODY_BYTES_CAP = 64 * 1024; + +/** One changed file from a `--numstat` line. */ +export interface GitFileChange { + /** Path as git prints it (renames keep the `{old => new}` form). */ + path: string; + /** Added lines, or null for binary files. */ + insertions: number | null; + /** Deleted lines, or null for binary files. */ + deletions: number | null; +} + +/** One parsed commit from the log walk. */ +export interface GitCommit { + sha: string; + authorName: string; + authorEmail: string; + /** ISO 8601 author date (%aI). */ + authorDate: string; + /** ISO 8601 committer date (%cI). */ + commitDate: string; + /** Parent shas; length >= 2 marks a merge commit. */ + parents: string[]; + /** First line of the message. */ + subject: string; + /** Message after the subject (trimmed; may be empty). */ + body: string; + files: GitFileChange[]; +} + +/** Record start marker + the number of NUL-separated fields before the tail. */ +const RECORD_START = "\x01"; +const FIELD_COUNT = 7; +/** `\x01` + 40-hex sha + NUL — what a genuine record header looks like. */ +// biome-ignore lint/suspicious/noControlCharactersInRegex: the log wire format uses \x01/\x00 separators by design +const RECORD_HEADER_RE = /^\x01[0-9a-f]{40}\x00/; +/** A `--numstat` line: added/deleted counts (or `-` for binary) + path. */ +const NUMSTAT_LINE_RE = /^(\d+|-)\t(\d+|-)\t(.+)$/; + +/** + * Incremental parser for the custom `git log` format above. Feed decoded + * stdout text via `push` (returns the records completed by that chunk) and + * call `end` for the final record. Pure — unit-testable without git. + */ +export class GitLogParser { + private buf = ""; + + push(text: string): GitCommit[] { + this.buf += text; + const out: GitCommit[] = []; + // A record is complete once the NEXT record's header is visible. + for (;;) { + const next = findNextHeader(this.buf, 1); + if (next === -1) break; + out.push(parseRecord(this.buf.slice(0, next))); + this.buf = this.buf.slice(next); + } + return out; + } + + end(): GitCommit[] { + const rest = this.buf; + this.buf = ""; + if (rest.trim().length === 0) return []; + return [parseRecord(rest)]; + } +} + +/** Find the next genuine record header at or after `from` (-1 if none). */ +function findNextHeader(buf: string, from: number): number { + let i = from; + for (;;) { + const at = buf.indexOf(RECORD_START, i); + if (at === -1) return -1; + if (RECORD_HEADER_RE.test(buf.slice(at, at + 42))) return at; + i = at + 1; + } +} + +/** Parse one complete record (from its `\x01` up to the next header). */ +function parseRecord(record: string): GitCommit { + if (!RECORD_HEADER_RE.test(record.slice(0, 42))) { + throw new Error( + `malformed git log record: ${JSON.stringify(record.slice(0, 60))}`, + ); + } + // Split off the 7 NUL-separated fields; the remainder is the numstat tail. + const parts = record.slice(1).split("\x00"); + if (parts.length < FIELD_COUNT + 1) { + throw new Error( + `malformed git log record for ${parts[0]}: expected ${FIELD_COUNT} fields`, + ); + } + const [sha, authorName, authorEmail, authorDate, commitDate, parentsRaw] = + parts; + // The body is everything between the 6th NUL and the 7th — but a body + // cannot contain NUL, so it is exactly parts[6]. + const bodyRaw = parts[6] ?? ""; + const tail = parts.slice(FIELD_COUNT).join("\x00"); + + const files: GitFileChange[] = []; + for (const line of tail.split("\n")) { + const m = NUMSTAT_LINE_RE.exec(line); + if (!m) continue; + const [, ins, del, path] = m; + files.push({ + path: path ?? "", + insertions: ins === "-" ? null : Number(ins), + deletions: del === "-" ? null : Number(del), + }); + } + + const message = bodyRaw.replace(/\r\n/g, "\n").trimEnd(); + const nl = message.indexOf("\n"); + const subject = (nl === -1 ? message : message.slice(0, nl)).trim(); + const body = nl === -1 ? "" : message.slice(nl + 1).trim(); + + return { + sha: sha ?? "", + authorName: authorName ?? "", + authorEmail: authorEmail ?? "", + authorDate: authorDate ?? "", + commitDate: commitDate ?? "", + parents: (parentsRaw ?? "").split(" ").filter((p) => p.length > 0), + subject, + body, + files, + }; +} + +/** Options narrowing the `git log` walk. */ +export interface GitLogOptions { + /** Rev to walk (default HEAD). Ignored when `range` is set. */ + rev?: string; + /** Explicit rev range (e.g. `..HEAD`) for incremental walks. */ + range?: string; + /** Only commits at/after this date (passed to `git log --since`). */ + since?: string; + /** Only commits at/before this date (passed to `git log --until`). */ + until?: string; + /** Cap on walked commits (`git log --max-count`). */ + maxCount?: number; + /** Drop all merge commits in git itself (`git log --no-merges`). */ + noMerges?: boolean; +} + +/** + * Stream the commit log of the repo at `repoRoot`, newest first. + * + * An empty repo (no commits on the rev) yields nothing rather than failing, + * so `me claude init` is safe on fresh repos. + */ +export async function* walkGitLog( + repoRoot: string, + options: GitLogOptions = {}, +): AsyncIterable { + const args = [ + "-C", + repoRoot, + "log", + "--encoding=UTF-8", + "--numstat", + "--pretty=format:%x01%H%x00%an%x00%ae%x00%aI%x00%cI%x00%P%x00%B%x00", + ]; + if (options.since) args.push(`--since=${options.since}`); + if (options.until) args.push(`--until=${options.until}`); + if (options.maxCount !== undefined) + args.push(`--max-count=${options.maxCount}`); + if (options.noMerges) args.push("--no-merges"); + args.push(options.range ?? options.rev ?? "HEAD"); + args.push("--"); + + const proc = Bun.spawn(["git", ...args], { + stdout: "pipe", + stderr: "pipe", + }); + // Drain stderr concurrently so a chatty git can't fill the pipe and stall. + const stderrText = new Response(proc.stderr).text(); + + const parser = new GitLogParser(); + const decoder = new TextDecoder("utf-8"); + for await (const chunk of proc.stdout) { + yield* parser.push(decoder.decode(chunk, { stream: true })); + } + const final = decoder.decode(); + if (final.length > 0) yield* parser.push(final); + + const exitCode = await proc.exited; + if (exitCode !== 0) { + const stderr = await stderrText; + // A rev with no commits yet (fresh repo) is "nothing to import", not an + // error. Git phrases this a few ways depending on version/state. + if (/does not have any commits|unknown revision.*HEAD/i.test(stderr)) { + return; + } + throw new Error(`git log failed (exit ${exitCode}): ${stderr.trim()}`); + } + yield* parser.end(); +} + +/** + * True when `sha` is an ancestor of `rev` in the repo at `repoRoot`. Any + * failure (unknown sha after a force-push, detached state, …) is false — + * callers fall back to a full walk, which deterministic ids make safe. + */ +export async function isAncestor( + repoRoot: string, + sha: string, + rev: string, +): Promise { + try { + await execFileAsync( + "git", + ["-C", repoRoot, "merge-base", "--is-ancestor", sha, rev], + { timeout: 5000, encoding: "utf8" }, + ); + return true; + } catch { + return false; + } +} + +/** + * Why a merge commit is skipped, or null to import it. + * + * Default policy: keep merges that carry a message body (GitHub PR merges + * put the PR title there) and drop subject-only boilerplate + * (`Merge branch 'x' into y`). `--no-merges` drops them all — that case is + * filtered by git itself (see walkGitLog), so it never reaches here. + */ +export function mergeSkipReason(commit: GitCommit): string | null { + if (commit.parents.length < 2) return null; + return commit.body.length === 0 ? "merge_boilerplate" : null; +} + +/** Context shared by every memory built in one import run. */ +export interface CommitMemoryContext { + /** Full target tree (e.g. `share.projects.foo.git_history`). */ + tree: string; + /** Project slug the tree was derived from. */ + projectSlug: string; + /** Git remote URL, if the repo has one. */ + gitRemote?: string; + /** Render the changed-file list into the content. */ + fileList: boolean; + /** `meta.imported_at` for this run. */ + importedAt: string; +} + +/** + * Build the memory payload for one commit, or an error string when the + * commit has an unusable date. Content is the message plus a capped file + * list; everything queryable lives in meta; temporal is the commit date. + */ +export function buildCommitMemory( + commit: GitCommit, + ctx: CommitMemoryContext, +): MemoryCreateParams | { error: string } { + const commitMs = Date.parse(commit.commitDate); + if (Number.isNaN(commitMs)) { + return { error: `invalid commit date: ${commit.commitDate}` }; + } + + const id = deterministicUuidV7(`git:${ctx.tree}:${commit.sha}`, commitMs); + + let content = commit.subject; + const body = truncateUtf8(commit.body, BODY_BYTES_CAP); + if (body.length > 0) content += `\n\n${body}`; + if (ctx.fileList && commit.files.length > 0) { + const lines = commit.files + .slice(0, FILE_LIST_CAP) + .map((f) => + f.insertions === null || f.deletions === null + ? ` ${f.path} (binary)` + : ` ${f.path} (+${f.insertions} -${f.deletions})`, + ); + if (commit.files.length > FILE_LIST_CAP) { + lines.push(` … and ${commit.files.length - FILE_LIST_CAP} more files`); + } + content += `\n\nFiles:\n${lines.join("\n")}`; + } + + const insertions = commit.files.reduce( + (sum, f) => sum + (f.insertions ?? 0), + 0, + ); + const deletions = commit.files.reduce( + (sum, f) => sum + (f.deletions ?? 0), + 0, + ); + + const meta: Record = { + type: "git_commit", + sha: commit.sha, + source_project_slug: ctx.projectSlug, + author_name: commit.authorName, + author_email: commit.authorEmail, + author_date: commit.authorDate, + commit_date: commit.commitDate, + files_changed: commit.files.length, + insertions, + deletions, + imported_at: ctx.importedAt, + importer_version: GIT_IMPORTER_VERSION, + }; + if (ctx.gitRemote) meta.source_git_repo = ctx.gitRemote; + if (commit.parents.length >= 2) meta.is_merge = true; + + return { + id, + content, + meta, + tree: ctx.tree, + temporal: { start: new Date(commitMs).toISOString() }, + }; +} + +/** Truncate to a UTF-8 byte budget on a char boundary, marking the cut. */ +function truncateUtf8(text: string, maxBytes: number): string { + if (Buffer.byteLength(text, "utf8") <= maxBytes) return text; + let end = text.length; + while (end > 0 && Buffer.byteLength(text.slice(0, end), "utf8") > maxBytes) { + // Shrink geometrically instead of stepping one char at a time. + end = Math.min(end - 1, Math.floor(end * 0.9)); + } + // Don't cut a surrogate pair in half. + const last = text.charCodeAt(end - 1); + if (last >= 0xd800 && last <= 0xdbff) end--; + return `${text.slice(0, end)}\n…[truncated]`; +} diff --git a/packages/cli/importers/import-transcript.test.ts b/packages/cli/importers/import-transcript.test.ts new file mode 100644 index 0000000..7e96a9d --- /dev/null +++ b/packages/cli/importers/import-transcript.test.ts @@ -0,0 +1,251 @@ +/** + * Unit tests for importTranscriptFile — the live-capture (Claude hook) path. + * + * Uses a fake importer (parseFile returns a synthetic session) + an in-memory + * mock client that round-trips meta through the real buildMeta and simulates + * the server's conditional upsert, so the watermark / incremental-delta / + * version-bump re-render logic is exercised without a database. + */ +import { describe, expect, test } from "bun:test"; +import type { MemoryClient } from "../client.ts"; +import { + type Importer, + importTranscriptFile, + runImport, + type WriteOptions, +} from "./index.ts"; +import type { + ConversationMessage, + ImportedSession, + ImporterOptions, +} from "./types.ts"; + +const WRITE: WriteOptions = { + treeRoot: "share.projects", + sessionsNodeName: "agent_sessions", + fullTranscript: false, + dryRun: false, + verbose: false, +}; + +/** A mock engine backed by an in-memory id→row store, mimicking the server. */ +function mockEngine() { + const store = new Map< + string, + { id: string; meta: Record; content: string } + >(); + const client = { + memory: { + // Filter by source_session_id, order by id desc (server default), slice to limit. + search: async (p: { meta?: Record; limit?: number }) => { + const sid = p.meta?.source_session_id; + const all = [...store.values()] + .filter((m) => m.meta.source_session_id === sid) + .sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)); + const limit = p.limit ?? 10; + return { results: all.slice(0, limit), total: all.length, limit }; + }, + // The server's conditional upsert: insert new ids; replace an existing + // row when its meta value for `replaceIfMetaDiffers` differs; else skip. + batchCreate: async (p: { + memories: Array<{ + id: string; + meta: Record; + content: string; + }>; + replaceIfMetaDiffers?: string; + }) => { + const ids: string[] = []; + const updatedIds: string[] = []; + for (const m of p.memories) { + const existing = store.get(m.id); + if (!existing) { + store.set(m.id, { id: m.id, meta: m.meta, content: m.content }); + ids.push(m.id); + } else if ( + p.replaceIfMetaDiffers !== undefined && + existing.meta[p.replaceIfMetaDiffers] !== + m.meta[p.replaceIfMetaDiffers] + ) { + store.set(m.id, { id: m.id, meta: m.meta, content: m.content }); + updatedIds.push(m.id); + } + } + return { ids, updatedIds }; + }, + }, + } as unknown as MemoryClient; + return { client, store }; +} + +/** An importer whose parseFile returns a fixed session (or null). */ +function importerFor(session: ImportedSession | null): Importer { + return { + tool: "claude", + defaultSource: "", + // biome-ignore lint/correctness/useYield: empty stub generator + discoverSessions: async function* () {}, + parseFile: async () => session, + }; +} + +/** An importer whose discoverSessions yields one fixed session (the `me import claude` path). */ +function discoverImporter(session: ImportedSession): Importer { + return { + tool: "claude", + defaultSource: "", + discoverSessions: async function* () { + yield session; + }, + parseFile: async () => session, + }; +} + +/** Build a session whose messages have strictly-increasing timestamps. */ +function session(messageIds: string[]): ImportedSession { + const messages: ConversationMessage[] = messageIds.map((id, i) => ({ + messageId: id, + timestamp: new Date(Date.UTC(2026, 0, 1, 0, 0, i)).toISOString(), + role: i % 2 === 0 ? "user" : "assistant", + blocks: [{ kind: "text", text: `message ${id}` }], + })); + return { + tool: "claude", + sessionId: "sess-1", + cwd: "/tmp/nonexistent-import-transcript-test/myproj", + sourceFile: "/tmp/transcript.jsonl", + startedAt: messages[0]?.timestamp ?? "2026-01-01T00:00:00.000Z", + endedAt: messages.at(-1)?.timestamp ?? "2026-01-01T00:00:00.000Z", + sourceModifiedAt: "2026-01-01T00:00:00.000Z", + messages, + }; +} + +describe("importTranscriptFile", () => { + test("returns null when the file has no session", async () => { + const { client, store } = mockEngine(); + expect( + await importTranscriptFile(client, importerFor(null), "/x.jsonl", WRITE), + ).toBeNull(); + expect(store.size).toBe(0); + }); + + test("first import writes every message (reconcile path)", async () => { + const { client, store } = mockEngine(); + const out = await importTranscriptFile( + client, + importerFor(session(["a", "b", "c"])), + "/x.jsonl", + WRITE, + ); + expect(out?.inserted).toBe(3); + expect(store.size).toBe(3); + }); + + test("re-importing the same transcript is a no-op (watermark fast path)", async () => { + const { client, store } = mockEngine(); + const imp = () => + importTranscriptFile( + client, + importerFor(session(["a", "b", "c"])), + "/x.jsonl", + WRITE, + ); + await imp(); + const again = await imp(); + expect(again?.inserted).toBe(0); + expect(store.size).toBe(3); + }); + + test("only messages new since the watermark are written", async () => { + const { client, store } = mockEngine(); + await importTranscriptFile( + client, + importerFor(session(["a", "b", "c"])), + "/x.jsonl", + WRITE, + ); + const out = await importTranscriptFile( + client, + importerFor(session(["a", "b", "c", "d"])), + "/x.jsonl", + WRITE, + ); + expect(out?.inserted).toBe(1); // just "d" + expect(store.size).toBe(4); + }); + + // The hook (importTranscriptFile) and `me import claude` (runImport) must be + // idempotent w.r.t. each other: both derive the same tree + deterministic ids + // from the same parse, so importing a session via one path and then the other + // inserts nothing the second time. Guards the shared-derivation assumption. + test("hook capture then `me import claude` over the same session is a no-op", async () => { + const { client, store } = mockEngine(); + const s = session(["a", "b", "c"]); + + await importTranscriptFile(client, importerFor(s), "/x.jsonl", WRITE); + expect(store.size).toBe(3); + + const res = await runImport( + client, + discoverImporter(s), + {} as ImporterOptions, + WRITE, + ); + expect(res.inserted).toBe(0); + expect(res.skipped).toBe(3); + expect(store.size).toBe(3); + }); + + test("`me import claude` then hook capture over the same session is a no-op", async () => { + const { client, store } = mockEngine(); + const s = session(["a", "b", "c"]); + + const res = await runImport( + client, + discoverImporter(s), + {} as ImporterOptions, + WRITE, + ); + expect(res.inserted).toBe(3); + expect(store.size).toBe(3); + + const out = await importTranscriptFile( + client, + importerFor(s), + "/x.jsonl", + WRITE, + ); + expect(out?.inserted).toBe(0); + expect(store.size).toBe(3); + }); + + test("a stale importer_version is re-rendered in place (server-side upsert)", async () => { + const { client, store } = mockEngine(); + const s = session(["a", "b", "c"]); + await importTranscriptFile(client, importerFor(s), "/x.jsonl", WRITE); + expect(store.size).toBe(3); + + // Simulate rows written by an older importer build. + for (const row of store.values()) { + row.meta = { ...row.meta, importer_version: "0" }; + row.content = "stale render"; + } + + // The high-water row is stale → no narrowing; the full plan is submitted + // and the server's upsert rewrites every stale row in one pass. + const out = await importTranscriptFile( + client, + importerFor(s), + "/x.jsonl", + WRITE, + ); + expect(out?.updated).toBe(3); + expect(out?.inserted).toBe(0); + expect(out?.skipped).toBe(0); + for (const row of store.values()) { + expect(row.meta.importer_version).toBe("1"); + expect(row.content).not.toBe("stale render"); + } + }); +}); diff --git a/packages/cli/importers/index.ts b/packages/cli/importers/index.ts index 521b190..cfc89db 100644 --- a/packages/cli/importers/index.ts +++ b/packages/cli/importers/index.ts @@ -7,16 +7,20 @@ * writes one memory per message, using deterministic UUIDv7s keyed * by `(tool, sessionId, messageId)` so re-imports are idempotent. * - * Performance shape: each session does at most two RPCs against the - * engine — one `memory.search` to fetch existing message ids for the - * session, and one `memory.batchCreate` for everything new. Updates - * (only triggered by an `importer_version` bump) are issued one at a - * time and are expected to be rare. + * Reconciliation happens server-side: every planned message is submitted + * through the conditional upsert (`memory.batchCreate` with + * `replaceIfMetaDiffers: "importer_version"`) — new ids insert, rows whose + * stored `importer_version` differs are rewritten in place, and + * already-current rows are skipped, all classified from the batch + * response. No existing-state pre-fetch, so sessions of any size (including + * past the 1000-row search page) reconcile exactly. Per session that is + * ceil(n/chunk) `memory.batchCreate` calls; the live-capture hook adds one + * `memory.search` to narrow the submission to the new suffix. */ -import type { MemoryCreateParams } from "@memory.build/protocol/engine"; +import type { MemoryCreateParams } from "@memory.build/protocol/memory"; import { batchCreateChunked } from "../chunk.ts"; -import type { EngineClient } from "../client.ts"; +import type { MemoryClient } from "../client.ts"; import type { ProgressReporter } from "./progress.ts"; import { SlugRegistry } from "./slug.ts"; import { renderMessageContent, synthesizeTitle } from "./transcript.ts"; @@ -30,21 +34,28 @@ import { deterministicMessageUuidV7 } from "./uuid.ts"; /** * Version tag stored in `meta.importer_version`. Bumping this forces a - * re-render of every previously-imported message on the next run (via - * the version check in `planSession`) so parser changes propagate - * without manual intervention. + * re-render of every previously-imported message on the next run: the + * server's conditional upsert replaces any row whose stored value for + * `IMPORTER_VERSION_KEY` differs from the submitted one, so parser changes + * propagate without manual intervention. * * Locked at "1" during pre-release iteration — bump only after the first * real release so early adopters get parser fixes without a manual wipe. */ export const IMPORTER_VERSION = "1"; +/** The meta key the server compares for the conditional replace. */ +const IMPORTER_VERSION_KEY = "importer_version"; + /** - * Maximum memories per `memory.search` lookup. Same protocol limit. A - * session with more existing messages than this triggers a fallback to - * paged lookups (rare in practice). + * Default capture layout, shared by `me import claude` and the Claude Code capture + * hook so live + imported sessions land in the same place: + * `..`. Under + * `share` so a session-authenticated user (owner@share, not arbitrary top-level + * paths) can write there. */ -const SEARCH_PAGE_LIMIT = 1000; +export const DEFAULT_TREE_ROOT = "share.projects"; +export const DEFAULT_SESSIONS_NODE_NAME = "agent_sessions"; /** An importer's discovery interface — yields normalized sessions. */ export interface Importer { @@ -55,6 +66,12 @@ export interface Importer { stats: ImporterStats, progress?: ProgressReporter, ): AsyncIterable; + /** + * Parse a single transcript file into one session (or null if empty / no + * messages). Used by the live capture hook (`importTranscriptFile`); only the + * Claude importer implements it for now. + */ + parseFile?(path: string): Promise; } /** Result of the orchestration pass. */ @@ -98,7 +115,7 @@ export interface WriteOptions { /** Run discovery + writes for a single importer. */ export async function runImport( - engine: EngineClient, + engine: MemoryClient, importer: Importer, importerOptions: ImporterOptions, writeOptions: WriteOptions, @@ -163,61 +180,143 @@ export async function runImport( } /** - * Write all messages for one session. + * Import a single transcript file — the live-capture path used by the Claude + * Code hook. Reuses the same parse + render + write as `me import claude`, so live + * captures and bulk imports produce identical memories (tree, ids, `source_*` + * metadata). * - * Strategy: - * 1. One `memory.search` to fetch existing message ids + their - * `importer_version` for this session. - * 2. Diff each rendered message against the existing set: - * - id absent → queue for batch insert - * - id present, ver matches → skip - * - id present, ver differs → queue for update - * 3. Issue one `memory.batchCreate` (in chunks of 1000) for inserts; - * updates are issued one at a time (rare path). + * Incremental + stateless: it asks the server for the session's high-water + * message (`searchSessionHighWater` — one `limit 1`, newest-first search) and + * submits only the messages after it. The narrowing is purely a bandwidth + * optimization — a new session, an `importer_version` bump, or a lost anchor + * (compaction/reorder) submits the full plan, and the server's conditional + * upsert reconciles whatever overlaps. Returns null when the file has no + * session. */ -async function writeSession( - engine: EngineClient, - session: ImportedSession, - title: string, - tree: string, - projectSlug: string, - gitRoot: string | undefined, - gitRemote: string | undefined, +export async function importTranscriptFile( + engine: MemoryClient, + importer: Importer, + filePath: string, options: WriteOptions, -): Promise { +): Promise { + if (!importer.parseFile) { + throw new Error( + `importer '${importer.tool}' does not support single-file parsing`, + ); + } + const session = await importer.parseFile(filePath); + if (!session) return null; + + const { slug, gitRoot, gitRemote } = await new SlugRegistry().resolve( + session.cwd, + ); + const tree = `${options.treeRoot}.${slug}.${options.sessionsNodeName}`; + + const plan = planSession(session, tree, slug, gitRoot, gitRemote, options); const outcome: SessionOutcome = { sessionId: session.sessionId, - title, + title: synthesizeTitle(session), tree, sourceFile: session.sourceFile, inserted: 0, updated: 0, - skipped: 0, - failed: 0, - errors: [], + skipped: plan.skipped, + failed: plan.failed, + errors: [...plan.errors], }; - // Build the per-message write payloads up front so we can plan the - // batch in one pass and skip any messages that render to empty - // content under the chosen mode. - const planned: Array<{ - message: ConversationMessage; - memoryId: string; - payload: MemoryCreateParams; - }> = []; + let planned = plan.planned; + const hw = await searchSessionHighWater( + engine, + tree, + session.tool, + session.sessionId, + ); + if (hw && hw.importerVersion === IMPORTER_VERSION) { + const idx = planned.findIndex((p) => p.message.messageId === hw.messageId); + if (idx !== -1) { + // The anchor and everything before it are already imported at the + // current version (transcripts are append-only). + outcome.skipped += idx + 1; + planned = planned.slice(idx + 1); + } + } + + await submitPlanned(engine, planned, outcome, options); + return outcome; +} + +/** + * The session's high-water message: the latest already-imported message for + * (tool, sessionId) under `tree`. One `memory.search` with `limit: 1` — unranked + * search defaults to newest-first (by id, which encodes the message timestamp), + * so results[0] is the most recent. Null when nothing is imported yet. + */ +async function searchSessionHighWater( + engine: MemoryClient, + tree: string, + tool: ImportedSession["tool"], + sessionId: string, +): Promise<{ messageId: string; importerVersion?: string } | null> { + const res = await engine.memory.search({ + tree, + meta: { source_tool: tool, source_session_id: sessionId }, + limit: 1, + }); + const top = res.results[0]; + if (!top) return null; + const messageId = top.meta.source_message_id; + if (typeof messageId !== "string") return null; + const v = top.meta.importer_version; + return { messageId, importerVersion: typeof v === "string" ? v : undefined }; +} + +/** One planned message write (post-render, pre-dedup/diff). */ +interface PlannedMessage { + message: ConversationMessage; + memoryId: string; + payload: MemoryCreateParams; +} + +/** Result of planning a session's writes (rendered, deduped). */ +interface PlanResult { + planned: PlannedMessage[]; + skipped: number; + failed: number; + errors: Array<{ messageId: string; error: string }>; +} + +/** + * Render + dedup a session's messages into write payloads (no RPCs). Skips + * messages that render empty under the chosen mode, records bad timestamps as + * failures, and collapses events sharing a deterministic id (resume/replay + * artefacts) so the batch can't trip the unique constraint server-side. + */ +function planSession( + session: ImportedSession, + tree: string, + projectSlug: string, + gitRoot: string | undefined, + gitRemote: string | undefined, + options: WriteOptions, +): PlanResult { + const planned: PlannedMessage[] = []; + let skipped = 0; + let failed = 0; + const errors: Array<{ messageId: string; error: string }> = []; for (const message of session.messages) { const content = renderMessageContent(message, { fullTranscript: options.fullTranscript, }); if (content === null) { - outcome.skipped++; + skipped++; continue; } const timestampMs = Number(Date.parse(message.timestamp)); if (Number.isNaN(timestampMs)) { - outcome.failed++; - outcome.errors.push({ + failed++; + errors.push({ messageId: message.messageId, error: `invalid message timestamp: ${message.timestamp}`, }); @@ -241,143 +340,108 @@ async function writeSession( planned.push({ message, memoryId, - payload: { - id: memoryId, - content, - meta, - tree, - temporal, - }, + payload: { id: memoryId, content, meta, tree, temporal }, }); } - // A session JSONL can contain two events that share the same `event.uuid` - // (resume artefacts, sidechain merges, replay wrappers). The deterministic - // UUIDv7 is keyed on (tool, sessionId, messageId), so duplicates collapse - // to the same id and would otherwise hit the unique constraint server-side - // — taking the whole batch's transaction down with them. Drop them here. const dedup = dedupByMemoryId(planned); - outcome.skipped += dedup.duplicates; - const deduped = dedup.unique; - - if (deduped.length === 0) return outcome; - - // Bulk-fetch existing message ids for this session in one search. - let existing: Map; - try { - existing = await fetchExistingMessageVersions(engine, session, tree); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - // The whole session fails if we can't determine existing state. - outcome.failed += deduped.length; - for (const p of deduped) { - outcome.errors.push({ - messageId: p.message.messageId, - error: `existing-state lookup failed: ${msg}`, - }); - } - return outcome; - } - - const toInsert: MemoryCreateParams[] = []; - const toUpdate: Array<{ messageId: string; payload: MemoryCreateParams }> = - []; - - for (const p of deduped) { - const existingVersion = existing.get(p.memoryId); - if (existingVersion === undefined) { - toInsert.push(p.payload); - } else if (existingVersion === IMPORTER_VERSION) { - outcome.skipped++; - } else { - toUpdate.push({ messageId: p.message.messageId, payload: p.payload }); - } - } + return { + planned: dedup.unique, + skipped: skipped + dedup.duplicates, + failed, + errors, + }; +} - // Inserts: one batchCreate per chunk. Chunks are cut by byte budget OR - // count cap, whichever fires first, so a chunk's serialized request body - // stays under the server's request size limit. - if (toInsert.length > 0) { - if (options.dryRun) { - outcome.inserted += toInsert.length; - } else { - const { insertedIds, errors } = await batchCreateChunked( - engine, - toInsert, - ); - outcome.inserted += insertedIds.length; - // Each chunk error contributes its full itemCount to `failed` and - // attaches the same message to each id in that chunk — matching the - // pre-chunking behavior of one error row per attempted message. - for (const e of errors) { - outcome.failed += e.itemCount; - for (const id of e.ids) { - outcome.errors.push({ messageId: id, error: e.error }); - } - } - } - } +/** + * Write all messages for one session: plan + dedup, then submit everything + * through the server's conditional upsert (see `submitPlanned`). No + * existing-state read — classification comes from the batch response. + */ +async function writeSession( + engine: MemoryClient, + session: ImportedSession, + title: string, + tree: string, + projectSlug: string, + gitRoot: string | undefined, + gitRemote: string | undefined, + options: WriteOptions, +): Promise { + const outcome: SessionOutcome = { + sessionId: session.sessionId, + title, + tree, + sourceFile: session.sourceFile, + inserted: 0, + updated: 0, + skipped: 0, + failed: 0, + errors: [], + }; - // Updates: rare, issued one at a time. - for (const u of toUpdate) { - if (options.dryRun) { - outcome.updated++; - continue; - } - try { - await engine.memory.update({ - id: u.payload.id as string, - content: u.payload.content, - meta: u.payload.meta, - tree: u.payload.tree, - temporal: u.payload.temporal, - }); - outcome.updated++; - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - outcome.failed++; - outcome.errors.push({ messageId: u.messageId, error: msg }); - } - } + const plan = planSession( + session, + tree, + projectSlug, + gitRoot, + gitRemote, + options, + ); + outcome.skipped += plan.skipped; + outcome.failed += plan.failed; + outcome.errors.push(...plan.errors); + await submitPlanned(engine, plan.planned, outcome, options); return outcome; } /** - * Fetch existing message ids + their `importer_version` for one session. + * Submit planned messages through the conditional upsert and fold the + * outcome into `outcome`: new ids insert, rows whose stored + * `importer_version` differs are rewritten in place (the version-bump + * re-render), and already-current rows are skipped — all classified from + * the batch response, independent of how many messages the session already + * has server-side. + * + * Chunks are cut by byte budget OR count cap (see batchCreateChunked) so + * each request body stays under the server's size limit; a failed chunk + * contributes its full itemCount to `failed` with one error row per id. * - * Uses `memory.search` with a `meta` filter on `source_session_id` + - * `source_tool`. Restricted to the session's tree so the search is - * indexed by tree first. Returns `id → importer_version` (version is - * `undefined` when the record was written before the field existed). + * Dry runs report every planned message as an insert — there is no server + * classification without submitting. */ -async function fetchExistingMessageVersions( - engine: EngineClient, - session: ImportedSession, - tree: string, -): Promise> { - const result = await engine.memory.search({ - tree, - meta: { - source_tool: session.tool, - source_session_id: session.sessionId, - }, - limit: SEARCH_PAGE_LIMIT, - }); - if (result.total > result.results.length) { - // Sessions exceeding 1000 already-imported messages would silently - // re-insert and hit duplicate-id errors. Surface that loudly so we - // can paginate when it actually happens. - throw new Error( - `session has ${result.total} existing messages but bulk-fetch is capped at ${SEARCH_PAGE_LIMIT}; pagination not yet implemented`, - ); +async function submitPlanned( + engine: MemoryClient, + planned: PlannedMessage[], + outcome: SessionOutcome, + options: WriteOptions, +): Promise { + if (planned.length === 0) return; + if (options.dryRun) { + outcome.inserted += planned.length; + return; } - const map = new Map(); - for (const r of result.results) { - const v = r.meta.importer_version; - map.set(r.id, typeof v === "string" ? v : undefined); + + const { insertedIds, updatedIds, errors } = await batchCreateChunked( + engine, + planned.map((p) => p.payload), + { replaceIfMetaDiffers: IMPORTER_VERSION_KEY }, + ); + outcome.inserted += insertedIds.length; + outcome.updated += updatedIds.length; + let failedCount = 0; + for (const e of errors) { + failedCount += e.itemCount; + outcome.failed += e.itemCount; + for (const id of e.ids) { + outcome.errors.push({ messageId: id, error: e.error }); + } } - return map; + // Whatever the server neither inserted, updated, nor failed already + // exists at the current importer_version. + outcome.skipped += + planned.length - insertedIds.length - updatedIds.length - failedCount; } /** Build the full meta object for one message memory. */ @@ -400,7 +464,7 @@ function buildMeta( source_file: session.sourceFile, content_mode: options.fullTranscript ? "full_transcript" : "default", imported_at: new Date().toISOString(), - importer_version: IMPORTER_VERSION, + [IMPORTER_VERSION_KEY]: IMPORTER_VERSION, }; if (session.title) meta.source_session_title = session.title; @@ -448,7 +512,7 @@ function logOutcome( /** * Drop items whose `memoryId` has already been seen, preserving order. * Exported so the dedup behavior can be unit-tested without standing up - * a fake EngineClient. Used by `writeSession` to absorb sessions whose + * a fake MemoryClient. Used by `writeSession` to absorb sessions whose * JSONL has duplicate `event.uuid` entries (which would otherwise produce * two planned memories with the same deterministic UUIDv7). */ diff --git a/packages/cli/importers/slug.test.ts b/packages/cli/importers/slug.test.ts index 506cbbb..9f9735b 100644 --- a/packages/cli/importers/slug.test.ts +++ b/packages/cli/importers/slug.test.ts @@ -6,7 +6,19 @@ * `undefined` and the fallback to `basename(cwd)` is exercised. */ import { describe, expect, test } from "bun:test"; -import { normalizeSlug, SlugRegistry } from "./slug.ts"; +import { normalizeSlug, repoNameFromRemote, SlugRegistry } from "./slug.ts"; + +describe("repoNameFromRemote", () => { + test("extracts the repo name from https and ssh remotes (sans .git)", () => { + expect(repoNameFromRemote("https://github.com/org/memory-engine.git")).toBe( + "memory-engine", + ); + expect(repoNameFromRemote("git@github.com:org/memory-engine.git")).toBe( + "memory-engine", + ); + expect(repoNameFromRemote("https://example.com/a/b/repo")).toBe("repo"); + }); +}); describe("normalizeSlug", () => { test("lowercases and replaces non-alphanumeric with underscore", () => { diff --git a/packages/cli/importers/slug.ts b/packages/cli/importers/slug.ts index daf25bd..c3567c9 100644 --- a/packages/cli/importers/slug.ts +++ b/packages/cli/importers/slug.ts @@ -2,8 +2,11 @@ * Project slug derivation for agent conversation imports. * * A "slug" is an ltree-safe label derived from the session's cwd: - * - Prefer the git repo root directory name if the cwd is inside a repo. - * - Fall back to `basename(cwd)`. + * - Prefer the git `origin` remote's repo name (stable across clone locations, + * and shared with the Claude Code capture hook via the import path + * (`importTranscriptFile`), so live + imported sessions for the same repo + * share one project label). + * - Else the git repo root directory name, else `basename(cwd)`. * - Normalize to `[a-z0-9_]+`. * * Slug collisions (different cwds that normalize to the same label) are @@ -54,6 +57,14 @@ export function normalizeSlug(raw: string): string { return collapsed; } +/** + * Extract the repo name (last path segment, sans `.git`) from a git remote URL. + * Handles `https://github.com/org/repo.git` and `git@github.com:org/repo.git`. + */ +export function repoNameFromRemote(url: string): string | undefined { + return url.trim().match(/[/:]([^/:]+?)(?:\.git)?$/)?.[1]; +} + /** * Detect the git top-level directory for `cwd`, if any. * @@ -126,6 +137,22 @@ function shortHash(input: string): string { return createHash("sha256").update(input).digest("hex").slice(0, 4); } +/** + * Derive the base (pre-collision) project slug for a cwd: the git `origin` + * remote's repo name if available, else the git repo root dir name, else + * `basename(cwd)` — normalized to an ltree label. Returns the git info too so + * callers can use it for collision disambiguation. + */ +async function deriveBaseSlug( + cwd: string, +): Promise<{ baseSlug: string; gitRoot?: string; gitRemote?: string }> { + const { gitRoot, gitRemote } = await getGitInfo(cwd); + const rawName = + (gitRemote ? repoNameFromRemote(gitRemote) : undefined) ?? + basename(gitRoot ?? cwd); + return { baseSlug: normalizeSlug(rawName), gitRoot, gitRemote }; +} + /** * Registry used to track slug assignments across a single import run so * colliding base slugs from different projects get distinct suffixes. @@ -149,9 +176,7 @@ export class SlugRegistry { return { slug: UNKNOWN_SLUG, baseSlug: UNKNOWN_SLUG, cwd: "" }; } - const { gitRoot, gitRemote } = await getGitInfo(cwd); - const source = gitRoot ?? cwd; - const baseSlug = normalizeSlug(basename(source)); + const { baseSlug, gitRoot, gitRemote } = await deriveBaseSlug(cwd); const bucket = this.assignments.get(baseSlug) ?? []; diff --git a/packages/cli/importers/uuid.test.ts b/packages/cli/importers/uuid.test.ts index 8c9a4ab..5a3b126 100644 --- a/packages/cli/importers/uuid.test.ts +++ b/packages/cli/importers/uuid.test.ts @@ -1,8 +1,8 @@ /** - * Tests for deterministic per-message UUIDv7 derivation. + * Tests for deterministic UUIDv7 derivation. */ import { describe, expect, test } from "bun:test"; -import { deterministicMessageUuidV7 } from "./uuid.ts"; +import { deterministicMessageUuidV7, deterministicUuidV7 } from "./uuid.ts"; const UUIDV7_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -103,4 +103,23 @@ describe("deterministicMessageUuidV7", () => { // Position 19 = variant high nibble; top 2 bits must be 10 → hex 8/9/a/b. expect(["8", "9", "a", "b"]).toContain(id.charAt(19)); }); + + test("equals deterministicUuidV7 over the tool:session:message key", () => { + // The message variant is a thin wrapper; the key format is load-bearing + // for ids already written to engines, so lock it down. + const ts = 1_700_000_000_000; + expect(deterministicMessageUuidV7("claude", "abc", "m1", ts)).toBe( + deterministicUuidV7("claude:abc:m1", ts), + ); + }); +}); + +describe("deterministicUuidV7", () => { + test("namespaced keys produce distinct ids at the same timestamp", () => { + const ts = 1_700_000_000_000; + const a = deterministicUuidV7("git:share.projects.x.git_history:abc", ts); + const b = deterministicUuidV7("git:share.projects.y.git_history:abc", ts); + expect(a).toMatch(UUIDV7_RE); + expect(a).not.toBe(b); + }); }); diff --git a/packages/cli/importers/uuid.ts b/packages/cli/importers/uuid.ts index 19323d9..2077921 100644 --- a/packages/cli/importers/uuid.ts +++ b/packages/cli/importers/uuid.ts @@ -1,35 +1,32 @@ /** * Deterministic UUIDv7 derivation for idempotent imports. * - * We need stable UUIDs so that re-importing the same message collides + * We need stable UUIDs so that re-importing the same record collides * with the existing row in the database and becomes a no-op. Regular * UUIDv7 is random, so we derive a deterministic variant: * - * - 48 bits: Unix ms timestamp (message timestamp) — keeps chronological sort + * - 48 bits: Unix ms timestamp (record timestamp) — keeps chronological sort * - 4 bits: version = 7 - * - 12 bits: rand_a ← SHA-256(tool + ':' + sessionId + ':' + messageId), bits 0..11 + * - 12 bits: rand_a ← SHA-256(key), bits 0..11 * - 2 bits: variant = 10 - * - 62 bits: rand_b ← SHA-256(tool + ':' + sessionId + ':' + messageId), bits 12..73 + * - 62 bits: rand_b ← SHA-256(key), bits 12..73 * * The result passes the `uuid_extract_version(id) = 7` check in the engine's - * memory schema, sorts by message time, and is stable across re-imports of + * memory schema, sorts by record time, and is stable across re-imports of * the same source data. */ import { createHash } from "node:crypto"; import type { SourceTool } from "./types.ts"; /** - * Compute a deterministic UUIDv7 from `(tool, sessionId, messageId, timestampMs)`. + * Compute a deterministic UUIDv7 from an identity `key` and a timestamp. * * Same inputs always return the same UUID; different inputs produce - * different UUIDs (cryptographically, with SHA-256). + * different UUIDs (cryptographically, with SHA-256). Each importer owns + * its key format (messages: `tool:sessionId:messageId`; git commits: + * `git::`) — keys must be namespaced so importers can't collide. */ -export function deterministicMessageUuidV7( - tool: SourceTool, - sessionId: string, - messageId: string, - timestampMs: number, -): string { +export function deterministicUuidV7(key: string, timestampMs: number): string { // 16 bytes = 128 bits. const bytes = new Uint8Array(16); @@ -42,11 +39,9 @@ export function deterministicMessageUuidV7( bytes[4] = Math.floor(ts / 2 ** 8) & 0xff; bytes[5] = ts & 0xff; - // SHA-256 over "tool:sessionId:messageId" gives 32 bytes of deterministic - // pseudo-random. We only need 74 bits (12 + 62) so 10 bytes is plenty. - const digest = createHash("sha256") - .update(`${tool}:${sessionId}:${messageId}`, "utf8") - .digest(); + // SHA-256 over the key gives 32 bytes of deterministic pseudo-random. + // We only need 74 bits (12 + 62) so 10 bytes is plenty. + const digest = createHash("sha256").update(key, "utf8").digest(); // Bytes 6..7: version (4 bits = 0x7) + rand_a (12 bits). const randA = ((digest[0] ?? 0) << 8) | (digest[1] ?? 0); @@ -63,6 +58,19 @@ export function deterministicMessageUuidV7( return bytesToUuid(bytes); } +/** + * Compute a deterministic UUIDv7 from `(tool, sessionId, messageId, timestampMs)`. + * The message-import key format; see `deterministicUuidV7`. + */ +export function deterministicMessageUuidV7( + tool: SourceTool, + sessionId: string, + messageId: string, + timestampMs: number, +): string { + return deterministicUuidV7(`${tool}:${sessionId}:${messageId}`, timestampMs); +} + /** Format 16 bytes as a canonical UUID string. */ function bytesToUuid(bytes: Uint8Array): string { const hex: string[] = []; diff --git a/packages/cli/index.ts b/packages/cli/index.ts index 3738174..d157a1a 100755 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -9,25 +9,26 @@ import { Command } from "commander"; * and all command groups, then runs. */ import { CLIENT_VERSION } from "../../version"; +import { createAccessCommand } from "./commands/access.ts"; +import { createAgentCommand } from "./commands/agent.ts"; import { createApiKeyCommand } from "./commands/apikey.ts"; import { createClaudeCommand } from "./commands/claude.ts"; import { createCodexCommand } from "./commands/codex.ts"; -import { createEngineCommand } from "./commands/engine.ts"; import { createGeminiCommand } from "./commands/gemini.ts"; -import { createGrantCommand } from "./commands/grant.ts"; -import { createInvitationCommand } from "./commands/invitation.ts"; +import { createGroupCommand } from "./commands/group.ts"; +import { createImportCommand } from "./commands/import-group.ts"; import { createLoginCommand } from "./commands/login.ts"; import { createLogoutCommand } from "./commands/logout.ts"; import { createMcpCommand } from "./commands/mcp.ts"; -import { createMemoryCommand } from "./commands/memory.ts"; +import { + createMemoryAliasCommands, + createMemoryCommand, +} from "./commands/memory.ts"; import { createOpenCodeCommand } from "./commands/opencode.ts"; -import { createOrgCommand } from "./commands/org.ts"; -import { createOwnerCommand } from "./commands/owner.ts"; import { createPackCommand } from "./commands/pack.ts"; -import { createRoleCommand } from "./commands/role.ts"; import { createServeCommand } from "./commands/serve.ts"; +import { createSpaceCommand } from "./commands/space.ts"; import { createUpgradeCommand } from "./commands/upgrade.ts"; -import { createUserCommand } from "./commands/user.ts"; import { createVersionCommand } from "./commands/version.ts"; import { createWhoamiCommand } from "./commands/whoami.ts"; import { setExpanded } from "./output.ts"; @@ -64,17 +65,19 @@ program.addCommand(createWhoamiCommand()); program.addCommand(createVersionCommand()); program.addCommand(createUpgradeCommand()); -// Engine commands -program.addCommand(createEngineCommand()); - -// Org commands -program.addCommand(createOrgCommand()); - -// Invitation commands -program.addCommand(createInvitationCommand()); +// Space commands (the new model: spaces, groups, access, agents, api keys) +program.addCommand(createSpaceCommand()); +program.addCommand(createGroupCommand()); +program.addCommand(createAccessCommand()); +program.addCommand(createAgentCommand()); +program.addCommand(createApiKeyCommand()); -// Memory commands +// Memory commands — both as `me memory ` and top-level aliases (`me search`) program.addCommand(createMemoryCommand()); +for (const c of createMemoryAliasCommands()) program.addCommand(c); + +// Import group — one subcommand per source (`me import memories|claude|codex|opencode|git`) +program.addCommand(createImportCommand()); // MCP server program.addCommand(createMcpCommand()); @@ -88,13 +91,6 @@ program.addCommand(createCodexCommand()); // Local web UI program.addCommand(createServeCommand()); -// Engine-level RBAC commands -program.addCommand(createUserCommand()); -program.addCommand(createGrantCommand()); -program.addCommand(createRoleCommand()); -program.addCommand(createOwnerCommand()); -program.addCommand(createApiKeyCommand()); - // Pack commands program.addCommand(createPackCommand()); diff --git a/packages/cli/keychain.test.ts b/packages/cli/keychain.test.ts new file mode 100644 index 0000000..5df81f8 --- /dev/null +++ b/packages/cli/keychain.test.ts @@ -0,0 +1,59 @@ +/** + * Keychain backend tests. + * + * The disabled path (ME_NO_KEYCHAIN) is deterministic everywhere. The live + * round-trip touches the real OS keychain, so it runs only where one is usable: + * it skips gracefully on Linux without libsecret, in CI, with a locked store, or + * when ME_NO_KEYCHAIN is set — but on an interactive mac, where a keychain + * should always work, a failure is treated as a real regression. + */ +import { afterEach, expect, test } from "bun:test"; +import { + keychainAvailable, + keychainDelete, + keychainGet, + keychainSet, + resetKeychainForTests, +} from "./keychain.ts"; + +// The ambient value, restored after each test so a dev/CI ME_NO_KEYCHAIN survives. +const AMBIENT_NO_KEYCHAIN = process.env.ME_NO_KEYCHAIN; + +afterEach(() => { + if (AMBIENT_NO_KEYCHAIN === undefined) delete process.env.ME_NO_KEYCHAIN; + else process.env.ME_NO_KEYCHAIN = AMBIENT_NO_KEYCHAIN; + resetKeychainForTests(); +}); + +test("ME_NO_KEYCHAIN forces the file fallback", () => { + process.env.ME_NO_KEYCHAIN = "1"; + resetKeychainForTests(); + expect(keychainAvailable()).toBe(false); + expect(keychainSet("acct", "secret")).toBe(false); + expect(keychainGet("acct")).toBeUndefined(); + keychainDelete("acct"); // no-op, must not throw +}); + +test("keychain round-trip when an OS keychain is usable", () => { + // Respect an ambient opt-out — nothing to exercise. + const v = process.env.ME_NO_KEYCHAIN; + if (v === "1" || v === "true") return; + resetKeychainForTests(); + + const account = `https://kc-test-${crypto.randomUUID()}.example.com`; + if (!keychainSet(account, "live-secret")) { + // No usable keychain (Linux without secret-tool, CI, locked store). Skip — + // but an interactive mac should always have one, so a miss there is a bug. + expect(process.platform === "darwin" && !process.env.CI).toBe(false); + return; + } + + try { + expect(keychainGet(account)).toBe("live-secret"); + keychainSet(account, "updated-secret"); // -U updates in place + expect(keychainGet(account)).toBe("updated-secret"); + } finally { + keychainDelete(account); + } + expect(keychainGet(account)).toBeUndefined(); +}); diff --git a/packages/cli/keychain.ts b/packages/cli/keychain.ts new file mode 100644 index 0000000..84bfd48 --- /dev/null +++ b/packages/cli/keychain.ts @@ -0,0 +1,189 @@ +/** + * OS keychain for the CLI session token, with a 0600-file fallback. + * + * The session token is the only secret the CLI persists. When an OS secret + * store is available we keep it there (one entry per server origin); otherwise + * the caller falls back to the 0600 credentials file. Backends shell out to the + * platform tool, so the compiled `me` binary needs no native module: + * + * - macOS: `security` (the login keychain) + * - Linux: `secret-tool` (libsecret / the Secret Service) + * + * Anything else (Windows, headless Linux without a Secret Service) reports + * unavailable and the caller uses the file. Set `ME_NO_KEYCHAIN=1` to force the + * file fallback everywhere (CI, debugging, sandboxes). + * + * Detection + operations are best-effort and defensive: a missing tool, a + * non-running secret service, a locked store, or a spawn error is treated as + * "not stored / not found" so the file fallback transparently kicks in. The + * `account` is the (normalized) server origin; the secret is the session token. + */ + +/** Keychain service name — how the entries appear in Keychain Access / seahorse. */ +const SERVICE = "memory.build"; + +/** A spawn timeout so a prompting/locked secret store can't hang the CLI. */ +const SPAWN_TIMEOUT_MS = 5_000; + +interface Backend { + get(account: string): string | undefined; + set(account: string, secret: string): boolean; + del(account: string): void; +} + +function keychainDisabled(): boolean { + const v = process.env.ME_NO_KEYCHAIN; + return v === "1" || v === "true"; +} + +/** Run a command, capturing stdout; returns null on spawn failure. */ +function run( + cmd: string[], + stdin?: string, +): { exitCode: number; stdout: string } | null { + try { + const r = Bun.spawnSync({ + cmd, + stdin: stdin !== undefined ? new TextEncoder().encode(stdin) : undefined, + stdout: "pipe", + stderr: "pipe", + timeout: SPAWN_TIMEOUT_MS, + }); + return { exitCode: r.exitCode ?? 1, stdout: r.stdout.toString() }; + } catch { + return null; + } +} + +// macOS — the `security` CLI against the login keychain. The secret is passed +// via argv (-w); it is briefly visible to `ps`, but only to the same user, who +// can already read the 0600 fallback file. +const darwinBackend: Backend = { + get(account) { + const r = run([ + "security", + "find-generic-password", + "-s", + SERVICE, + "-a", + account, + "-w", + ]); + if (!r || r.exitCode !== 0) return undefined; + const out = r.stdout.replace(/\n$/, ""); + return out.length > 0 ? out : undefined; + }, + set(account, secret) { + const r = run([ + "security", + "add-generic-password", + "-s", + SERVICE, + "-a", + account, + "-w", + secret, + "-U", // update the entry if it already exists + ]); + return r?.exitCode === 0; + }, + del(account) { + run(["security", "delete-generic-password", "-s", SERVICE, "-a", account]); + }, +}; + +// Linux — libsecret's `secret-tool` (Secret Service). The secret is read from +// stdin (never argv). `lookup` prints the secret with no trailing newline. +const linuxBackend: Backend = { + get(account) { + const r = run([ + "secret-tool", + "lookup", + "service", + SERVICE, + "account", + account, + ]); + if (!r || r.exitCode !== 0) return undefined; + return r.stdout.length > 0 ? r.stdout : undefined; + }, + set(account, secret) { + const r = run( + [ + "secret-tool", + "store", + "--label=memory.build CLI session", + "service", + SERVICE, + "account", + account, + ], + secret, + ); + return r?.exitCode === 0; + }, + del(account) { + run(["secret-tool", "clear", "service", SERVICE, "account", account]); + }, +}; + +let resolved: Backend | null | undefined; + +/** The backend for this host, or null when no keychain is usable. Memoized. */ +function backend(): Backend | null { + if (resolved !== undefined) return resolved; + resolved = selectBackend(); + return resolved; +} + +function selectBackend(): Backend | null { + if (keychainDisabled()) return null; + if (process.platform === "darwin") return darwinBackend; + if (process.platform === "linux" && Bun.which("secret-tool")) { + return linuxBackend; + } + return null; +} + +/** Reset the memoized backend — for tests that toggle `ME_NO_KEYCHAIN`. */ +export function resetKeychainForTests(): void { + resolved = undefined; +} + +/** True if an OS keychain backend is available on this host. */ +export function keychainAvailable(): boolean { + return backend() !== null; +} + +/** Read the secret for `account`, or undefined if absent / unavailable. */ +export function keychainGet(account: string): string | undefined { + const b = backend(); + if (!b) return undefined; + try { + return b.get(account); + } catch { + return undefined; + } +} + +/** Store the secret for `account`. Returns true iff it landed in the keychain. */ +export function keychainSet(account: string, secret: string): boolean { + const b = backend(); + if (!b) return false; + try { + return b.set(account, secret); + } catch { + return false; + } +} + +/** Remove the secret for `account` (no-op if absent / unavailable). */ +export function keychainDelete(account: string): void { + const b = backend(); + if (!b) return; + try { + b.del(account); + } catch { + // best-effort + } +} diff --git a/packages/cli/mcp/agent-install.ts b/packages/cli/mcp/agent-install.ts index fc1e62d..5f5fe69 100644 --- a/packages/cli/mcp/agent-install.ts +++ b/packages/cli/mcp/agent-install.ts @@ -11,6 +11,8 @@ import { buildMeCommand, installMcpServer, MCP_TOOLS } from "./install.ts"; export interface AgentInstallOptions { apiKey?: string; server?: string; + /** The space slug to bake into the MCP command (api keys are global). */ + space?: string; /** * Configuration scope for tools that support it (Claude Code, Gemini CLI). * Ignored by tools without a scope concept (Codex, OpenCode). @@ -34,26 +36,50 @@ export async function runAgentMcpInstall( process.exit(1); } - // Resolve credentials: flags > credentials file - let { apiKey, server } = opts; - if (!apiKey || !server) { - const creds = resolveCredentials(server); - if (!apiKey) apiKey = creds.apiKey; - if (!server) server = creds.server; - } - - if (!apiKey) { - clack.log.error( - "No API key available. Either pass --api-key or run 'me engine use' first.", - ); - process.exit(1); - } + // Resolve credentials: flags > env (ME_API_KEY / ME_SERVER / ME_SPACE) > + // stored config. + const creds = resolveCredentials(opts.server); + const apiKey = opts.apiKey ?? creds.apiKey; // --api-key > ME_API_KEY + const server = opts.server ?? creds.server; if (!server) { clack.log.error("No server URL available. Pass --server or set ME_SERVER."); process.exit(1); } + // Default path: no api key → the MCP server uses your login SESSION, resolved + // from the keychain/config at runtime each time it starts (so it survives + // `me login`). Pass --api-key / ME_API_KEY only for a headless agent that + // can't reach your keychain; that bakes a long-lived global key and must pin a + // space. The `--space` flag pins the space either way; otherwise the session + // path resolves it at runtime from ME_SPACE / active space. + let meCmd: string[]; + if (apiKey) { + const space = opts.space ?? creds.activeSpace; + if (!space) { + clack.log.error( + "No space for the API key. Pass --space, set ME_SPACE, or run 'me space use ' (keys are global, so the space must be fixed).", + ); + process.exit(1); + } + meCmd = buildMeCommand({ server, apiKey, space }); + } else { + if (!creds.sessionToken) { + clack.log.error( + "Not logged in. Run 'me login' (the MCP server will use your session), or pass --api-key / set ME_API_KEY for a headless agent.", + ); + process.exit(1); + } + // Bake only --server (+ an explicit --space pin if given); the session token + // and space resolve at runtime. + meCmd = buildMeCommand({ server, space: opts.space }); + if (!opts.space && !creds.activeSpace) { + clack.log.warn( + "No active space set — the MCP server will fail until you run 'me space use ' (or set ME_SPACE). Re-run with --space to pin one.", + ); + } + } + // For CLI tools, require the binary to be on PATH. JSON-file tools // (e.g. OpenCode) just edit a config file and don't need the binary. if (tool.method === "cli" && Bun.which(tool.bin) === null) { @@ -63,9 +89,6 @@ export async function runAgentMcpInstall( process.exit(1); } - // Build the me mcp command with baked-in credentials - const meCmd = buildMeCommand(apiKey, server); - const spin = clack.spinner(); spin.start(`Registering with ${tool.name}...`); const result = await installMcpServer(tool, meCmd, { scope: opts.scope }); diff --git a/packages/cli/mcp/install.test.ts b/packages/cli/mcp/install.test.ts index 2e59640..704b9bd 100644 --- a/packages/cli/mcp/install.test.ts +++ b/packages/cli/mcp/install.test.ts @@ -6,20 +6,48 @@ import { buildMeCommand, buildOpenCodeConfig, MCP_TOOLS } from "./install.ts"; describe("buildMeCommand", () => { test("uses bare 'me' command on PATH", () => { - const cmd = buildMeCommand("test-key-123", "https://api.memory.build"); + const cmd = buildMeCommand({ server: "https://api.memory.build" }); expect(cmd[0]).toBe("me"); expect(cmd[1]).toBe("mcp"); }); - test("includes --api-key and --server with correct values", () => { - const cmd = buildMeCommand("k", "https://example.com"); + test("session default bakes only --server (token + space resolve at runtime)", () => { + const cmd = buildMeCommand({ server: "https://example.com" }); + expect(cmd).toEqual(["me", "mcp", "--server", "https://example.com"]); + expect(cmd).not.toContain("--api-key"); + expect(cmd).not.toContain("--space"); + }); + + test("pins --space when given (session path with explicit space)", () => { + const cmd = buildMeCommand({ + server: "https://example.com", + space: "abc123def456", + }); + expect(cmd).toEqual([ + "me", + "mcp", + "--server", + "https://example.com", + "--space", + "abc123def456", + ]); + }); + + test("headless agent bakes --api-key and --space", () => { + const cmd = buildMeCommand({ + server: "https://example.com", + apiKey: "k", + space: "abc123def456", + }); expect(cmd).toEqual([ "me", "mcp", - "--api-key", - "k", "--server", "https://example.com", + "--api-key", + "k", + "--space", + "abc123def456", ]); }); }); diff --git a/packages/cli/mcp/install.ts b/packages/cli/mcp/install.ts index a2a6abb..a0ada7d 100644 --- a/packages/cli/mcp/install.ts +++ b/packages/cli/mcp/install.ts @@ -114,13 +114,29 @@ export function detectInstalledTools(): McpTool[] { } /** - * Build the `me mcp` command array with baked-in credentials. + * Build the `me mcp …` command array to embed in an MCP config. * - * Always uses bare `me` — the binary is expected to be on PATH - * whether installed via the install script, Homebrew, or npm. + * Only `--server` is always baked. `--api-key` and `--space` are baked **only** + * when provided: + * - **Default (no api key):** the MCP server resolves your login *session* from + * the keychain/config at runtime (so it keeps working across `me login`), and + * the space from `ME_SPACE`/active space at runtime — nothing secret or + * stateful is written into the config. + * - **Headless agent (`--api-key`):** the global key is baked in, along with a + * pinned `--space` (keys aren't space-bound, so the space must be fixed). + * + * Always uses bare `me` — the binary is expected to be on PATH whether installed + * via the install script, Homebrew, or npm. */ -export function buildMeCommand(apiKey: string, serverUrl: string): string[] { - return ["me", "mcp", "--api-key", apiKey, "--server", serverUrl]; +export function buildMeCommand(opts: { + server: string; + apiKey?: string; + space?: string; +}): string[] { + const cmd = ["me", "mcp", "--server", opts.server]; + if (opts.apiKey) cmd.push("--api-key", opts.apiKey); + if (opts.space) cmd.push("--space", opts.space); + return cmd; } // ============================================================================= diff --git a/packages/cli/mcp/server.ts b/packages/cli/mcp/server.ts index 8ba7cc3..3135e4b 100644 --- a/packages/cli/mcp/server.ts +++ b/packages/cli/mcp/server.ts @@ -8,14 +8,15 @@ import { existsSync, mkdirSync, statSync, writeFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { join, resolve } from "node:path"; +import { SHARE_NAMESPACE } from "@memory.build/protocol"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { stringify as yamlStringify } from "yaml"; import { z } from "zod"; import { CLIENT_VERSION } from "../../../version"; import { batchCreateChunked } from "../chunk.ts"; -import type { EngineClient } from "../client.ts"; -import { createClient } from "../client.ts"; +import type { MemoryClient } from "../client.ts"; +import { createMemoryClient } from "../client.ts"; import { formatMemoryAsMarkdown } from "../commands/memory.ts"; import { detectFormatFromExtension, @@ -47,7 +48,7 @@ Integration guide: ${DOCS_BASE}/mcp-integration.md`; // Tool Registration // ============================================================================= -function registerTools(server: McpServer, client: EngineClient): void { +function registerTools(server: McpServer, client: MemoryClient): void { // me_memory_create server.registerTool( "me_memory_create", @@ -72,10 +73,8 @@ Docs: ${docUrl("me_memory_create")}`, .describe("Key-value metadata pairs"), tree: z .string() - .optional() - .nullable() .describe( - "Hierarchical path (e.g., work.projects.me). Omit or null to store at the root.", + "Hierarchical path where the memory is stored (required — choose deliberately). Most memories should go under `share` (e.g. `share.work.projects`) so the rest of the space can see them. Use `~` — your private home (e.g. `~.notes`) — only for memories that must stay private to you.", ), temporal: z .object({ @@ -104,7 +103,7 @@ Docs: ${docUrl("me_memory_create")}`, id: args.id ?? undefined, content: args.content, meta: args.meta ?? undefined, - tree: args.tree ?? undefined, + tree: args.tree, temporal: args.temporal ? { start: args.temporal.start, @@ -584,9 +583,9 @@ Docs: ${docUrl("me_memory_import")}`, const format = (args.format as ImportFormat) ?? undefined; const allMemories: Array<{ content: string; + tree: string; id?: string; meta?: Record; - tree?: string; temporal?: { start: string; end?: string }; }> = []; @@ -627,9 +626,10 @@ Docs: ${docUrl("me_memory_import")}`, for (const mem of memories) { allMemories.push({ content: mem.content, + // tree is required on the wire; default bare records to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), }); } @@ -645,9 +645,10 @@ Docs: ${docUrl("me_memory_import")}`, for (const mem of memories) { allMemories.push({ content: mem.content, + // tree is required on the wire; default bare records to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), }); } @@ -657,9 +658,10 @@ Docs: ${docUrl("me_memory_import")}`, for (const mem of memories) { allMemories.push({ content: mem.content, + // tree is required on the wire; default bare records to `share`. + tree: mem.tree ?? SHARE_NAMESPACE, ...(mem.id ? { id: mem.id } : {}), ...(mem.meta ? { meta: mem.meta } : {}), - ...(mem.tree ? { tree: mem.tree } : {}), ...(mem.temporal ? { temporal: mem.temporal } : {}), }); } @@ -967,15 +969,23 @@ function setupShutdownHandlers(mcpServer: McpServer): void { // ============================================================================= export interface McpServerOptions { - apiKey: string; + /** Base server URL. */ server: string; + /** Bearer token — a session token (human) or an agent api key. */ + token: string; + /** Active space slug (sent as X-Me-Space). */ + space: string; } /** * Run MCP server over stdio. */ export async function runMcpServer(options: McpServerOptions): Promise { - const client = createClient({ url: options.server, apiKey: options.apiKey }); + const client = createMemoryClient({ + url: options.server, + token: options.token, + space: options.space, + }); const mcpServer = new McpServer( { diff --git a/packages/cli/serve/http-server.test.ts b/packages/cli/serve/http-server.test.ts index 95f627a..554e426 100644 --- a/packages/cli/serve/http-server.test.ts +++ b/packages/cli/serve/http-server.test.ts @@ -5,9 +5,10 @@ * upstream — no network access required. */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { SPACE_HEADER } from "@memory.build/protocol/headers"; import { - ENGINE_RPC_PATH, findAvailablePort, + MEMORY_RPC_PATH, type RunningServer, startHttpServer, } from "./http-server.ts"; @@ -72,8 +73,8 @@ describe("startHttpServer", () => { const servePort = await findAvailablePort("127.0.0.1", 34200); running = startHttpServer({ server: mock.url, - apiKey: "me.test.key", - engineSlug: "test-engine", + token: "sess-test-token", + space: "abc123def456", host: "127.0.0.1", port: servePort, }); @@ -127,18 +128,23 @@ describe("startHttpServer", () => { expect(json.result.ok).toBe(true); expect(mock.lastRequest?.method).toBe("POST"); - expect(mock.lastRequest?.path).toBe(ENGINE_RPC_PATH); + expect(mock.lastRequest?.path).toBe(MEMORY_RPC_PATH); expect(mock.lastRequest?.body).toBe(rpcBody); }); - test("/rpc injects Authorization: Bearer ", async () => { + test("/rpc injects Authorization: Bearer and X-Me-Space", async () => { await fetch(`${running.url}/rpc`, { method: "POST", headers: { "Content-Type": "application/json" }, body: "{}", }); - expect(mock.lastRequest?.headers.authorization).toBe("Bearer me.test.key"); + expect(mock.lastRequest?.headers.authorization).toBe( + "Bearer sess-test-token", + ); + expect(mock.lastRequest?.headers[SPACE_HEADER.toLowerCase()]).toBe( + "abc123def456", + ); }); test("/rpc surfaces upstream status codes", async () => { diff --git a/packages/cli/serve/http-server.ts b/packages/cli/serve/http-server.ts index 2b8cf1c..c7faa3c 100644 --- a/packages/cli/serve/http-server.ts +++ b/packages/cli/serve/http-server.ts @@ -3,23 +3,24 @@ * * Two concerns: * - * 1. Serve the web UI (placeholder HTML in step 2; embedded Vite build later). - * 2. Proxy `POST /rpc` to the configured engine's JSON-RPC endpoint with the - * stored API key injected (implemented in step 3). + * 1. Serve the embedded web UI (Vite build). + * 2. Proxy `POST /rpc` to the space memory JSON-RPC endpoint, injecting the + * session token (Authorization: Bearer) and the active space (X-Me-Space). * * The proxy is intentionally transparent — it forwards request bodies - * byte-for-byte and streams responses back, so any future `memory.*` RPC - * methods work without backend changes here. + * byte-for-byte and streams responses back, so any `memory.*` (and management) + * RPC methods work without backend changes here. */ +import { SPACE_HEADER } from "@memory.build/protocol/headers"; import { resolveAssetResponse } from "./web-assets.ts"; export interface ServeOptions { - /** Remote engine server URL (e.g., https://api.memory.build). */ + /** Remote server URL (e.g., https://api.memory.build). */ server: string; - /** API key for the active engine. Forwarded as Authorization: Bearer. */ - apiKey: string; - /** Active engine slug (for diagnostics / display). */ - engineSlug: string; + /** Session token. Forwarded as Authorization: Bearer. */ + token: string; + /** Active space slug. Forwarded as X-Me-Space. */ + space: string; /** Hostname to bind (defaults to 127.0.0.1). */ host: string; /** Port to bind. Use `findAvailablePort` first if you want auto-discovery. */ @@ -27,10 +28,10 @@ export interface ServeOptions { } /** - * Path on the remote server where the engine JSON-RPC endpoint lives. - * Kept as a constant so step 5's tests can assert the exact URL. + * Path on the remote server where the space memory JSON-RPC endpoint lives. + * Kept as a constant so tests can assert the exact URL. */ -export const ENGINE_RPC_PATH = "/api/v1/engine/rpc"; +export const MEMORY_RPC_PATH = "/api/v1/memory/rpc"; export interface RunningServer { /** The URL the server is listening on (e.g., http://127.0.0.1:3000). */ @@ -117,14 +118,15 @@ async function proxyRpc( req: Request, options: ServeOptions, ): Promise { - const targetUrl = new URL(ENGINE_RPC_PATH, options.server).toString(); + const targetUrl = new URL(MEMORY_RPC_PATH, options.server).toString(); // Rebuild the outgoing headers: drop hop-by-hop / host headers, keep - // Content-Type (the body needs it), and set our own Authorization. + // Content-Type (the body needs it), and set our own Authorization + space. const outHeaders = new Headers(); const contentType = req.headers.get("Content-Type"); if (contentType) outHeaders.set("Content-Type", contentType); - outHeaders.set("Authorization", `Bearer ${options.apiKey}`); + outHeaders.set("Authorization", `Bearer ${options.token}`); + outHeaders.set(SPACE_HEADER, options.space); outHeaders.set("Accept", "application/json"); try { diff --git a/packages/cli/util.test.ts b/packages/cli/util.test.ts index b405745..1059066 100644 --- a/packages/cli/util.test.ts +++ b/packages/cli/util.test.ts @@ -1,41 +1,73 @@ /** * Unit tests for CLI utility helpers. * - * Tests resolution functions with mocked clients. + * Tests the space-model resolution functions with mocked clients. */ import { describe, expect, mock, test } from "bun:test"; -import type { EngineClient } from "@memory.build/client"; +import type { MemoryClient, UserClient } from "@memory.build/client"; -// We test the exported functions via dynamic import to avoid -// pulling in @clack/prompts at top level (it touches process.stdin). -const { resolveUserId } = await import("./util.ts"); +// Dynamic import to avoid pulling in @clack/prompts at top level (it touches +// process.stdin). +const { resolveSpacePrincipalId, resolveAgentId } = await import("./util.ts"); + +const UUID = "019d694f-79f6-7595-8faf-b70b01c11f98"; // ============================================================================= -// resolveUserId +// resolveSpacePrincipalId // ============================================================================= -describe("resolveUserId", () => { - test("returns UUID as-is when input is a valid UUIDv7", async () => { - const engine = {} as EngineClient; // should not be called - const id = "019d694f-79f6-7595-8faf-b70b01c11f98"; - const result = await resolveUserId(engine, id); - expect(result).toBe(id); +describe("resolveSpacePrincipalId", () => { + test("returns a UUIDv7 as-is without resolving", async () => { + const memory = { + principal: { resolve: mock(() => Promise.reject(new Error("unused"))) }, + } as unknown as MemoryClient; + expect(await resolveSpacePrincipalId(memory, UUID, "text")).toBe(UUID); + expect(memory.principal.resolve).not.toHaveBeenCalled(); }); - test("resolves name via engine.user.getByName", async () => { - const engine = { - user: { - getByName: mock(() => + test("resolves a name via principal.resolve (with optional kind)", async () => { + const memory = { + principal: { + resolve: mock(() => Promise.resolve({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "alice", + principals: [{ id: UUID, kind: "g", name: "eng" }], }), ), }, - } as unknown as EngineClient; + } as unknown as MemoryClient; + + const id = await resolveSpacePrincipalId(memory, "eng", "text", "g"); + expect(id).toBe(UUID); + expect(memory.principal.resolve).toHaveBeenCalledWith({ + name: "eng", + kind: "g", + }); + }); +}); + +// ============================================================================= +// resolveAgentId +// ============================================================================= + +describe("resolveAgentId", () => { + test("returns a UUIDv7 as-is without listing agents", async () => { + const user = { + agent: { list: mock(() => Promise.reject(new Error("unused"))) }, + } as unknown as UserClient; + expect(await resolveAgentId(user, UUID, "text")).toBe(UUID); + expect(user.agent.list).not.toHaveBeenCalled(); + }); + + test("resolves a name via agent.list", async () => { + const user = { + agent: { + list: mock(() => + Promise.resolve({ agents: [{ id: UUID, name: "bot" }] }), + ), + }, + } as unknown as UserClient; - const result = await resolveUserId(engine, "alice"); - expect(result).toBe("019d694f-79f6-7595-8faf-b70b01c11f98"); - expect(engine.user.getByName).toHaveBeenCalledWith({ name: "alice" }); + expect(await resolveAgentId(user, "bot", "text")).toBe(UUID); + expect(user.agent.list).toHaveBeenCalled(); }); }); diff --git a/packages/cli/util.ts b/packages/cli/util.ts index 6c63eee..e658965 100644 --- a/packages/cli/util.ts +++ b/packages/cli/util.ts @@ -2,14 +2,14 @@ * Shared CLI utilities. * * Common patterns used across multiple command files: - * - Session token validation - * - Engine/API key validation - * - Org auto-resolution + * - Session / active-space validation + * - Memory / user client construction + * - Principal / agent resolution * - Error handling */ import * as clack from "@clack/prompts"; -import type { AccountsClient, EngineClient } from "./client.ts"; -import { RpcError } from "./client.ts"; +import type { MemoryClient, UserClient } from "./client.ts"; +import { createMemoryClient, createUserClient, RpcError } from "./client.ts"; import { clearSessionToken, type ResolvedCredentials } from "./credentials.ts"; import type { OutputFormat } from "./output.ts"; import { output } from "./output.ts"; @@ -19,6 +19,10 @@ const UUIDV7_RE = /** * Ensure the user has a session token. Exits with an error if not. + * + * Use this for the user endpoint (/api/v1/user/rpc), which is session-only — an + * api key never authenticates there (agents can't manage agents). For the memory + * endpoint, which accepts either bearer, use {@link requireMemoryAuth}. */ export function requireSession( creds: ResolvedCredentials, @@ -35,175 +39,101 @@ export function requireSession( } /** - * Ensure the user has an active engine with an API key. Exits with an error if not. + * Ensure the caller can authenticate to the memory endpoint + * (/api/v1/memory/rpc), which accepts either bearer: an agent api key + * (ME_API_KEY) or a human session token. Exits with an error if neither is + * present. Pair with {@link requireSpace}; then {@link buildMemoryClient} picks + * the bearer (api key first, mirroring `me mcp`). */ -export function requireEngine( +export function requireMemoryAuth( creds: ResolvedCredentials, fmt: OutputFormat, -): asserts creds is ResolvedCredentials & { apiKey: string } { - if (!creds.apiKey) { +): void { + if (!creds.apiKey && !creds.sessionToken) { + const msg = + "Not authenticated. Run 'me login', or set ME_API_KEY for an agent."; if (fmt === "text") { - clack.log.error("No active engine. Run 'me engine use' to select one."); + clack.log.error(msg); } else { - output({ error: "No active engine" }, fmt, () => {}); + output({ error: msg }, fmt, () => {}); } process.exit(1); } } -interface OrgInfo { - id: string; - name: string; - slug: string; -} - /** - * Resolve an org from a flag, positional argument, or auto-resolution. - * - * Accepts a UUID, name, or slug. Falls back to auto-resolution if only one org. - * Priority: positionalArg > flagValue > auto-resolve (if exactly one org). - * Exits with an error if the org cannot be determined. + * Ensure an active space (the X-Me-Space) is selected. Exits with an error if + * not. Used by the space-scoped commands (memory, group, access, …). */ -export async function resolveOrg( - accounts: AccountsClient, +export function requireSpace( + creds: ResolvedCredentials, fmt: OutputFormat, - flagValue?: string, - positionalArg?: string, -): Promise { - const { orgs } = await accounts.org.list(); - const input = positionalArg ?? flagValue; - - if (input) { - // Try UUID match first - if (UUIDV7_RE.test(input)) { - const match = orgs.find((o) => o.id === input); - if (match) return match; - // Might be a valid org ID the user isn't a member of — use it as-is - return { id: input, name: input, slug: input }; - } - - // Match by name or slug (case-insensitive) - const lower = input.toLowerCase(); - const matches = orgs.filter( - (o) => o.name.toLowerCase() === lower || o.slug.toLowerCase() === lower, - ); - - if (matches.length === 1 && matches[0]) return matches[0]; - - if (matches.length === 0) { - const msg = `No organization found matching '${input}'.`; - if (fmt === "text") { - clack.log.error(msg); - if (orgs.length > 0) { - console.log(" Your organizations:"); - for (const org of orgs) { - console.log(` ${org.name} (${org.slug})`); - } - } - } else { - output({ error: msg, orgs }, fmt, () => {}); - } - process.exit(1); - } - - // Multiple matches (same name, different orgs) - const msg = `Multiple organizations match '${input}'. Use the org ID instead:`; - if (fmt === "text") { - clack.log.error(msg); - for (const org of matches) { - console.log(` ${org.name} (${org.slug}) — ${org.id}`); - } - } else { - output({ error: msg, orgs: matches }, fmt, () => {}); - } - process.exit(1); - } - - // Auto-resolve: pick if exactly one - if (orgs.length === 1 && orgs[0]) return orgs[0]; - - if (orgs.length === 0) { - const msg = "You don't belong to any organizations."; +): asserts creds is ResolvedCredentials & { activeSpace: string } { + if (!creds.activeSpace) { if (fmt === "text") { - clack.log.error(msg); + clack.log.error( + "No active space. Run 'me space use ' to select one, or set ME_SPACE.", + ); } else { - output({ error: msg }, fmt, () => {}); + output({ error: "No active space" }, fmt, () => {}); } process.exit(1); } - - // Multiple orgs — can't auto-resolve - const msg = - "You belong to multiple organizations. Use --org to specify which one."; - if (fmt === "text") { - clack.log.error(msg); - for (const org of orgs) { - console.log(` ${org.name} (${org.slug}) — ${org.id}`); - } - } else { - output( - { - error: msg, - orgs: orgs.map((o: { id: string; name: string; slug: string }) => ({ - id: o.id, - name: o.name, - slug: o.slug, - })), - }, - fmt, - () => {}, - ); - } - process.exit(1); } /** - * Resolve an org ID from a flag, positional argument, or auto-resolution. - * - * Convenience wrapper around resolveOrg — returns just the ID. + * Build a user client (session-only, /api/v1/user/rpc). Call requireSession + * first so the token is present. */ -export async function resolveOrgId( - accounts: AccountsClient, - fmt: OutputFormat, - flagValue?: string, - positionalArg?: string, -): Promise { - const org = await resolveOrg(accounts, fmt, flagValue, positionalArg); - return org.id; +export function buildUserClient( + creds: ResolvedCredentials & { sessionToken: string }, +): UserClient { + return createUserClient({ url: creds.server, token: creds.sessionToken }); } /** - * Resolve a user or role by ID or name. If the argument looks like a UUIDv7, - * fetches by ID; otherwise fetches by name. Returns the UUID. + * Build a memory client (bearer + active space, /api/v1/memory/rpc). Call + * requireMemoryAuth and requireSpace first so a bearer and a space are present. + * The bearer is the agent api key when set (ME_API_KEY), else the session token + * — the memory endpoint accepts either, and this mirrors `me mcp`'s precedence. */ -export async function resolveUserId( - engine: EngineClient, - idOrName: string, -): Promise { - if (UUIDV7_RE.test(idOrName)) return idOrName; - const user = await engine.user.getByName({ name: idOrName }); - return user.id; +export function buildMemoryClient( + creds: ResolvedCredentials & { activeSpace: string }, +): MemoryClient { + return createMemoryClient({ + url: creds.server, + token: creds.apiKey ?? creds.sessionToken, + space: creds.activeSpace, + // Bulk imports send 1000-memory batchCreate chunks that the server + // processes row-by-row; on a loaded server (or one far from its + // database) a chunk can legitimately exceed the client's 30s default. + timeout: 120_000, + }); } /** - * Resolve an identity ID from an email, name, or UUID. - * - * - UUID: used as-is - * - Email (contains @): looked up via identity.getByEmail - * - Otherwise: error with guidance + * Resolve a principal in the active space to its id. Accepts a UUIDv7 (used + * as-is) or a name — for users the name is their email; for agents/groups it is + * the display name. Optionally constrained to a kind ('u' | 'a' | 'g'). Uses + * principal.resolve (a targeted lookup any space member may call). Exits with an + * actionable error on miss / ambiguity. */ -export async function resolveIdentityId( - accounts: AccountsClient, - fmt: OutputFormat, +export async function resolveSpacePrincipalId( + memory: MemoryClient, input: string, + fmt: OutputFormat, + kind?: "u" | "a" | "g", ): Promise { if (UUIDV7_RE.test(input)) return input; - if (input.includes("@")) { - const { identity } = await accounts.identity.getByEmail({ email: input }); - if (identity) return identity.id; + const { principals } = await memory.principal.resolve( + kind ? { name: input, kind } : { name: input }, + ); + + if (principals.length === 1 && principals[0]) return principals[0].id; - const msg = `No identity found with email '${input}'. They may need to sign up first, or use 'me invitation create' to invite them.`; + if (principals.length === 0) { + const msg = `No ${kind === "g" ? "group" : "principal"} named '${input}' in this space.`; if (fmt === "text") { clack.log.error(msg); } else { @@ -212,96 +142,63 @@ export async function resolveIdentityId( process.exit(1); } - const msg = `'${input}' is not a valid ID or email. Provide a UUID or email address.`; + const msg = `Multiple principals named '${input}'. Use the id instead:`; if (fmt === "text") { clack.log.error(msg); + for (const m of principals) + console.log(` ${m.name} (${m.kind}) — ${m.id}`); } else { - output({ error: msg }, fmt, () => {}); + output({ error: msg, matches: principals }, fmt, () => {}); } process.exit(1); } /** - * Resolve an identity from an org's member list by name, email, or UUID. - * - * Used for operations on existing members (e.g., remove). + * Resolve one of the caller's agents to its id, by UUIDv7 or name (agent names + * are unique per user). Exits with an actionable error on miss / ambiguity. */ -export async function resolveMember( - accounts: AccountsClient, - fmt: OutputFormat, - orgId: string, +export async function resolveAgentId( + user: UserClient, input: string, -): Promise<{ identityId: string; name: string; email: string }> { - const { members } = await accounts.org.member.list({ orgId }); - - // UUID match - if (UUIDV7_RE.test(input)) { - const match = members.find((m) => m.identityId === input); - if (match) return match; - // UUID not in member list — return it as-is (server will error if invalid) - return { identityId: input, name: input, email: "" }; - } - - // Email match - if (input.includes("@")) { - const lower = input.toLowerCase(); - const match = members.find((m) => m.email.toLowerCase() === lower); - if (match) return match; - - const msg = `No member with email '${input}' in this organization.`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - - // Name match + fmt: OutputFormat, +): Promise { + if (UUIDV7_RE.test(input)) return input; + const { agents } = await user.agent.list(); const lower = input.toLowerCase(); - const matches = members.filter((m) => m.name.toLowerCase() === lower); + const matches = agents.filter((a) => a.name.toLowerCase() === lower); + if (matches.length === 1 && matches[0]) return matches[0].id; - if (matches.length === 1 && matches[0]) return matches[0]; - - if (matches.length === 0) { - const msg = `No member named '${input}' in this organization.`; - if (fmt === "text") { - clack.log.error(msg); - } else { - output({ error: msg }, fmt, () => {}); - } - process.exit(1); - } - - // Multiple matches - const msg = `Multiple members named '${input}'. Use their email instead:`; + const msg = + matches.length === 0 + ? `No agent named '${input}'. Run 'me agent list'.` + : `Multiple agents named '${input}'. Use the agent id instead.`; if (fmt === "text") { clack.log.error(msg); - for (const m of matches) { - console.log(` ${m.name} — ${m.email}`); - } + if (matches.length > 1) + for (const a of matches) console.log(` ${a.name} — ${a.id}`); } else { - output({ error: msg, members: matches }, fmt, () => {}); + output({ error: msg, matches }, fmt, () => {}); } process.exit(1); } /** - * Detect an authentication error from the server. - * - * The server's `unauthorized()` helper sends an HTTP 401 with body - * `{"error":{"message": "...", "code": "UNAUTHORIZED"}}`. The transport wraps - * that into an RpcError. The string code lands either on `data.code` - * (`appCode`) or on `code` itself depending on which envelope path the - * response took, so we check both. + * True when `error` is a server AppError with the given string code. The code + * lands either on `data.code` (`appCode`) or on `code` itself depending on which + * envelope path the response took (RpcError types `code` as number, but the + * runtime value can be the string code when the response wasn't a strict + * JSON-RPC envelope), so we check both. */ -function isUnauthorized(error: unknown): boolean { +export function isAppErrorCode(error: unknown, code: string): boolean { if (!(error instanceof RpcError)) return false; - if (error.appCode === "UNAUTHORIZED") return true; - // The server's HTTP error envelope puts the string code on the top-level - // `code` field. RpcError types `code` as number, but the runtime value can - // be a string when the response wasn't a strict JSON-RPC envelope. - return (error.code as unknown) === "UNAUTHORIZED"; + return error.appCode === code || (error.code as unknown) === code; +} + +/** + * Detect an authentication error from the server (HTTP 401 / `UNAUTHORIZED`). + */ +function isUnauthorized(error: unknown): boolean { + return isAppErrorCode(error, "UNAUTHORIZED"); } /** diff --git a/packages/client/accounts.ts b/packages/client/accounts.ts deleted file mode 100644 index eebb3d4..0000000 --- a/packages/client/accounts.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Accounts client — for managing organizations, engines, and invitations. - * - * Authenticated via session token (obtained from the device flow login). - * Typically used by the CLI, not by end-user applications. - * - * @example - * ```ts - * import { createAccountsClient } from "@memory.build/client"; - * - * const accounts = createAccountsClient({ sessionToken: "..." }); - * - * const identity = await accounts.me.get(); - * const orgs = await accounts.org.list(); - * ``` - */ -import type { - AccountsMethodName, - AccountsParams, - AccountsResult, - EngineCreateParams, - EngineDeleteParams, - EngineDeleteResult, - EngineGetParams, - EngineListParams, - EngineListResult, - EngineResponse, - EngineSetupAccessParams, - EngineSetupAccessResult, - EngineUpdateParams, - IdentityGetByEmailParams, - IdentityGetByEmailResult, - IdentityResponse, - InvitationAcceptParams, - InvitationAcceptResult, - InvitationCreateParams, - InvitationCreateResult, - InvitationListParams, - InvitationListResult, - InvitationRevokeParams, - InvitationRevokeResult, - OrgCreateParams, - OrgDeleteParams, - OrgDeleteResult, - OrgGetParams, - OrgListResult, - OrgMemberAddParams, - OrgMemberListParams, - OrgMemberListResult, - OrgMemberRemoveParams, - OrgMemberRemoveResult, - OrgMemberResponse, - OrgMemberUpdateRoleParams, - OrgMemberUpdateRoleResult, - OrgResponse, - OrgUpdateParams, - SessionRevokeResult, -} from "@memory.build/protocol/accounts"; -import { rpcCall, type TransportConfig } from "./transport.ts"; - -// ============================================================================= -// Options -// ============================================================================= - -/** - * Options for creating an accounts client. - */ -export interface AccountsClientOptions { - /** Base URL of the Memory Engine server (default: "https://api.memory.build") */ - url?: string; - /** Session token for authentication */ - sessionToken?: string; - /** Request timeout in milliseconds (default: 30000) */ - timeout?: number; - /** Maximum retry attempts for transient failures (default: 3) */ - retries?: number; - /** - * CLIENT_VERSION of the caller. When set, sent as the `X-Client-Version` - * header on every RPC so the server can reject too-old clients with a - * typed `CLIENT_VERSION_INCOMPATIBLE` error before dispatch. - */ - clientVersion?: string; -} - -// ============================================================================= -// Namespace Types -// ============================================================================= - -export interface MeNamespace { - get(): Promise; -} - -export interface IdentityNamespace { - getByEmail( - params: IdentityGetByEmailParams, - ): Promise; -} - -export interface SessionNamespace { - revoke(): Promise; -} - -export interface OrgNamespace { - create(params: OrgCreateParams): Promise; - list(): Promise; - get(params: OrgGetParams): Promise; - update(params: OrgUpdateParams): Promise; - delete(params: OrgDeleteParams): Promise; - member: OrgMemberNamespace; -} - -export interface OrgMemberNamespace { - list(params: OrgMemberListParams): Promise; - add(params: OrgMemberAddParams): Promise; - remove(params: OrgMemberRemoveParams): Promise; - updateRole( - params: OrgMemberUpdateRoleParams, - ): Promise; -} - -export interface AccountsEngineNamespace { - create(params: EngineCreateParams): Promise; - list(params: EngineListParams): Promise; - get(params: EngineGetParams): Promise; - update(params: EngineUpdateParams): Promise; - delete(params: EngineDeleteParams): Promise; - setupAccess( - params: EngineSetupAccessParams, - ): Promise; -} - -export interface InvitationNamespace { - create(params: InvitationCreateParams): Promise; - list(params: InvitationListParams): Promise; - revoke(params: InvitationRevokeParams): Promise; - accept(params: InvitationAcceptParams): Promise; -} - -// ============================================================================= -// Client Type -// ============================================================================= - -/** - * Accounts client. - */ -export interface AccountsClient { - /** Current identity */ - me: MeNamespace; - /** Identity lookup */ - identity: IdentityNamespace; - /** Session management */ - session: SessionNamespace; - /** Organization management */ - org: OrgNamespace; - /** Engine management */ - engine: AccountsEngineNamespace; - /** Invitation management */ - invitation: InvitationNamespace; - - /** - * Low-level typed RPC call. - * Prefer the namespace methods for convenience. - */ - call( - method: M, - params: AccountsParams, - ): Promise>; - - /** Update the session token at runtime. */ - setSessionToken(token: string): void; - /** Get the current session token. */ - getSessionToken(): string | undefined; -} - -// ============================================================================= -// Factory -// ============================================================================= - -const DEFAULT_URL = "https://api.memory.build"; -const ACCOUNTS_RPC_PATH = "/api/v1/accounts/rpc"; -const DEFAULT_TIMEOUT = 30_000; -const DEFAULT_RETRIES = 3; - -/** - * Create an accounts client. - * - * Used for managing organizations, engines, members, and invitations. - * Requires a session token obtained from the device flow login. - * - * @example - * ```ts - * const accounts = createAccountsClient({ sessionToken: "..." }); - * - * const identity = await accounts.me.get(); - * const { orgs } = await accounts.org.list(); - * ``` - */ -export function createAccountsClient( - options: AccountsClientOptions = {}, -): AccountsClient { - const config: TransportConfig = { - url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), - path: ACCOUNTS_RPC_PATH, - token: options.sessionToken, - timeout: options.timeout ?? DEFAULT_TIMEOUT, - retries: options.retries ?? DEFAULT_RETRIES, - clientVersion: options.clientVersion, - }; - - function call( - method: M, - params: AccountsParams, - ): Promise> { - return rpcCall>(config, method, params); - } - - const member: OrgMemberNamespace = { - list: (params) => call("org.member.list", params), - add: (params) => call("org.member.add", params), - remove: (params) => call("org.member.remove", params), - updateRole: (params) => call("org.member.updateRole", params), - }; - - const me: MeNamespace = { - get: () => call("me.get", {}), - }; - - const identity: IdentityNamespace = { - getByEmail: (params) => call("identity.getByEmail", params), - }; - - const session: SessionNamespace = { - revoke: () => call("session.revoke", {}), - }; - - const org: OrgNamespace = { - create: (params) => call("org.create", params), - list: () => call("org.list", {}), - get: (params) => call("org.get", params), - update: (params) => call("org.update", params), - delete: (params) => call("org.delete", params), - member, - }; - - const engine: AccountsEngineNamespace = { - create: (params) => call("engine.create", params), - list: (params) => call("engine.list", params), - get: (params) => call("engine.get", params), - update: (params) => call("engine.update", params), - delete: (params) => call("engine.delete", params), - setupAccess: (params) => call("engine.setupAccess", params), - }; - - const invitation: InvitationNamespace = { - create: (params) => call("invitation.create", params), - list: (params) => call("invitation.list", params), - revoke: (params) => call("invitation.revoke", params), - accept: (params) => call("invitation.accept", params), - }; - - return { - me, - identity, - session, - org, - engine, - invitation, - call, - setSessionToken(token: string) { - config.token = token; - }, - getSessionToken() { - return config.token; - }, - }; -} diff --git a/packages/client/engine.ts b/packages/client/engine.ts deleted file mode 100644 index fe3e9d5..0000000 --- a/packages/client/engine.ts +++ /dev/null @@ -1,328 +0,0 @@ -/** - * Engine client — the primary client for interacting with Memory Engine. - * - * Provides typed, namespaced access to all 34 engine RPC methods - * (memory, user, grant, owner, role, apiKey) authenticated via API key. - * - * @example - * ```ts - * import { createClient } from "@memory.build/client"; - * - * const me = createClient({ apiKey: "me.xxx.yyy" }); - * - * const memory = await me.memory.create({ content: "hello world" }); - * const results = await me.memory.search({ semantic: "hello" }); - * const tree = await me.memory.tree(); - * ``` - */ -import type { - ApiKeyCreateParams, - ApiKeyCreateResult, - ApiKeyDeleteParams, - ApiKeyDeleteResult, - ApiKeyGetParams, - ApiKeyListParams, - ApiKeyListResult, - ApiKeyResponse, - ApiKeyRevokeParams, - ApiKeyRevokeResult, - EngineMethodName, - EngineParams, - EngineResult, - GrantCheckParams, - GrantCheckResult, - GrantCreateParams, - GrantCreateResult, - GrantGetParams, - GrantListParams, - GrantListResult, - GrantResponse, - GrantRevokeParams, - GrantRevokeResult, - MemoryBatchCreateParams, - MemoryBatchCreateResult, - MemoryCountTreeParams, - MemoryCountTreeResult, - MemoryCreateParams, - MemoryDeleteParams, - MemoryDeleteResult, - MemoryDeleteTreeParams, - MemoryDeleteTreeResult, - MemoryGetParams, - MemoryMoveParams, - MemoryMoveResult, - MemoryResponse, - MemorySearchParams, - MemorySearchResult, - MemoryTreeParams, - MemoryTreeResult, - MemoryUpdateParams, - OwnerGetParams, - OwnerListParams, - OwnerListResult, - OwnerRemoveParams, - OwnerRemoveResult, - OwnerResponse, - OwnerSetParams, - OwnerSetResult, - RoleAddMemberParams, - RoleAddMemberResult, - RoleCreateParams, - RoleListForUserParams, - RoleListForUserResult, - RoleListMembersParams, - RoleListMembersResult, - RoleRemoveMemberParams, - RoleRemoveMemberResult, - RoleResponse, - UserCreateParams, - UserDeleteParams, - UserDeleteResult, - UserGetByNameParams, - UserGetParams, - UserListParams, - UserListResult, - UserRenameParams, - UserRenameResult, - UserResponse, -} from "@memory.build/protocol/engine"; -import { rpcCall, type TransportConfig } from "./transport.ts"; - -// ============================================================================= -// Options -// ============================================================================= - -/** - * Options for creating an engine client. - */ -export interface ClientOptions { - /** Base URL of the Memory Engine server (default: "https://api.memory.build") */ - url?: string; - /** Engine JSON-RPC endpoint path (default: "/api/v1/engine/rpc") */ - rpcPath?: string; - /** API key for authentication (format: "me.lookupId.secret") */ - apiKey?: string; - /** Request timeout in milliseconds (default: 30000) */ - timeout?: number; - /** Maximum retry attempts for transient failures (default: 3) */ - retries?: number; - /** - * CLIENT_VERSION of the caller. When set, sent as the `X-Client-Version` - * header on every RPC so the server can reject too-old clients with a - * typed `CLIENT_VERSION_INCOMPATIBLE` error before dispatch. - */ - clientVersion?: string; -} - -// ============================================================================= -// Namespace Types -// ============================================================================= - -export interface MemoryNamespace { - create(params: MemoryCreateParams): Promise; - batchCreate( - params: MemoryBatchCreateParams, - ): Promise; - get(params: MemoryGetParams): Promise; - update(params: MemoryUpdateParams): Promise; - delete(params: MemoryDeleteParams): Promise; - search(params: MemorySearchParams): Promise; - tree(params?: MemoryTreeParams): Promise; - move(params: MemoryMoveParams): Promise; - deleteTree(params: MemoryDeleteTreeParams): Promise; - countTree(params: MemoryCountTreeParams): Promise; -} - -export interface UserNamespace { - create(params: UserCreateParams): Promise; - get(params: UserGetParams): Promise; - getByName(params: UserGetByNameParams): Promise; - list(params?: UserListParams): Promise; - rename(params: UserRenameParams): Promise; - delete(params: UserDeleteParams): Promise; -} - -export interface GrantNamespace { - create(params: GrantCreateParams): Promise; - list(params?: GrantListParams): Promise; - get(params: GrantGetParams): Promise; - revoke(params: GrantRevokeParams): Promise; - check(params: GrantCheckParams): Promise; -} - -export interface RoleNamespace { - create(params: RoleCreateParams): Promise; - addMember(params: RoleAddMemberParams): Promise; - removeMember(params: RoleRemoveMemberParams): Promise; - listMembers(params: RoleListMembersParams): Promise; - listForUser(params: RoleListForUserParams): Promise; -} - -export interface OwnerNamespace { - set(params: OwnerSetParams): Promise; - get(params: OwnerGetParams): Promise; - remove(params: OwnerRemoveParams): Promise; - list(params?: OwnerListParams): Promise; -} - -export interface ApiKeyNamespace { - create(params: ApiKeyCreateParams): Promise; - get(params: ApiKeyGetParams): Promise; - list(params: ApiKeyListParams): Promise; - revoke(params: ApiKeyRevokeParams): Promise; - delete(params: ApiKeyDeleteParams): Promise; -} - -// ============================================================================= -// Client Type -// ============================================================================= - -/** - * Memory Engine client. - */ -export interface EngineClient { - /** Memory operations (create, search, tree, etc.) */ - memory: MemoryNamespace; - /** User management */ - user: UserNamespace; - /** Tree grant management */ - grant: GrantNamespace; - /** Role management */ - role: RoleNamespace; - /** Tree owner management */ - owner: OwnerNamespace; - /** API key management */ - apiKey: ApiKeyNamespace; - - /** - * Low-level typed RPC call. - * Prefer the namespace methods for convenience. - */ - call( - method: M, - params: EngineParams, - ): Promise>; - - /** Update the API key at runtime. */ - setApiKey(apiKey: string): void; - /** Get the current API key. */ - getApiKey(): string | undefined; -} - -// ============================================================================= -// Factory -// ============================================================================= - -const DEFAULT_URL = "https://api.memory.build"; -const ENGINE_RPC_PATH = "/api/v1/engine/rpc"; -const DEFAULT_TIMEOUT = 30_000; -const DEFAULT_RETRIES = 3; - -/** - * Create a Memory Engine client. - * - * This is the primary entry point for interacting with Memory Engine. - * It connects to the engine RPC endpoint using API key authentication. - * - * @example - * ```ts - * const me = createClient({ apiKey: "me.xxx.yyy" }); - * - * // Create a memory - * const memory = await me.memory.create({ - * content: "TypeScript was released in 2012", - * tree: "knowledge.programming", - * }); - * - * // Search memories - * const results = await me.memory.search({ - * semantic: "when was TypeScript created", - * }); - * ``` - */ -export function createClient(options: ClientOptions = {}): EngineClient { - const config: TransportConfig = { - url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), - path: options.rpcPath ?? ENGINE_RPC_PATH, - token: options.apiKey, - timeout: options.timeout ?? DEFAULT_TIMEOUT, - retries: options.retries ?? DEFAULT_RETRIES, - clientVersion: options.clientVersion, - }; - - function call( - method: M, - params: EngineParams, - ): Promise> { - return rpcCall>(config, method, params); - } - - const memory: MemoryNamespace = { - create: (params) => call("memory.create", params), - batchCreate: (params) => call("memory.batchCreate", params), - get: (params) => call("memory.get", params), - update: (params) => call("memory.update", params), - delete: (params) => call("memory.delete", params), - search: (params) => call("memory.search", params), - tree: (params) => call("memory.tree", params ?? {}), - move: (params) => call("memory.move", params), - deleteTree: (params) => call("memory.deleteTree", params), - countTree: (params) => call("memory.countTree", params), - }; - - const user: UserNamespace = { - create: (params) => call("user.create", params), - get: (params) => call("user.get", params), - getByName: (params) => call("user.getByName", params), - list: (params) => call("user.list", params ?? {}), - rename: (params) => call("user.rename", params), - delete: (params) => call("user.delete", params), - }; - - const grant: GrantNamespace = { - create: (params) => call("grant.create", params), - list: (params) => call("grant.list", params ?? {}), - get: (params) => call("grant.get", params), - revoke: (params) => call("grant.revoke", params), - check: (params) => call("grant.check", params), - }; - - const role: RoleNamespace = { - create: (params) => call("role.create", params), - addMember: (params) => call("role.addMember", params), - removeMember: (params) => call("role.removeMember", params), - listMembers: (params) => call("role.listMembers", params), - listForUser: (params) => call("role.listForUser", params), - }; - - const owner: OwnerNamespace = { - set: (params) => call("owner.set", params), - get: (params) => call("owner.get", params), - remove: (params) => call("owner.remove", params), - list: (params) => call("owner.list", params ?? {}), - }; - - const apiKey: ApiKeyNamespace = { - create: (params) => call("apiKey.create", params), - get: (params) => call("apiKey.get", params), - list: (params) => call("apiKey.list", params), - revoke: (params) => call("apiKey.revoke", params), - delete: (params) => call("apiKey.delete", params), - }; - - return { - memory, - user, - grant, - role, - owner, - apiKey, - call, - setApiKey(apiKey: string) { - config.token = apiKey; - }, - getApiKey() { - return config.token; - }, - }; -} diff --git a/packages/client/index.ts b/packages/client/index.ts index 7784c91..d015b53 100644 --- a/packages/client/index.ts +++ b/packages/client/index.ts @@ -1,22 +1,23 @@ /** * @memory.build/client — Client library for Memory Engine. * - * Three clients for different use cases: + * Two clients, both authenticated by a bearer token: * - * - {@link createClient} — Engine client (API key auth). - * The primary client for memory operations, search, user/grant management. + * - {@link createMemoryClient} — space data-plane + management. + * Talks to /api/v1/memory/rpc with the active space carried as X-Me-Space. + * Memory CRUD/search plus principal/group/grant/invite management. * - * - {@link createAccountsClient} — Accounts client (session token auth). - * For managing organizations, engines, and invitations. Used by CLI. + * - {@link createUserClient} — session-only, user-scoped. + * Talks to /api/v1/user/rpc: whoami, agent lifecycle, api keys, space discovery. * - * - {@link createAuthClient} — Auth client (no auth). + * - {@link createAuthClient} — auth client (no auth). * OAuth device flow for CLI login. Returns a session token. * * @example * ```ts - * import { createClient } from "@memory.build/client"; + * import { createMemoryClient } from "@memory.build/client"; * - * const me = createClient({ apiKey: "me.xxx.yyy" }); + * const me = createMemoryClient({ token: sessionToken, space: "abc123def456" }); * * await me.memory.create({ * content: "TypeScript was released in 2012", @@ -29,44 +30,39 @@ * ``` */ -export type * from "@memory.build/protocol/engine"; export type { Meta, SearchWeights, Temporal, TemporalFilter, } from "@memory.build/protocol/fields"; +export type * from "@memory.build/protocol/memory"; -export type { - AccountsClient, - AccountsClientOptions, - AccountsEngineNamespace, - InvitationNamespace, - MeNamespace, - OrgMemberNamespace, - OrgNamespace, - SessionNamespace, -} from "./accounts.ts"; -// Accounts client -export { createAccountsClient } from "./accounts.ts"; export type { AuthClient, AuthClientOptions, PollOptions } from "./auth.ts"; // Auth client export { createAuthClient, DeviceFlowError } from "./auth.ts"; -export type { - ApiKeyNamespace, - ClientOptions, - EngineClient, - GrantNamespace, - MemoryNamespace, - OwnerNamespace, - RoleNamespace, - UserNamespace, -} from "./engine.ts"; -// Engine client (primary) -export { createClient } from "./engine.ts"; - // Errors export { isRpcError, RpcError } from "./errors.ts"; +// Memory client (space data-plane + management) +export { + createMemoryClient, + type GrantNamespace, + type GroupNamespace, + type InviteNamespace, + type MemoryClient, + type MemoryClientOptions, + type MemoryNamespace, + type PrincipalNamespace, +} from "./memory.ts"; +// User client (session-only: whoami, agent lifecycle, api keys, space discovery) +export { + type AgentNamespace, + type ApiKeyNamespace, + createUserClient, + type SpaceNamespace, + type UserClient, + type UserClientOptions, +} from "./user.ts"; // Version compatibility check export { type CheckServerVersionOptions, diff --git a/packages/client/memory.test.ts b/packages/client/memory.test.ts new file mode 100644 index 0000000..cb4867a --- /dev/null +++ b/packages/client/memory.test.ts @@ -0,0 +1,72 @@ +import { afterEach, expect, test } from "bun:test"; +import { createMemoryClient } from "./memory.ts"; +import { createUserClient } from "./user.ts"; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function captureFetch() { + const captured = { headers: {} as Record, url: "" }; + globalThis.fetch = (async ( + input: string | URL | Request, + init?: RequestInit, + ) => { + captured.url = typeof input === "string" ? input : input.toString(); + const headers = init?.headers as Record | undefined; + if (headers) Object.assign(captured.headers, headers); + return new Response(JSON.stringify({ jsonrpc: "2.0", id: 1, result: {} }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }) as typeof fetch; + return captured; +} + +test("memory client sends X-Me-Space and Bearer token to the memory endpoint", async () => { + const captured = captureFetch(); + const client = createMemoryClient({ + url: "https://api.example.com", + token: "sess-tok", + space: "abc123def456", + retries: 0, + }); + + await client.principal.list({}); + + expect(captured.url).toBe("https://api.example.com/api/v1/memory/rpc"); + expect(captured.headers["X-Me-Space"]).toBe("abc123def456"); + expect(captured.headers.Authorization).toBe("Bearer sess-tok"); +}); + +test("memory client setSpace updates the X-Me-Space header", async () => { + const captured = captureFetch(); + const client = createMemoryClient({ + url: "https://api.example.com", + token: "t", + space: "aaaaaaaaaaaa", + retries: 0, + }); + client.setSpace("bbbbbbbbbbbb"); + + await client.memory.tree(); + + expect(captured.headers["X-Me-Space"]).toBe("bbbbbbbbbbbb"); +}); + +test("user client targets the user endpoint with no X-Me-Space", async () => { + const captured = captureFetch(); + const client = createUserClient({ + url: "https://api.example.com", + token: "sess-tok", + retries: 0, + }); + + await client.space.list(); + + expect(captured.url).toBe("https://api.example.com/api/v1/user/rpc"); + expect(captured.headers["X-Me-Space"]).toBeUndefined(); + expect(captured.headers.Authorization).toBe("Bearer sess-tok"); +}); diff --git a/packages/client/memory.ts b/packages/client/memory.ts new file mode 100644 index 0000000..377c4af --- /dev/null +++ b/packages/client/memory.ts @@ -0,0 +1,231 @@ +/** + * Memory client — the space data-plane + management client. + * + * Talks to POST /api/v1/memory/rpc, authenticated by a session token (human) or + * an api key (agent), with the active space selected via the X-Me-Space header. + * Namespaces: memory (data plane) + principal / group / grant / invite (management). + * (Agent lifecycle and api keys live on the user client.) + * + * @example + * ```ts + * const me = createMemoryClient({ token: sessionToken, space: "abc123def456" }); + * await me.memory.create({ content: "hello", tree: "notes" }); + * await me.principal.list({}); + * ``` + */ + +import { SPACE_HEADER } from "@memory.build/protocol/headers"; +import type { + MemoryBatchCreateParams, + MemoryBatchCreateResult, + MemoryCountTreeParams, + MemoryCountTreeResult, + MemoryCreateParams, + MemoryDeleteParams, + MemoryDeleteResult, + MemoryDeleteTreeParams, + MemoryDeleteTreeResult, + MemoryGetParams, + MemoryMoveParams, + MemoryMoveResult, + MemoryResponse, + MemorySearchParams, + MemorySearchResult, + MemoryTreeParams, + MemoryTreeResult, + MemoryUpdateParams, +} from "@memory.build/protocol/memory"; +import type { + GrantListParams, + GrantListResult, + GrantRemoveParams, + GrantRemoveResult, + GrantSetParams, + GrantSetResult, + GroupAddMemberParams, + GroupAddMemberResult, + GroupCreateParams, + GroupCreateResult, + GroupDeleteParams, + GroupDeleteResult, + GroupListForMemberParams, + GroupListForMemberResult, + GroupListMembersParams, + GroupListMembersResult, + GroupListParams, + GroupListResult, + GroupRemoveMemberParams, + GroupRemoveMemberResult, + GroupRenameParams, + GroupRenameResult, + InviteCreateParams, + InviteCreateResult, + InviteListParams, + InviteListResult, + InviteRevokeParams, + InviteRevokeResult, + PrincipalAddParams, + PrincipalAddResult, + PrincipalListParams, + PrincipalListResult, + PrincipalLookupParams, + PrincipalLookupResult, + PrincipalRemoveParams, + PrincipalRemoveResult, + PrincipalResolveParams, + PrincipalResolveResult, +} from "@memory.build/protocol/space"; +import { rpcCall, type TransportConfig } from "./transport.ts"; + +export interface MemoryClientOptions { + /** Base URL of the server (default: "https://api.memory.build") */ + url?: string; + /** Memory RPC endpoint path (default: "/api/v1/memory/rpc") */ + rpcPath?: string; + /** Bearer token: a session token (human) or an api key (agent). */ + token?: string; + /** The active space slug, sent as X-Me-Space. */ + space?: string; + /** Request timeout in milliseconds (default: 30000) */ + timeout?: number; + /** Maximum retry attempts for transient failures (default: 3) */ + retries?: number; + /** CLIENT_VERSION of the caller (sent as X-Client-Version). */ + clientVersion?: string; +} + +export interface MemoryNamespace { + create(params: MemoryCreateParams): Promise; + batchCreate( + params: MemoryBatchCreateParams, + ): Promise; + get(params: MemoryGetParams): Promise; + update(params: MemoryUpdateParams): Promise; + delete(params: MemoryDeleteParams): Promise; + search(params: MemorySearchParams): Promise; + tree(params?: MemoryTreeParams): Promise; + move(params: MemoryMoveParams): Promise; + deleteTree(params: MemoryDeleteTreeParams): Promise; + countTree(params: MemoryCountTreeParams): Promise; +} + +export interface PrincipalNamespace { + list(params?: PrincipalListParams): Promise; + add(params: PrincipalAddParams): Promise; + remove(params: PrincipalRemoveParams): Promise; + /** Resolve principals in the space by name (member-accessible). */ + resolve(params: PrincipalResolveParams): Promise; + /** Reverse-lookup principal ids → name/kind (member-accessible). */ + lookup(params: PrincipalLookupParams): Promise; +} + +export interface GroupNamespace { + create(params: GroupCreateParams): Promise; + list(params?: GroupListParams): Promise; + rename(params: GroupRenameParams): Promise; + delete(params: GroupDeleteParams): Promise; + addMember(params: GroupAddMemberParams): Promise; + removeMember( + params: GroupRemoveMemberParams, + ): Promise; + listMembers(params: GroupListMembersParams): Promise; + listForMember( + params: GroupListForMemberParams, + ): Promise; +} + +export interface GrantNamespace { + set(params: GrantSetParams): Promise; + remove(params: GrantRemoveParams): Promise; + list(params?: GrantListParams): Promise; +} + +export interface InviteNamespace { + create(params: InviteCreateParams): Promise; + list(params?: InviteListParams): Promise; + revoke(params: InviteRevokeParams): Promise; +} + +export interface MemoryClient { + memory: MemoryNamespace; + principal: PrincipalNamespace; + group: GroupNamespace; + grant: GrantNamespace; + invite: InviteNamespace; + + /** Update the bearer token (session or api key) at runtime. */ + setToken(token: string): void; + /** Update the active space slug (X-Me-Space) at runtime. */ + setSpace(space: string): void; +} + +const DEFAULT_URL = "https://api.memory.build"; +const MEMORY_RPC_PATH = "/api/v1/memory/rpc"; +const DEFAULT_TIMEOUT = 30_000; +const DEFAULT_RETRIES = 3; + +export function createMemoryClient( + options: MemoryClientOptions = {}, +): MemoryClient { + const config: TransportConfig = { + url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), + path: options.rpcPath ?? MEMORY_RPC_PATH, + token: options.token, + timeout: options.timeout ?? DEFAULT_TIMEOUT, + retries: options.retries ?? DEFAULT_RETRIES, + clientVersion: options.clientVersion, + headers: options.space ? { [SPACE_HEADER]: options.space } : undefined, + }; + + function rpc(method: string, params: unknown): Promise { + return rpcCall(config, method, params); + } + + return { + memory: { + create: (p) => rpc("memory.create", p), + batchCreate: (p) => rpc("memory.batchCreate", p), + get: (p) => rpc("memory.get", p), + update: (p) => rpc("memory.update", p), + delete: (p) => rpc("memory.delete", p), + search: (p) => rpc("memory.search", p), + tree: (p) => rpc("memory.tree", p ?? {}), + move: (p) => rpc("memory.move", p), + deleteTree: (p) => rpc("memory.deleteTree", p), + countTree: (p) => rpc("memory.countTree", p), + }, + principal: { + list: (p) => rpc("principal.list", p ?? {}), + add: (p) => rpc("principal.add", p), + remove: (p) => rpc("principal.remove", p), + resolve: (p) => rpc("principal.resolve", p), + lookup: (p) => rpc("principal.lookup", p), + }, + group: { + create: (p) => rpc("group.create", p), + list: (p) => rpc("group.list", p ?? {}), + rename: (p) => rpc("group.rename", p), + delete: (p) => rpc("group.delete", p), + addMember: (p) => rpc("group.addMember", p), + removeMember: (p) => rpc("group.removeMember", p), + listMembers: (p) => rpc("group.listMembers", p), + listForMember: (p) => rpc("group.listForMember", p), + }, + grant: { + set: (p) => rpc("grant.set", p), + remove: (p) => rpc("grant.remove", p), + list: (p) => rpc("grant.list", p ?? {}), + }, + invite: { + create: (p) => rpc("invite.create", p), + list: (p) => rpc("invite.list", p ?? {}), + revoke: (p) => rpc("invite.revoke", p), + }, + setToken(token: string) { + config.token = token; + }, + setSpace(space: string) { + config.headers = { ...config.headers, [SPACE_HEADER]: space }; + }, + }; +} diff --git a/packages/client/transport.test.ts b/packages/client/transport.test.ts index 4a4e791..d9d12b4 100644 --- a/packages/client/transport.test.ts +++ b/packages/client/transport.test.ts @@ -45,7 +45,7 @@ function captureFetch(): { const baseConfig = { url: "https://api.example.com", - path: "/api/v1/engine/rpc", + path: "/api/v1/memory/rpc", timeout: 5_000, retries: 0, } satisfies Omit; diff --git a/packages/client/transport.ts b/packages/client/transport.ts index c70bc8d..020b2ce 100644 --- a/packages/client/transport.ts +++ b/packages/client/transport.ts @@ -4,7 +4,7 @@ * Handles HTTP communication, retry logic with exponential backoff, * timeouts, and JSON-RPC envelope formatting. */ -import { CLIENT_VERSION_HEADER } from "@memory.build/protocol"; +import { CLIENT_VERSION_HEADER } from "@memory.build/protocol/headers"; import type { JsonRpcErrorResponse, JsonRpcResponse, @@ -35,6 +35,11 @@ export interface TransportConfig { * before dispatch. Optional — older callers without this set still work. */ clientVersion?: string; + /** + * Extra headers sent on every RPC (e.g. `X-Me-Space` to select the space for + * the memory endpoint). Merged after the built-in headers. + */ + headers?: Record; } // ============================================================================= @@ -94,6 +99,9 @@ export async function rpcCall( if (config.clientVersion) { headers[CLIENT_VERSION_HEADER] = config.clientVersion; } + if (config.headers) { + Object.assign(headers, config.headers); + } let lastError: Error | undefined; diff --git a/packages/client/user.ts b/packages/client/user.ts new file mode 100644 index 0000000..4c200cf --- /dev/null +++ b/packages/client/user.ts @@ -0,0 +1,128 @@ +/** + * User client — session-only, user-scoped operations. + * + * Talks to POST /api/v1/user/rpc, authenticated by a session token. Namespaces: + * agent (a user's global service accounts), apiKey (those agents' global keys), + * and space (discover/create/manage the user's spaces — used by the CLI to pick + * the active X-Me-Space). + */ +import type { + AgentCreateParams, + AgentCreateResult, + AgentDeleteParams, + AgentDeleteResult, + AgentListParams, + AgentListResult, + AgentRenameParams, + AgentRenameResult, + ApiKeyCreateParams, + ApiKeyCreateResult, + ApiKeyDeleteParams, + ApiKeyDeleteResult, + ApiKeyGetParams, + ApiKeyGetResult, + ApiKeyListParams, + ApiKeyListResult, + SpaceCreateParams, + SpaceCreateResult, + SpaceDeleteParams, + SpaceDeleteResult, + SpaceListParams, + SpaceListResult, + SpaceRenameParams, + SpaceRenameResult, + WhoamiParams, + WhoamiResult, +} from "@memory.build/protocol/user"; +import { rpcCall, type TransportConfig } from "./transport.ts"; + +export interface UserClientOptions { + /** Base URL of the server (default: "https://api.memory.build") */ + url?: string; + /** User RPC endpoint path (default: "/api/v1/user/rpc") */ + rpcPath?: string; + /** Session token (humans only). */ + token?: string; + /** Request timeout in milliseconds (default: 30000) */ + timeout?: number; + /** Maximum retry attempts for transient failures (default: 3) */ + retries?: number; + /** CLIENT_VERSION of the caller (sent as X-Client-Version). */ + clientVersion?: string; +} + +export interface AgentNamespace { + create(params: AgentCreateParams): Promise; + list(params?: AgentListParams): Promise; + rename(params: AgentRenameParams): Promise; + delete(params: AgentDeleteParams): Promise; +} + +export interface ApiKeyNamespace { + create(params: ApiKeyCreateParams): Promise; + list(params: ApiKeyListParams): Promise; + get(params: ApiKeyGetParams): Promise; + delete(params: ApiKeyDeleteParams): Promise; +} + +export interface SpaceNamespace { + list(params?: SpaceListParams): Promise; + create(params: SpaceCreateParams): Promise; + rename(params: SpaceRenameParams): Promise; + delete(params: SpaceDeleteParams): Promise; +} + +export interface UserClient { + /** The identity behind the session token. */ + whoami(params?: WhoamiParams): Promise; + agent: AgentNamespace; + apiKey: ApiKeyNamespace; + space: SpaceNamespace; + /** Update the session token at runtime. */ + setToken(token: string): void; +} + +const DEFAULT_URL = "https://api.memory.build"; +const USER_RPC_PATH = "/api/v1/user/rpc"; +const DEFAULT_TIMEOUT = 30_000; +const DEFAULT_RETRIES = 3; + +export function createUserClient(options: UserClientOptions = {}): UserClient { + const config: TransportConfig = { + url: (options.url ?? DEFAULT_URL).replace(/\/+$/, ""), + path: options.rpcPath ?? USER_RPC_PATH, + token: options.token, + timeout: options.timeout ?? DEFAULT_TIMEOUT, + retries: options.retries ?? DEFAULT_RETRIES, + clientVersion: options.clientVersion, + }; + + function rpc(method: string, params: unknown): Promise { + return rpcCall(config, method, params); + } + + return { + whoami: (p) => rpc("whoami", p ?? {}), + agent: { + create: (p) => rpc("agent.create", p), + list: (p) => rpc("agent.list", p ?? {}), + rename: (p) => rpc("agent.rename", p), + delete: (p) => rpc("agent.delete", p), + }, + apiKey: { + create: (p) => rpc("apiKey.create", p), + list: (p) => rpc("apiKey.list", p), + get: (p) => rpc("apiKey.get", p), + delete: (p) => rpc("apiKey.delete", p), + }, + space: { + list: (p) => rpc("space.list", p ?? {}), + create: (p) => rpc("space.create", p), + rename: (p) => rpc("space.rename", p), + delete: (p) => rpc("space.delete", p), + }, + setToken(token: string) { + config.token = token; + }, + }; +} diff --git a/packages/database/auth/index.ts b/packages/database/auth/index.ts new file mode 100644 index 0000000..8e5c50d --- /dev/null +++ b/packages/database/auth/index.ts @@ -0,0 +1,6 @@ +export { + AUTH_SCHEMA, + type MigrateAuthOptions, + migrateAuth, +} from "./migrate/migrate"; +export { AUTH_SCHEMA_VERSION } from "./version"; diff --git a/packages/database/auth/migrate/idempotent/000_update.sql b/packages/database/auth/migrate/idempotent/000_update.sql new file mode 100644 index 0000000..78e9f07 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/000_update.sql @@ -0,0 +1,27 @@ +-- generic trigger function to update updated_at timestamp +create or replace function {{schema}}.update_updated_at() +returns trigger +as $func$ +begin + new.updated_at = pg_catalog.now(); + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, pg_temp; + +-- only tables that carry an updated_at column get the trigger +-- (sessions and device_authorization are insert/delete-only and have none) +create or replace trigger users_before_update_trg +before update on {{schema}}.users +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger accounts_before_update_trg +before update on {{schema}}.accounts +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger verifications_before_update_trg +before update on {{schema}}.verifications +for each row +execute function {{schema}}.update_updated_at(); diff --git a/packages/database/auth/migrate/idempotent/001_user.sql b/packages/database/auth/migrate/idempotent/001_user.sql new file mode 100644 index 0000000..f681f1c --- /dev/null +++ b/packages/database/auth/migrate/idempotent/001_user.sql @@ -0,0 +1,60 @@ +------------------------------------------------------------------------------- +-- create_user +-- email_verified is set from the provider's verified-email flag by the caller. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_user +( _email text +, _name text +, _email_verified bool default false +, _image text default null +) +returns uuid +as $func$ + insert into {{schema}}.users (email, name, email_verified, image) + values (_email, _name, coalesce(_email_verified, false), _image) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_user +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_user(_id uuid) +returns table +( id uuid +, email text +, name text +, email_verified bool +, image text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select u.id, u.email::text, u.name, u.email_verified, u.image, u.created_at, u.updated_at + from {{schema}}.users u + where u.id = _id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_user_by_email (citext column -> case-insensitive match) +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_user_by_email(_email text) +returns table +( id uuid +, email text +, name text +, email_verified bool +, image text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select u.id, u.email::text, u.name, u.email_verified, u.image, u.created_at, u.updated_at + from {{schema}}.users u + where u.email = _email::citext -- compare as citext (case-insensitive); a text param would force text=text +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/idempotent/002_session.sql b/packages/database/auth/migrate/idempotent/002_session.sql new file mode 100644 index 0000000..b56d750 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/002_session.sql @@ -0,0 +1,109 @@ +------------------------------------------------------------------------------- +-- create_session +-- The caller generates the token and passes its hash (sha256); the plaintext +-- token is never stored. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_session +( _user_id uuid +, _token_hash bytea +, _expires_at timestamptz +) +returns uuid +as $func$ + insert into {{schema}}.sessions (user_id, token_hash, expires_at) + values (_user_id, _token_hash, _expires_at) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- validate_session +-- Looks up an unexpired session by token hash and returns the session + its +-- user. No rows if missing or expired. +-- +-- Rolling session (better-auth model): on a valid lookup the expiry slides +-- forward to now + 7 days, but at most once per day — only when the remaining +-- lifetime has dropped below (window - updateAge) = 6 days. So an actively-used +-- session never expires, an idle one lapses 7 days after last use, and the hot +-- path writes at most ~once/day/session (the function is therefore volatile). +-- No absolute cap, matching better-auth's defaults (expiresIn=7d, updateAge=1d). +------------------------------------------------------------------------------- +create or replace function {{schema}}.validate_session(_token_hash bytea) +returns table +( session_id uuid +, user_id uuid +, email text +, name text +, expires_at timestamptz +) +as $func$ + with valid as + ( + select s.id, s.user_id, s.expires_at + from {{schema}}.sessions s + where s.token_hash = _token_hash + and s.expires_at > now() + ) + , bumped as + ( + update {{schema}}.sessions s + set expires_at = now() + interval '7 days' -- window (expiresIn) + from valid v + where s.id = v.id + and v.expires_at < now() + interval '6 days' -- throttle: window - updateAge (1d) + returning s.id, s.expires_at + ) + select v.id, u.id, u.email::text, u.name + , coalesce(b.expires_at, v.expires_at) as expires_at + from valid v + inner join {{schema}}.users u on (u.id = v.user_id) + left join bumped b on (b.id = v.id) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_session +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_session(_id uuid) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.sessions where id = _id returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_sessions_by_user (revoke all) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_sessions_by_user(_user_id uuid) +returns bigint +as $func$ + with d as + ( + delete from {{schema}}.sessions where user_id = _user_id returning 1 + ) + select count(*) from d +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- cleanup_expired_sessions (cron) +------------------------------------------------------------------------------- +create or replace function {{schema}}.cleanup_expired_sessions() +returns bigint +as $func$ + with d as + ( + delete from {{schema}}.sessions where expires_at <= now() returning 1 + ) + select count(*) from d +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/idempotent/003_account.sql b/packages/database/auth/migrate/idempotent/003_account.sql new file mode 100644 index 0000000..e2b01ba --- /dev/null +++ b/packages/database/auth/migrate/idempotent/003_account.sql @@ -0,0 +1,80 @@ +------------------------------------------------------------------------------- +-- upsert_account +-- Links an OAuth provider account to a user. Login-only: no tokens stored, and +-- no email (the verified email lives on users.email — better-auth-shaped). +-- The (provider_id, account_id) pair is the stable identity key. +------------------------------------------------------------------------------- +create or replace function {{schema}}.upsert_account +( _user_id uuid +, _provider_id text +, _account_id text +) +returns uuid +as $func$ + insert into {{schema}}.accounts (user_id, provider_id, account_id) + values (_user_id, _provider_id, _account_id) + on conflict (provider_id, account_id) do update set + user_id = excluded.user_id -- updated_at maintained by the before-update trigger + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_account_by_provider +-- The login lookup: resolves the owning user from the provider account id. +-- Resolving by (provider_id, account_id) — NOT by email — is what prevents +-- account-takeover via a different provider asserting the same address. +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_account_by_provider +( _provider_id text +, _account_id text +) +returns table +( id uuid +, user_id uuid +, provider_id text +, account_id text +) +as $func$ + select a.id, a.user_id, a.provider_id, a.account_id + from {{schema}}.accounts a + where a.provider_id = _provider_id + and a.account_id = _account_id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_accounts_by_user +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_accounts_by_user(_user_id uuid) +returns table +( id uuid +, user_id uuid +, provider_id text +, account_id text +) +as $func$ + select a.id, a.user_id, a.provider_id, a.account_id + from {{schema}}.accounts a + where a.user_id = _user_id + order by a.created_at +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- unlink_account +------------------------------------------------------------------------------- +create or replace function {{schema}}.unlink_account(_id uuid) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.accounts where id = _id returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/idempotent/004_device_auth.sql b/packages/database/auth/migrate/idempotent/004_device_auth.sql new file mode 100644 index 0000000..1f17b84 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/004_device_auth.sql @@ -0,0 +1,207 @@ +------------------------------------------------------------------------------- +-- create_device_auth (OAuth 2.0 device flow). status defaults to 'pending'. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_device_auth +( _device_code text +, _user_code text +, _provider text +, _oauth_state text +, _expires_at timestamptz +) +returns void +as $func$ + insert into {{schema}}.device_authorization + (device_code, user_code, provider, oauth_state, expires_at) + values + (_device_code, _user_code, _provider, _oauth_state, _expires_at) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_device_by_user_code (browser code entry; caller normalizes the code) +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_device_by_user_code(_user_code text) +returns table +( device_code text +, user_code text +, provider text +, oauth_state text +, expires_at timestamptz +, last_poll timestamptz +, user_id uuid +, status text +, created_at timestamptz +) +as $func$ + select d.device_code, d.user_code, d.provider, d.oauth_state, d.expires_at, + d.last_poll, d.user_id, d.status, d.created_at + from {{schema}}.device_authorization d + where d.user_code = _user_code and d.expires_at > now() +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_device_by_oauth_state (OAuth callback + consent) +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_device_by_oauth_state(_oauth_state text) +returns table +( device_code text +, user_code text +, provider text +, oauth_state text +, expires_at timestamptz +, last_poll timestamptz +, user_id uuid +, status text +, created_at timestamptz +) +as $func$ + select d.device_code, d.user_code, d.provider, d.oauth_state, d.expires_at, + d.last_poll, d.user_id, d.status, d.created_at + from {{schema}}.device_authorization d + where d.oauth_state = _oauth_state and d.expires_at > now() +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- bind_device_user (callback resolved the user; status stays 'pending' until +-- the human consents). Returns false if already bound / not pending / expired. +------------------------------------------------------------------------------- +create or replace function {{schema}}.bind_device_user(_device_code text, _user_id uuid) +returns bool +as $func$ + with u as + ( + update {{schema}}.device_authorization + set user_id = _user_id + where device_code = _device_code + and expires_at > now() + and status = 'pending' + and user_id is null + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- approve_device (the human consented). Requires a bound user + pending status. +------------------------------------------------------------------------------- +create or replace function {{schema}}.approve_device(_device_code text) +returns bool +as $func$ + with u as + ( + update {{schema}}.device_authorization + set status = 'approved' + where device_code = _device_code + and expires_at > now() + and status = 'pending' + and user_id is not null + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- deny_device (the human denied, or the OAuth step failed) +------------------------------------------------------------------------------- +create or replace function {{schema}}.deny_device(_device_code text) +returns bool +as $func$ + with u as + ( + update {{schema}}.device_authorization + set status = 'denied' + where device_code = _device_code + and expires_at > now() + and status = 'pending' + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- poll_device (CLI polling — resolves the poll state in one call). Returns the +-- stored status (pending|approved|denied) straight through, plus two poll-only +-- outcomes: +-- 'expired' — no unexpired device with this code +-- 'slow_down' — polled within _min_interval_secs (last_poll NOT advanced) +-- 'pending' — created/bound but not yet approved +-- 'denied' — the request was denied +-- 'approved' — bound user_id; the caller mints a session + deletes the device +------------------------------------------------------------------------------- +create or replace function {{schema}}.poll_device +( _device_code text +, _min_interval_secs double precision default 5 +) +returns table (status text, user_id uuid) +as $func$ +declare + _d record; +begin + select d.* into _d + from {{schema}}.device_authorization d + where d.device_code = _device_code and d.expires_at > now(); + + if not found then + return query select 'expired'::text, null::uuid; + return; + end if; + + -- rate limit: polled too recently -> slow_down, without advancing last_poll + if _d.last_poll is not null + and extract(epoch from now() - _d.last_poll) < _min_interval_secs then + return query select 'slow_down'::text, null::uuid; + return; + end if; + + update {{schema}}.device_authorization + set last_poll = now() + where device_code = _device_code; + + -- stored status passes straight through; user_id only when approved + return query + select _d.status, case when _d.status = 'approved' then _d.user_id else null::uuid end; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_device (cleanup after completion) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_device(_device_code text) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.device_authorization where device_code = _device_code returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_expired_devices (cron) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_expired_devices() +returns bigint +as $func$ + with d as + ( + delete from {{schema}}.device_authorization where expires_at <= now() returning 1 + ) + select count(*) from d +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/auth/migrate/idempotent/sql.d.ts b/packages/database/auth/migrate/idempotent/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/auth/migrate/idempotent/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/auth/migrate/incremental/001_users.sql b/packages/database/auth/migrate/incremental/001_users.sql new file mode 100644 index 0000000..2c8df93 --- /dev/null +++ b/packages/database/auth/migrate/incremental/001_users.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- users (better-auth model: user) +-- "users" (plural) avoids the SQL reserved word "user". +------------------------------------------------------------------------------- +create table {{schema}}.users +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, name text not null +, email citext not null unique -- citext: case-insensitive, even if app-layer lowercasing is bypassed +, email_verified boolean not null default false -- set from the provider's verified-email flag +, image text -- optional avatar url (better-auth parity) +, created_at timestamptz not null default now() +, updated_at timestamptz -- maintained by update_updated_at() trigger (idempotent/000) +); diff --git a/packages/database/auth/migrate/incremental/002_accounts.sql b/packages/database/auth/migrate/incremental/002_accounts.sql new file mode 100644 index 0000000..aebae98 --- /dev/null +++ b/packages/database/auth/migrate/incremental/002_accounts.sql @@ -0,0 +1,26 @@ +------------------------------------------------------------------------------- +-- accounts (better-auth model: account) +-- One row per provider link. LOGIN-ONLY: we authenticate via GitHub/Google but +-- never call their APIs on the user's behalf, so the token/password columns are +-- kept (for better-auth shape parity) but left null and never written. Because +-- nothing sensitive is stored at rest, there is no token-encryption subsystem. +------------------------------------------------------------------------------- +create table {{schema}}.accounts +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid not null references {{schema}}.users (id) on delete cascade +, provider_id text not null check (provider_id in ('google', 'github')) -- was `provider` +, account_id text not null -- provider's stable user id (was `provider_account_id`) +, access_token text -- nullable, unused (login-only) +, refresh_token text -- nullable, unused +, id_token text -- nullable, unused +, access_token_expires_at timestamptz +, refresh_token_expires_at timestamptz +, scope text +, password text -- nullable, unused (OAuth-only, no email/password) +, created_at timestamptz not null default now() +, updated_at timestamptz +-- the OAuth sign-in lookup key + integrity rule: one external account -> one row +, unique (provider_id, account_id) +); + +create index accounts_user_id_idx on {{schema}}.accounts (user_id); diff --git a/packages/database/auth/migrate/incremental/003_sessions.sql b/packages/database/auth/migrate/incremental/003_sessions.sql new file mode 100644 index 0000000..19ad415 --- /dev/null +++ b/packages/database/auth/migrate/incremental/003_sessions.sql @@ -0,0 +1,22 @@ +------------------------------------------------------------------------------- +-- sessions (better-auth model: session) +-- DELIBERATE DIVERGENCE FROM better-auth: we store sha256(token) in `token_hash`, +-- not the raw token. Our own validateSession hashes the presented token and looks +-- up by hash, so a database read never yields usable bearer tokens. (The BA library +-- reads sessions by raw-token equality and can't hash here; if we ever adopt it, +-- reconciling is cheap — switch to a plaintext `token` column and truncate, since +-- sessions are disposable.) +------------------------------------------------------------------------------- +create table {{schema}}.sessions +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid not null references {{schema}}.users (id) on delete cascade +, token_hash bytea not null -- sha256(rawToken); rawToken is 256-bit CSPRNG, shown to client only +, expires_at timestamptz not null +, ip_address text -- better-auth parity, nullable +, user_agent text -- better-auth parity, nullable +, created_at timestamptz not null default now() +); + +create unique index sessions_token_hash_uniq on {{schema}}.sessions (token_hash); -- the auth lookup +create index sessions_user_id_idx on {{schema}}.sessions (user_id); -- revoke-all-by-user +create index sessions_expires_at_idx on {{schema}}.sessions (expires_at); -- expired-row sweeps diff --git a/packages/database/auth/migrate/incremental/004_device_authorization.sql b/packages/database/auth/migrate/incremental/004_device_authorization.sql new file mode 100644 index 0000000..7953408 --- /dev/null +++ b/packages/database/auth/migrate/incremental/004_device_authorization.sql @@ -0,0 +1,19 @@ +------------------------------------------------------------------------------- +-- device_authorization (OAuth 2.0 device flow — RFC 8628) +-- Our own device-flow state (not a better-auth table). `user_id` (was identity_id) +-- is filled in by the OAuth callback once the human authorizes; the CLI polls by +-- device_code and exchanges an authorized row for a session. +------------------------------------------------------------------------------- +create table {{schema}}.device_authorization +( device_code text not null primary key -- CLI polling secret (32-byte base64url) +, user_code text not null unique -- human-entered code, XXXX-XXXX +, provider text not null check (provider in ('google', 'github')) +, oauth_state text not null unique -- CSRF binding for the OAuth callback +, expires_at timestamptz not null -- short TTL (~15 min) +, last_poll timestamptz -- rate-limiting the CLI poll +, user_id uuid references {{schema}}.users (id) on delete cascade -- bound once the callback resolves the user +, status text not null default 'pending' check (status in ('pending', 'approved', 'denied')) -- approved only after explicit consent +, created_at timestamptz not null default now() +); + +create index device_authorization_expires_at_idx on {{schema}}.device_authorization (expires_at); -- expired-row sweeps diff --git a/packages/database/auth/migrate/incremental/005_verifications.sql b/packages/database/auth/migrate/incremental/005_verifications.sql new file mode 100644 index 0000000..dbf1e5b --- /dev/null +++ b/packages/database/auth/migrate/incremental/005_verifications.sql @@ -0,0 +1,17 @@ +------------------------------------------------------------------------------- +-- verifications (better-auth model: verification) +-- Generic key/value-with-expiry store: email verification, password reset, magic +-- links, OTPs. Unused today (we do GitHub/Google OAuth only) but kept empty for +-- better-auth shape parity, so enabling the library later needs no migration. +------------------------------------------------------------------------------- +create table {{schema}}.verifications +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, identifier text not null +, value text not null +, expires_at timestamptz not null +, created_at timestamptz not null default now() +, updated_at timestamptz +); + +create index verifications_identifier_idx on {{schema}}.verifications (identifier); +create index verifications_expires_at_idx on {{schema}}.verifications (expires_at); -- expired-row sweeps diff --git a/packages/database/auth/migrate/incremental/sql.d.ts b/packages/database/auth/migrate/incremental/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/auth/migrate/incremental/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/auth/migrate/migrate.integration.test.ts b/packages/database/auth/migrate/migrate.integration.test.ts new file mode 100644 index 0000000..134a92d --- /dev/null +++ b/packages/database/auth/migrate/migrate.integration.test.ts @@ -0,0 +1,525 @@ +// Integration tests for the `auth` schema migrations (migrateAuth). +// +// The auth migrations are templated, so each test targets its own throwaway +// `auth_test_` schema — never the real `auth`. That makes these tests +// isolated and safe to run against any database (including a shared dev one). +// Read-only shape assertions share one canonical auth schema provisioned in +// beforeAll; the few behavior tests provision their own. +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Sql as SQL } from "postgres"; +import { AUTH_SCHEMA_VERSION } from "../version"; +import { migrateAuth } from "./migrate"; +import { + appliedMigrations, + connect, + expectReject, + extensionInstalled, + getSchemaVersion, + listFunctions, + listTables, + listTriggers, + randomAuthSchema, + schemaExists, + TestAuth, + tableExists, + withTestAuth, +} from "./test-utils"; + +const EXPECTED_TABLES = [ + "accounts", + "device_authorization", + "migration", + "sessions", + "users", + "verifications", + "version", +]; + +const EXPECTED_MIGRATIONS = [ + "001_users", + "002_accounts", + "003_sessions", + "004_device_authorization", + "005_verifications", +]; + +const EXPECTED_FUNCTIONS = [ + "update_updated_at", + "create_user", + "get_user", + "get_user_by_email", + "create_session", + "validate_session", + "upsert_account", + "get_account_by_provider", + "create_device_auth", + "bind_device_user", + "approve_device", + "poll_device", +]; + +// The auth schema deliberately requires only citext — not the engine extensions. +const REQUIRED_EXTENSIONS = ["citext"]; + +const V7 = "00000000-0000-7000-8000-000000000000"; +const V4 = "00000000-0000-4000-8000-000000000000"; + +/** Insert a user and return its id (most tables FK to users). */ +async function insertUser(sql: SQL, schema: string): Promise { + const email = `u_${crypto.randomUUID().slice(0, 8)}@example.com`; + const [row] = await sql.unsafe( + `insert into ${schema}.users (name, email) values ('Test', '${email}') returning id`, + ); + return row?.id as string; +} + +let sql: SQL; +// One migrated auth schema shared by all read-only shape/function assertions. +let canonical: TestAuth; + +beforeAll(async () => { + sql = connect(12); + canonical = await TestAuth.create(sql); // migrateAuth installs citext itself +}); + +afterAll(async () => { + await canonical?.drop(); + await sql.end(); +}); + +describe("provisioned auth schema", () => { + test("provisions into the requested (templated) schema", async () => { + expect(canonical.schema).toMatch(/^auth_test_/); + expect(await schemaExists(sql, canonical.schema)).toBe(true); + }); + + test("creates infrastructure and domain tables", async () => { + const tables = await listTables(sql, canonical.schema); + for (const table of EXPECTED_TABLES) { + expect(tables).toContain(table); + } + }); + + test("records every incremental migration exactly once", async () => { + expect(await appliedMigrations(sql, canonical.schema)).toEqual( + EXPECTED_MIGRATIONS, + ); + }); + + test("stamps the schema version", async () => { + expect(await getSchemaVersion(sql, canonical.schema)).toBe( + AUTH_SCHEMA_VERSION, + ); + }); + + test("installs only the required extensions", async () => { + for (const ext of REQUIRED_EXTENSIONS) { + expect(await extensionInstalled(sql, ext)).toBe(true); + } + }); + + test("creates the updated_at trigger function in the schema", async () => { + const functions = await listFunctions(sql, canonical.schema); + for (const fn of EXPECTED_FUNCTIONS) { + expect(functions).toContain(fn); + } + }); + + test("installs updated_at triggers on mutable tables only", async () => { + for (const table of ["users", "accounts", "verifications"]) { + const triggers = await listTriggers(sql, canonical.schema, table); + expect(triggers).toContain(`${table}_before_update_trg`); + } + // insert/delete-only tables have no updated_at and thus no trigger + for (const table of ["sessions", "device_authorization"]) { + const triggers = await listTriggers(sql, canonical.schema, table); + expect(triggers).not.toContain(`${table}_before_update_trg`); + } + }); +}); + +describe("schema constraints enforce", () => { + test("user ids must be UUIDv7", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.users (id, name, email) + values ('${V4}', 'v4', 'v4@example.com')`, + ), + ); + }); + + test("user email is unique and case-insensitive (citext)", async () => { + const s = canonical.schema; + const email = `Dup_${crypto.randomUUID().slice(0, 8)}@Example.com`; + await sql.unsafe( + `insert into ${s}.users (name, email) values ('a', '${email}')`, + ); + try { + await expectReject(() => + sql.unsafe( + `insert into ${s}.users (name, email) values ('b', '${email.toLowerCase()}')`, + ), + ); + } finally { + await sql.unsafe(`delete from ${s}.users where email = '${email}'`); + } + }); + + test("accounts.provider_id is restricted to google/github", async () => { + const userId = await insertUser(sql, canonical.schema); + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.accounts (user_id, provider_id, account_id) + values ('${userId}', 'facebook', 'x')`, + ), + ); + }); + + test("accounts are unique per (provider_id, account_id)", async () => { + const s = canonical.schema; + const userId = await insertUser(sql, s); + const acct = crypto.randomUUID(); + await sql.unsafe( + `insert into ${s}.accounts (user_id, provider_id, account_id) + values ('${userId}', 'github', '${acct}')`, + ); + await expectReject(() => + sql.unsafe( + `insert into ${s}.accounts (user_id, provider_id, account_id) + values ('${userId}', 'github', '${acct}')`, + ), + ); + }); + + test("accounts.user_id must reference an existing user", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.accounts (user_id, provider_id, account_id) + values ('${V7}', 'github', 'orphan')`, + ), + ); + }); + + test("session token_hash is unique", async () => { + const s = canonical.schema; + const userId = await insertUser(sql, s); + await sql.unsafe( + `insert into ${s}.sessions (user_id, token_hash, expires_at) + values ('${userId}', '\\xdeadbeef', now() + interval '1 day')`, + ); + await expectReject(() => + sql.unsafe( + `insert into ${s}.sessions (user_id, token_hash, expires_at) + values ('${userId}', '\\xdeadbeef', now() + interval '1 day')`, + ), + ); + }); + + test("device_authorization.user_code is unique", async () => { + const s = canonical.schema; + const code = `AB${crypto.randomUUID().slice(0, 4).toUpperCase()}`; + await sql.unsafe( + `insert into ${s}.device_authorization (device_code, user_code, provider, oauth_state, expires_at) + values ('${crypto.randomUUID()}', '${code}', 'google', '${crypto.randomUUID()}', now() + interval '15 min')`, + ); + await expectReject(() => + sql.unsafe( + `insert into ${s}.device_authorization (device_code, user_code, provider, oauth_state, expires_at) + values ('${crypto.randomUUID()}', '${code}', 'google', '${crypto.randomUUID()}', now() + interval '15 min')`, + ), + ); + }); +}); + +describe("auth functions", () => { + const email = () => `fn_${crypto.randomUUID().slice(0, 8)}@example.com`; + + test("create_user + get_user + get_user_by_email (citext)", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const e = email(); + const [u] = await sql.unsafe( + `select ${s}.create_user($1, $2, $3) as id`, + [e, "Alice", true], + ); + const id = u?.id as string; + + const [byId] = await sql.unsafe(`select * from ${s}.get_user($1)`, [id]); + expect(byId?.id).toBe(id); + expect(byId?.email).toBe(e); + expect(byId?.email_verified).toBe(true); + + // citext: lookup is case-insensitive + const [byEmail] = await sql.unsafe( + `select * from ${s}.get_user_by_email($1)`, + [e.toUpperCase()], + ); + expect(byEmail?.id).toBe(id); + }); + }); + + test("create_session + validate_session (valid + expired)", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const [u] = await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + email(), + "Bob", + ]); + const userId = u?.id as string; + + await sql.unsafe( + `select ${s}.create_session($1, $2::bytea, now() + interval '1 day')`, + [userId, "\\xabcd"], + ); + const valid = await sql.unsafe( + `select * from ${s}.validate_session($1::bytea)`, + ["\\xabcd"], + ); + expect(valid.length).toBe(1); + expect(valid[0]?.user_id).toBe(userId); + + await sql.unsafe( + `select ${s}.create_session($1, $2::bytea, now() - interval '1 second')`, + [userId, "\\xbeef"], + ); + const expired = await sql.unsafe( + `select * from ${s}.validate_session($1::bytea)`, + ["\\xbeef"], + ); + expect(expired.length).toBe(0); + }); + }); + + test("upsert_account + get_account_by_provider (login lookup, idempotent)", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const [u] = await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + email(), + "Carol", + ]); + const userId = u?.id as string; + const acct = crypto.randomUUID(); + + await sql.unsafe(`select ${s}.upsert_account($1, 'github', $2)`, [ + userId, + acct, + ]); + const found = await sql.unsafe( + `select * from ${s}.get_account_by_provider('github', $1)`, + [acct], + ); + expect(found[0]?.user_id).toBe(userId); + + // re-upsert the same (provider, account) stays one row + await sql.unsafe(`select ${s}.upsert_account($1, 'github', $2)`, [ + userId, + acct, + ]); + const [n] = await sql.unsafe( + `select count(*)::int as n from ${s}.accounts where provider_id='github' and account_id=$1`, + [acct], + ); + expect(n?.n).toBe(1); + + const none = await sql.unsafe( + `select * from ${s}.get_account_by_provider('github', $1)`, + [crypto.randomUUID()], + ); + expect(none.length).toBe(0); + }); + }); + + test("device flow: create → bind → consent → authorized (and deny path)", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const [u] = await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + email(), + "Dave", + ]); + const userId = u?.id as string; + + const deviceCode = crypto.randomUUID(); + const userCode = "ABCD-2345"; + const oauthState = crypto.randomUUID(); + await sql.unsafe( + `select ${s}.create_device_auth($1, $2, 'google', $3, now() + interval '15 min')`, + [deviceCode, userCode, oauthState], + ); + + const byState = await sql.unsafe( + `select * from ${s}.get_device_by_oauth_state($1)`, + [oauthState], + ); + expect(byState[0]?.device_code).toBe(deviceCode); + expect(byState[0]?.status).toBe("pending"); + const byUserCode = await sql.unsafe( + `select * from ${s}.get_device_by_user_code($1)`, + [userCode], + ); + expect(byUserCode[0]?.device_code).toBe(deviceCode); + + // before binding → pending (interval 0 bypasses rate limit) + const [p1] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + deviceCode, + ]); + expect(p1?.status).toBe("pending"); + + // immediate re-poll with the default interval → slow_down + const [sd] = await sql.unsafe(`select * from ${s}.poll_device($1)`, [ + deviceCode, + ]); + expect(sd?.status).toBe("slow_down"); + + // callback binds the user, but status stays pending until consent + const [b] = await sql.unsafe( + `select ${s}.bind_device_user($1, $2) as ok`, + [deviceCode, userId], + ); + expect(b?.ok).toBe(true); + const [pBound] = await sql.unsafe( + `select * from ${s}.poll_device($1, 0)`, + [deviceCode], + ); + expect(pBound?.status).toBe("pending"); // bound but NOT yet approved + + // consent: approve → authorized; a second approve is a no-op + const [ap] = await sql.unsafe(`select ${s}.approve_device($1) as ok`, [ + deviceCode, + ]); + expect(ap?.ok).toBe(true); + const [ap2] = await sql.unsafe(`select ${s}.approve_device($1) as ok`, [ + deviceCode, + ]); + expect(ap2?.ok).toBe(false); + + const [p2] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + deviceCode, + ]); + expect(p2?.status).toBe("approved"); + expect(p2?.user_id).toBe(userId); + + // deny path on a separate device → poll resolves to denied + const dc2 = crypto.randomUUID(); + await sql.unsafe( + `select ${s}.create_device_auth($1, $2, 'google', $3, now() + interval '15 min')`, + [dc2, "WXYZ-3456", crypto.randomUUID()], + ); + expect( + (await sql.unsafe(`select ${s}.deny_device($1) as ok`, [dc2]))[0]?.ok, + ).toBe(true); + const [pd] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + dc2, + ]); + expect(pd?.status).toBe("denied"); + + // unknown / expired device code → expired + const [ex] = await sql.unsafe(`select * from ${s}.poll_device($1, 0)`, [ + crypto.randomUUID(), + ]); + expect(ex?.status).toBe("expired"); + }); + }); +}); + +describe("cascade + trigger behavior", () => { + test("deleting a user cascades to accounts and sessions", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const userId = await insertUser(sql, s); + await sql.unsafe( + `insert into ${s}.accounts (user_id, provider_id, account_id) values ('${userId}', 'github', '1')`, + ); + await sql.unsafe( + `insert into ${s}.sessions (user_id, token_hash, expires_at) values ('${userId}', '\\xaa', now() + interval '1 day')`, + ); + + await sql.unsafe(`delete from ${s}.users where id = '${userId}'`); + + const [acct] = await sql.unsafe( + `select count(*)::int as n from ${s}.accounts where user_id = '${userId}'`, + ); + const [sess] = await sql.unsafe( + `select count(*)::int as n from ${s}.sessions where user_id = '${userId}'`, + ); + expect(acct?.n).toBe(0); + expect(sess?.n).toBe(0); + }); + }); + + test("updating a user bumps updated_at via trigger", async () => { + await withTestAuth(sql, {}, async (auth) => { + const s = auth.schema; + const userId = await insertUser(sql, s); + const [before] = await sql.unsafe( + `select updated_at from ${s}.users where id = '${userId}'`, + ); + expect(before?.updated_at).toBeNull(); + + await sql.unsafe( + `update ${s}.users set name = 'Renamed' where id = '${userId}'`, + ); + const [after] = await sql.unsafe( + `select updated_at from ${s}.users where id = '${userId}'`, + ); + expect(after?.updated_at).not.toBeNull(); + }); + }); +}); + +describe("migration behavior", () => { + test("is idempotent: re-running changes no migration rows or version", async () => { + await withTestAuth(sql, {}, async (auth) => { + const before = await appliedMigrations(sql, auth.schema); + await migrateAuth(sql, { schema: auth.schema }); + expect(await appliedMigrations(sql, auth.schema)).toEqual(before); + expect(await getSchemaVersion(sql, auth.schema)).toBe( + AUTH_SCHEMA_VERSION, + ); + }); + }); + + test("rejects a downgrade (db version newer than app)", async () => { + await withTestAuth(sql, {}, async (auth) => { + await sql.unsafe(`update ${auth.schema}.version set version = '99.0.0'`); + await expect(migrateAuth(sql, { schema: auth.schema })).rejects.toThrow( + /older than database version/, + ); + }); + }); + + test("rejects invalid schema names", async () => { + for (const schema of ["Bad-Schema", "1auth", "auth test", "auth;drop"]) { + await expect(migrateAuth(sql, { schema })).rejects.toThrow( + /Invalid auth schema name/, + ); + } + }); + + test("concurrent migrateAuth on one schema is serialized safely", async () => { + // The advisory lock serializes writers. A loser may exhaust its retry + // budget and throw "Unable to acquire lock" — expected, not corruption. + // What must hold: at least one succeeds and the schema stays valid. + const schema = randomAuthSchema(); + try { + const results = await Promise.allSettled([ + migrateAuth(sql, { schema }), + migrateAuth(sql, { schema }), + migrateAuth(sql, { schema }), + ]); + + expect(results.some((r) => r.status === "fulfilled")).toBe(true); + for (const r of results) { + if (r.status === "rejected") { + expect(String((r.reason as Error)?.message ?? r.reason)).toContain( + "Unable to acquire lock", + ); + } + } + + expect(await getSchemaVersion(sql, schema)).toBe(AUTH_SCHEMA_VERSION); + expect(await tableExists(sql, schema, "users")).toBe(true); + } finally { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + } + }); +}); diff --git a/packages/database/auth/migrate/migrate.ts b/packages/database/auth/migrate/migrate.ts new file mode 100644 index 0000000..545f2ad --- /dev/null +++ b/packages/database/auth/migrate/migrate.ts @@ -0,0 +1,242 @@ +import { info, reportError, span } from "@pydantic/logfire-node"; +import { semver } from "bun"; +import type { Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + doesSchemaExist, + ensureExtension, + ensurePostgresVersion, + executeSqlFile, + isValidSchemaName, + type Migration, + runSchemaMigrations, + template, +} from "../../migrate/kit"; +import { AUTH_SCHEMA_VERSION } from "../version"; +import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; +import idempotent001 from "./idempotent/001_user.sql" with { type: "text" }; +import idempotent002 from "./idempotent/002_session.sql" with { type: "text" }; +import idempotent003 from "./idempotent/003_account.sql" with { type: "text" }; +import idempotent004 from "./idempotent/004_device_auth.sql" with { + type: "text", +}; +import incremental001 from "./incremental/001_users.sql" with { type: "text" }; +import incremental002 from "./incremental/002_accounts.sql" with { + type: "text", +}; +import incremental003 from "./incremental/003_sessions.sql" with { + type: "text", +}; +import incremental004 from "./incremental/004_device_authorization.sql" with { + type: "text", +}; +import incremental005 from "./incremental/005_verifications.sql" with { + type: "text", +}; +import provisionSql from "./provision.sql" with { type: "text" }; + +const DIR = "packages/database/auth/migrate"; + +// The auth schema only needs citext (case-insensitive email). It deliberately +// does NOT require the engine extensions (ltree / vector / pg_textsearch), so it +// can live in a database with no pgvector. +const AUTH_REQUIRED_EXTENSIONS = [ + { name: "citext", minVersion: "1.6" }, +] as const; + +const incrementals: Migration[] = [ + { name: "001_users", file: "incremental/001_users.sql", sql: incremental001 }, + { + name: "002_accounts", + file: "incremental/002_accounts.sql", + sql: incremental002, + }, + { + name: "003_sessions", + file: "incremental/003_sessions.sql", + sql: incremental003, + }, + { + name: "004_device_authorization", + file: "incremental/004_device_authorization.sql", + sql: incremental004, + }, + { + name: "005_verifications", + file: "incremental/005_verifications.sql", + sql: incremental005, + }, +]; + +const idempotents: Migration[] = [ + { name: "000_update", file: "idempotent/000_update.sql", sql: idempotent000 }, + { name: "001_user", file: "idempotent/001_user.sql", sql: idempotent001 }, + { + name: "002_session", + file: "idempotent/002_session.sql", + sql: idempotent002, + }, + { + name: "003_account", + file: "idempotent/003_account.sql", + sql: idempotent003, + }, + { + name: "004_device_auth", + file: "idempotent/004_device_auth.sql", + sql: idempotent004, + }, +]; + +/** + * The authentication schema name. Production uses "auth"; the name is a + * parameter so tests can provision throwaway, isolated auth schemas (and so the + * SQL is templated symmetrically with the core/space migrations). Reference this + * constant rather than hardcoding "auth" elsewhere. + */ +export const AUTH_SCHEMA = "auth"; + +export interface MigrateAuthOptions { + schema?: string; + logSqlFiles?: boolean; + statementTimeout?: string; + lockTimeout?: string; + transactionTimeout?: string; + idleInTransactionSessionTimeout?: string; +} + +interface NormalizedMigrateAuthOptions { + schema: string; + logSqlFiles: boolean; + schemaVersion: string; + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function migrateAuth( + sql: SQL, + options: MigrateAuthOptions = {}, +): Promise { + const opts = normalizeMigrateAuthOptions(options); + const attributes = migrateAttributes(opts); + + await span("auth.migrate", { + attributes, + callback: async () => { + try { + if (!isValidSchemaName(opts.schema)) { + throw new Error( + `Invalid auth schema name: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, + ); + } + if (!semver.satisfies(opts.schemaVersion, "*")) { + throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); + } + const [key1, key2] = advisoryLockKey( + `memory-auth:schema:${opts.schema}`, + ); + + await sql.begin(async (tx) => { + await applySessionTimeouts(tx, opts); + const acquired = await span("auth.migrate.acquire_lock", { + attributes, + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error("Unable to acquire lock for auth migrations."); + } + + await ensurePostgresVersion(tx); + for (const extension of AUTH_REQUIRED_EXTENSIONS) { + await span("auth.migrate.ensure_extension", { + attributes: { + "db.extension": extension.name, + "db.extension_min_version": extension.minVersion, + }, + callback: () => + ensureExtension(tx, extension.name, extension.minVersion), + }); + } + + if (!(await doesSchemaExist(tx, opts.schema))) { + await span("auth.migrate.provision", { + attributes: { + ...attributes, + "auth.migration_file": "provision.sql", + "auth.migration_type": "provision", + }, + callback: () => + executeSqlFile( + tx, + template(provisionSql, { schema: opts.schema }), + { + logSqlFiles: opts.logSqlFiles, + label: "auth", + schema: opts.schema, + type: "provision", + dir: DIR, + file: "provision.sql", + }, + ), + }); + info("Auth schema provisioned", attributes); + } + await span("auth.migrate.run", { + attributes, + callback: () => + runSchemaMigrations(tx, { + schema: opts.schema, + schemaVersion: opts.schemaVersion, + incrementals, + idempotents, + templateVars: { schema: opts.schema }, + label: "auth", + dir: DIR, + logSqlFiles: opts.logSqlFiles, + }), + }); + }); + info("Auth migrations completed", attributes); + } catch (error) { + reportError("Auth migration failed", error as Error, attributes); + throw error; + } + }, + }); +} + +function migrateAttributes( + options: NormalizedMigrateAuthOptions, +): Record { + return { + "db.schema": options.schema, + "auth.schema_version": options.schemaVersion, + "auth.required_extensions": AUTH_REQUIRED_EXTENSIONS.map( + (extension) => `${extension.name}@>=${extension.minVersion}`, + ), + "db.statement_timeout": options.statementTimeout, + "db.lock_timeout": options.lockTimeout, + "db.transaction_timeout": options.transactionTimeout, + "db.idle_in_transaction_session_timeout": + options.idleInTransactionSessionTimeout, + }; +} + +function normalizeMigrateAuthOptions( + options: MigrateAuthOptions, +): NormalizedMigrateAuthOptions { + return { + schema: options.schema ?? AUTH_SCHEMA, + logSqlFiles: options.logSqlFiles ?? false, + schemaVersion: AUTH_SCHEMA_VERSION, + statementTimeout: options.statementTimeout ?? "20s", + lockTimeout: options.lockTimeout ?? "5s", + transactionTimeout: options.transactionTimeout ?? "1min", + idleInTransactionSessionTimeout: + options.idleInTransactionSessionTimeout ?? "5s", + }; +} diff --git a/packages/database/auth/migrate/provision.sql b/packages/database/auth/migrate/provision.sql new file mode 100644 index 0000000..e98b9d9 --- /dev/null +++ b/packages/database/auth/migrate/provision.sql @@ -0,0 +1,15 @@ +create schema {{schema}}; + +create table {{schema}}.version +( version text not null +, at timestamptz not null default now() +); + +create unique index version_singleton_idx on {{schema}}.version ((true)); -- only ONE row allowed +insert into {{schema}}.version (version) values ('0.0.0'); + +create table {{schema}}.migration +( name text not null constraint migration_pkey primary key +, applied_at_version text not null +, applied_at timestamptz not null default pg_catalog.clock_timestamp() +); diff --git a/packages/database/auth/migrate/sql.d.ts b/packages/database/auth/migrate/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/auth/migrate/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/auth/migrate/test-utils.ts b/packages/database/auth/migrate/test-utils.ts new file mode 100644 index 0000000..2d242bb --- /dev/null +++ b/packages/database/auth/migrate/test-utils.ts @@ -0,0 +1,65 @@ +import type { Sql as SQL } from "postgres"; +import { type MigrateAuthOptions, migrateAuth } from "./migrate"; + +// Connection, failure assertions, and schema introspection are shared with the +// core and space suites. +export * from "../../migrate/test-utils"; + +const SCHEMA_SUFFIX_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; + +/** A unique, valid auth schema name, e.g. "auth_test_a1b2c3d4". */ +export function randomAuthSchema(): string { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let suffix = ""; + for (const b of bytes) suffix += SCHEMA_SUFFIX_ALPHABET[b % 36]; + return `auth_test_${suffix}`; +} + +// --------------------------------------------------------------------------- +// TestAuth — a provisioned, isolated auth schema +// --------------------------------------------------------------------------- +// +// The auth migrations are templated (production uses the "auth" schema; tests +// pass a unique throwaway name), so every test gets its own isolated auth schema +// and they run concurrently without ever touching a real `auth` schema. + +export class TestAuth { + readonly schema: string; + private readonly sql: SQL; + + private constructor(sql: SQL, schema: string) { + this.sql = sql; + this.schema = schema; + } + + static async create( + sql: SQL, + options: Omit & { schema?: string } = {}, + ): Promise { + const schema = options.schema ?? randomAuthSchema(); + await migrateAuth(sql, { ...options, schema }); + return new TestAuth(sql, schema); + } + + async drop(): Promise { + await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); + } +} + +/** + * Provision a fresh auth schema, run `fn` against it, and always drop it + * afterward. Safe to call from concurrent tests — each gets its own unique + * schema. + */ +export async function withTestAuth( + sql: SQL, + options: Omit & { schema?: string }, + fn: (auth: TestAuth) => Promise, +): Promise { + const auth = await TestAuth.create(sql, options); + try { + return await fn(auth); + } finally { + await auth.drop(); + } +} diff --git a/packages/database/auth/version.ts b/packages/database/auth/version.ts new file mode 100644 index 0000000..3bc3270 --- /dev/null +++ b/packages/database/auth/version.ts @@ -0,0 +1 @@ +export const AUTH_SCHEMA_VERSION = "0.0.1"; diff --git a/packages/database/core/index.ts b/packages/database/core/index.ts new file mode 100644 index 0000000..8dd575c --- /dev/null +++ b/packages/database/core/index.ts @@ -0,0 +1,6 @@ +export { + CORE_SCHEMA, + type MigrateCoreOptions, + migrateCore, +} from "./migrate/migrate"; +export { CORE_SCHEMA_VERSION } from "./version"; diff --git a/packages/database/core/migrate/idempotent/000_update.sql b/packages/database/core/migrate/idempotent/000_update.sql new file mode 100644 index 0000000..61d6878 --- /dev/null +++ b/packages/database/core/migrate/idempotent/000_update.sql @@ -0,0 +1,40 @@ +-- generic trigger function to update updated_at timestamp +create or replace function {{schema}}.update_updated_at() +returns trigger +as $func$ +begin + new.updated_at = pg_catalog.now(); + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, pg_temp; + +create or replace trigger space_before_update_trg +before update on {{schema}}.space +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger principal_before_update_trg +before update on {{schema}}.principal +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger principal_space_before_update_trg +before update on {{schema}}.principal_space +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger group_member_before_update_trg +before update on {{schema}}.group_member +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger tree_access_before_update_trg +before update on {{schema}}.tree_access +for each row +execute function {{schema}}.update_updated_at(); + +create or replace trigger space_invitation_before_update_trg +before update on {{schema}}.space_invitation +for each row +execute function {{schema}}.update_updated_at(); diff --git a/packages/database/core/migrate/idempotent/001_principal_space.sql b/packages/database/core/migrate/idempotent/001_principal_space.sql new file mode 100644 index 0000000..ba7132f --- /dev/null +++ b/packages/database/core/migrate/idempotent/001_principal_space.sql @@ -0,0 +1,161 @@ +------------------------------------------------------------------------------- +-- is_principal_in_space +------------------------------------------------------------------------------- +create or replace function {{schema}}.is_principal_in_space +( _principal_id uuid +, _space_id uuid +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.principal_space ps + where ps.principal_id = _principal_id + and ps.space_id = _space_id + ) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- is_principal_space_admin +-- A principal is a space admin if it has a direct admin membership, OR it is a +-- member of a group whose own space-membership is admin (admin transfers +-- transitively through groups, like access does — Model 2). Agents are never +-- space admins. +------------------------------------------------------------------------------- +create or replace function {{schema}}.is_principal_space_admin +( _principal_id uuid +, _space_id uuid +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.principal p + where p.id = _principal_id + and p.kind <> 'a' -- agents cannot be space admins + and + ( + -- direct admin membership + exists + ( + select 1 + from {{schema}}.principal_space ps + where ps.principal_id = p.id + and ps.space_id = _space_id + and ps.admin + ) + -- admin inherited from an admin group the principal belongs to + or exists + ( + select 1 + from {{schema}}.group_member gm + inner join {{schema}}.principal_space gps + on (gps.principal_id = gm.group_id and gps.space_id = _space_id and gps.admin) + where gm.member_id = p.id + and gm.space_id = _space_id + ) + ) + ) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- enforce_last_admin (trigger fn on principal_space + group_member) +-- Invariant: a live space must always have at least one *effective* admin — a +-- user who is a direct admin (principal_space.admin) OR a member of an +-- admin-flagged group. Agents are never admins, and an admin-flagged group with +-- no user members does NOT count. Checking the effective set (not just the +-- principal_space.admin flag) closes the brick where a space's sole admin is an +-- empty admin group, leaving it unrecoverable. +-- +-- Guards every path that could drop the effective set, uniformly: +-- * principal_space remove/demote — incl. a group losing its admin flag, and +-- delete_principal cascades (deleting an admin user or group); +-- * group_member removal from an admin group — incl. remove_principal_from_space +-- and delete_principal cascades (a user leaving the sole admin group). +-- +-- Whole-space teardown is exempt: delete_space drops the core.space row and lets +-- the FK cascade scrub the roster, so by the time this fires the space row is +-- gone. The `for update` both detects that (no row -> skip) and serializes +-- concurrent admin removals on the same space (so two txns can't each drop a +-- different last-ish admin and race to zero). +------------------------------------------------------------------------------- +create or replace function {{schema}}.enforce_last_admin() +returns trigger +as $func$ +begin + -- group_member path: only an ADMIN group's membership can affect the effective + -- admin set; non-admin group churn (the common case) returns immediately. + if tg_table_name = 'group_member' then + if not exists + ( + select 1 + from {{schema}}.principal_space gps + where gps.principal_id = old.group_id + and gps.space_id = old.space_id + and gps.admin + ) then + return null; + end if; + end if; + + perform 1 from {{schema}}.space s where s.id = old.space_id for update; + if not found then + return null; -- space is being deleted: teardown, not a demote/removal + end if; + + if not + ( + -- a direct admin user + exists + ( + select 1 + from {{schema}}.principal_space ps + join {{schema}}.principal p on p.id = ps.principal_id + where ps.space_id = old.space_id + and ps.admin + and p.kind = 'u' + ) + -- or a user member of an admin-flagged group + or exists + ( + select 1 + from {{schema}}.group_member gm + join {{schema}}.principal_space gps + on gps.principal_id = gm.group_id and gps.space_id = gm.space_id and gps.admin + join {{schema}}.principal mp on mp.id = gm.member_id and mp.kind = 'u' + where gm.space_id = old.space_id + ) + ) then + raise exception + 'cannot leave space % without an effective admin', old.space_id + using errcode = 'ME001' + , hint = 'a space needs a user admin — direct, or via an admin group with at least one member'; + end if; + + return null; +end; +$func$ language plpgsql +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +-- principal_space: fire only when an admin row is removed or demoted (NEW can't +-- be referenced in a DELETE trigger's WHEN, hence two). group_member: fire on any +-- removal; the fn early-outs unless the group is an admin group. +create or replace trigger principal_space_keep_admin_del +after delete on {{schema}}.principal_space +for each row when (old.admin) +execute function {{schema}}.enforce_last_admin(); + +create or replace trigger principal_space_keep_admin_upd +after update on {{schema}}.principal_space +for each row when (old.admin and not new.admin) +execute function {{schema}}.enforce_last_admin(); + +create or replace trigger group_member_keep_admin_del +after delete on {{schema}}.group_member +for each row +execute function {{schema}}.enforce_last_admin(); diff --git a/packages/database/core/migrate/idempotent/002_group_member.sql b/packages/database/core/migrate/idempotent/002_group_member.sql new file mode 100644 index 0000000..f5a3fdd --- /dev/null +++ b/packages/database/core/migrate/idempotent/002_group_member.sql @@ -0,0 +1,49 @@ + +------------------------------------------------------------------------------- +-- member_groups +------------------------------------------------------------------------------- +create or replace function {{schema}}.member_groups +( _member_id uuid +, _space_id uuid +) +returns table +( group_id uuid +, admin bool +) +as $func$ + -- Group membership is space-scoped by group_member.space_id and confers + -- space access transitively — a member need NOT have a direct principal_space + -- row to inherit a group's grants (Model 2). The FKs already constrain + -- group_id to a group and member_id to a user/agent. + select + gm.group_id + , gm.admin and (not m.kind = 'a') -- agents cannot be group admins + from {{schema}}.group_member gm + inner join {{schema}}.principal m on (m.member_id = gm.member_id) + where gm.member_id = _member_id + and gm.space_id = _space_id +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- is_group_admin +-- Whether a member is an admin of a specific group in a space. (Agents are +-- never group admins — enforced by member_groups.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.is_group_admin +( _member_id uuid +, _group_id uuid +, _space_id uuid +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.member_groups(_member_id, _space_id) mg + where mg.group_id = _group_id + and mg.admin + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/003_tree_access.sql b/packages/database/core/migrate/idempotent/003_tree_access.sql new file mode 100644 index 0000000..bc55608 --- /dev/null +++ b/packages/database/core/migrate/idempotent/003_tree_access.sql @@ -0,0 +1,169 @@ +------------------------------------------------------------------------------- +-- member_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.member_tree_access +( _member_id uuid +, _space_id uuid +) +returns table +( tree_path ltree +, access int +) +as $func$ + -- member's grants via groups + select + ta.tree_path + , ta.access + from {{schema}}.member_groups(_member_id, _space_id) mg + inner join {{schema}}.tree_access ta on (mg.group_id = ta.principal_id and ta.space_id = _space_id) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- user_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.user_tree_access +( _user_id uuid +, _space_id uuid +) +returns table +( tree_path ltree +, access int +) +as $func$ + -- user's direct grants + select + ta.tree_path + , ta.access + from {{schema}}.principal u + inner join {{schema}}.principal_space psu on (u.id = psu.principal_id and psu.space_id = _space_id) + inner join {{schema}}.tree_access ta on (u.id = ta.principal_id and ta.space_id = _space_id) + where u.user_id = _user_id + union + -- user's access via groups + select + x.tree_path + , x.access + from {{schema}}.member_tree_access(_user_id, _space_id) x +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- agent_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.agent_tree_access +( _agent_id uuid +, _space_id uuid +) +returns table +( tree_path ltree +, access int +) +as $func$ + with agent_access as materialized + ( + -- agent's direct grants + select + ta.tree_path + , ta.access + from {{schema}}.principal a + inner join {{schema}}.principal_space ps on (a.id = ps.principal_id and ps.space_id = _space_id) + inner join {{schema}}.tree_access ta on (a.id = ta.principal_id and ta.space_id = _space_id) + where a.agent_id = _agent_id + union + -- agent's access via groups + select + x.tree_path + , x.access + from {{schema}}.member_tree_access(_agent_id, _space_id) x + ) + , owner_access as materialized + ( + -- get the access for the user that owns the agent + select + x.tree_path + , x.access + from + ( + select p.owner_id + from {{schema}}.principal p + where p.agent_id = _agent_id + ) a + cross join lateral {{schema}}.user_tree_access(a.owner_id, _space_id) x + ) + select + x.tree_path + , max(x.access) + from + ( + -- take the agent's access when it is covered by the owner's access + select + aa.tree_path + , aa.access + from agent_access aa + where exists + ( + -- the owner must have access that is the same or greater than the agent's + select 1 + from owner_access oa + where oa.tree_path @> aa.tree_path + and oa.access >= aa.access + ) + union + -- when the agent has more access than the owner, take the owner's access + select + oa.tree_path + , oa.access + from owner_access oa + where exists + ( + select 1 + from agent_access aa + where aa.tree_path @> oa.tree_path + and aa.access >= oa.access + ) + ) x + group by x.tree_path +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- build_tree_access +-- +-- The bridge from core's access model to the space data-plane functions: +-- resolves a member's (user or agent) effective grants in a space and returns +-- them as the jsonb array shape that space.search_memory / *_memory consume via +-- jsonb_to_recordset(...) x(tree_path ltree, access int). +------------------------------------------------------------------------------- +create or replace function {{schema}}.build_tree_access +( _member_id uuid +, _space_id uuid +) +returns jsonb +as $func$ + with access as + ( + select ta.tree_path, ta.access + from {{schema}}.principal p + cross join lateral + ( + -- dispatch on kind; the off-kind branch's id column is null -> no rows + select uta.tree_path, uta.access + from {{schema}}.user_tree_access(p.user_id, _space_id) uta + where p.kind = 'u' + union all + select ata.tree_path, ata.access + from {{schema}}.agent_tree_access(p.agent_id, _space_id) ata + where p.kind = 'a' + ) ta + where p.member_id = _member_id + ) + select coalesce + ( + jsonb_agg(jsonb_build_object('tree_path', a.tree_path::text, 'access', a.access)) + , '[]'::jsonb + ) + from access a +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/004_space.sql b/packages/database/core/migrate/idempotent/004_space.sql new file mode 100644 index 0000000..1e7ffa7 --- /dev/null +++ b/packages/database/core/migrate/idempotent/004_space.sql @@ -0,0 +1,146 @@ +------------------------------------------------------------------------------- +-- create_space +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_space +( _slug text +, _name text +, _language text default 'english' +) +returns uuid +as $func$ + insert into {{schema}}.space (slug, name, language) + values (_slug, _name, coalesce(_language, 'english')) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- rename_space +------------------------------------------------------------------------------- +create or replace function {{schema}}.rename_space +( _slug text +, _name text +) +returns bool +as $func$ + with u as + ( + update {{schema}}.space set name = _name where slug = _slug returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_space +-- Deletes the core.space row; FKs cascade its memberships, groups, grants, and +-- group memberships. The me_ data schema is dropped separately by the +-- caller (DDL). Returns true if a space with this slug existed. +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_space +( _slug text +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.space where slug = _slug returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_space +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_space +( _slug text +) +returns table +( id uuid +, slug text +, name text +, language text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select s.id, s.slug, s.name::text, s.language, s.created_at, s.updated_at + from {{schema}}.space s + where s.slug = _slug +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_spaces +-- All spaces, newest first. Used by the embedding worker to discover the +-- me_ data schemas to process. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_spaces() +returns table +( id uuid +, slug text +, name text +, language text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select s.id, s.slug, s.name::text, s.language, s.created_at, s.updated_at + from {{schema}}.space s + order by s.created_at desc +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_spaces_for_member +-- Spaces a member (user/agent) belongs to — directly (principal_space) or +-- through a group (Model 2). `admin` is the direct-membership admin flag. +-- Used by the user endpoint so a logged-in human can pick their space. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_spaces_for_member +( _member_id uuid +) +returns table +( id uuid +, slug text +, name text +, language text +, admin bool +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + -- Drive from the membership tables (indexed by the member) and PK-join to + -- space, rather than scanning every space and probing membership per row. + with space_ids as + ( + select ps.space_id + from {{schema}}.principal_space ps + where ps.principal_id = _member_id + union + select gm.space_id + from {{schema}}.group_member gm + where gm.member_id = _member_id + ) + select + s.id + , s.slug + , s.name::text + , s.language + -- derived from is_principal_space_admin so it matches the authority gate + -- (includes admin inherited via an admin group) + , {{schema}}.is_principal_space_admin(_member_id, s.id) as admin + , s.created_at + , s.updated_at + from {{schema}}.space s + inner join space_ids si on si.space_id = s.id + order by s.created_at desc +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/005_principal.sql b/packages/database/core/migrate/idempotent/005_principal.sql new file mode 100644 index 0000000..0bf6dbf --- /dev/null +++ b/packages/database/core/migrate/idempotent/005_principal.sql @@ -0,0 +1,200 @@ +------------------------------------------------------------------------------- +-- create_user +-- Users are global (no space_id, no owner_id). The id is supplied by the caller +-- so it equals auth.users.id (one identity across auth + core). +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_user +( _id uuid +, _name text +) +returns uuid +as $func$ + insert into {{schema}}.principal (id, kind, name) + values (_id, 'u', _name) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- create_agent +-- Agents are owned by a user (owner_id -> a user principal's id) and are global. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_agent +( _owner_id uuid +, _name text +, _id uuid default null +) +returns uuid +as $func$ + insert into {{schema}}.principal (id, kind, name, owner_id) + values (coalesce(_id, uuidv7()), 'a', _name, _owner_id) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- create_group +-- Groups belong to a single space. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_group +( _space_id uuid +, _name text +, _id uuid default null +) +returns uuid +as $func$ + insert into {{schema}}.principal (id, kind, name, space_id) + values (coalesce(_id, uuidv7()), 'g', _name, _space_id) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_principal +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_principal +( _id uuid +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, space_id uuid +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.kind, p.name::text, p.owner_id, p.space_id, p.created_at, p.updated_at + from {{schema}}.principal p + where p.id = _id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_user_by_name +-- Resolve a global user (kind 'u') by name. User names are globally unique +-- (citext), so this returns at most one row. +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_user_by_name +( _name text +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, space_id uuid +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.kind, p.name::text, p.owner_id, p.space_id, p.created_at, p.updated_at + from {{schema}}.principal p + where p.kind = 'u' + and p.name = _name::citext +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_agents +-- A user's agents (global; agents are owned by a user, not scoped to a space). +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_agents +( _owner_id uuid +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, space_id uuid +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.kind, p.name::text, p.owner_id, p.space_id, p.created_at, p.updated_at + from {{schema}}.principal p + where p.kind = 'a' + and p.owner_id = _owner_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_space_groups +-- All groups belonging to a space (groups are space-scoped via space_id). +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_space_groups +( _space_id uuid +) +returns table +( id uuid +, name text +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select p.id, p.name::text, p.created_at, p.updated_at + from {{schema}}.principal p + where p.kind = 'g' + and p.space_id = _space_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- rename_principal +-- Rename an agent or group. Users are intentionally excluded: a user's name is +-- its email — the global identity handle that mirrors auth.users — so changing +-- it is an account concern, not a space-management one. Returns true if an +-- agent/group with this id was renamed. Name uniqueness is enforced by the +-- principal table indexes. +------------------------------------------------------------------------------- +create or replace function {{schema}}.rename_principal +( _id uuid +, _name text +) +returns bool +as $func$ + with u as + ( + update {{schema}}.principal + set name = _name::citext + where id = _id + and kind in ('a', 'g') -- never rename users (kind 'u') + returning 1 + ) + select exists (select 1 from u) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_principal +-- Delete a principal row. Foreign keys cascade: a user's agents (owner_id), +-- its space memberships, group memberships, tree-access grants, and api keys +-- all go with it. Returns true if a row was deleted. +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_principal +( _id uuid +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.principal + where id = _id + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/006_membership.sql b/packages/database/core/migrate/idempotent/006_membership.sql new file mode 100644 index 0000000..9d318e8 --- /dev/null +++ b/packages/database/core/migrate/idempotent/006_membership.sql @@ -0,0 +1,221 @@ +------------------------------------------------------------------------------- +-- add_principal_to_space +-- Adds (or updates the admin flag of) a principal's membership in a space, and +-- grants a joining user owner over its home directory. The single chokepoint +-- every join path goes through (provisioning, invite redemption, direct add), +-- so a user's membership always implies home ownership. +------------------------------------------------------------------------------- +create or replace function {{schema}}.add_principal_to_space +( _space_id uuid +, _principal_id uuid +, _admin bool default false +) +returns void +as $func$ + insert into {{schema}}.principal_space (space_id, principal_id, admin) + values (_space_id, _principal_id, _admin) + on conflict (principal_id, space_id) do update set + admin = excluded.admin; -- updated_at maintained by the before-update trigger + + -- A user owns its home directory (home., hyphens stripped); see + -- packages/database/space/path.ts homePrefix() for the matching client form. + -- Users only: an agent's effective grants are clamped to its owner's by + -- agent_tree_access, so an auto home grant would be inert (the owner has no + -- access over the agent's home); groups have no home either. Idempotent and + -- non-clobbering: an existing home grant is left untouched. + insert into {{schema}}.tree_access (space_id, principal_id, tree_path, access) + select _space_id, _principal_id + , ('home.' || replace(_principal_id::text, '-', ''))::ltree + , 3 -- owner + from {{schema}}.principal p + where p.id = _principal_id + and p.kind = 'u' + on conflict (space_id, principal_id, tree_path) do nothing +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- add_group_member +-- Adds a user/agent member to a group within a space. +------------------------------------------------------------------------------- +create or replace function {{schema}}.add_group_member +( _space_id uuid +, _group_id uuid +, _member_id uuid +, _admin bool default false +) +returns void +as $func$ + insert into {{schema}}.group_member (space_id, group_id, member_id, admin) + values (_space_id, _group_id, _member_id, _admin) + on conflict (space_id, member_id, group_id) do update set + admin = excluded.admin -- updated_at maintained by the before-update trigger +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- remove_principal_from_space +-- Removes a principal from a space and cascades: scrubs its tree_access grants +-- and its group_member rows in that space (both as a member and, if it is a +-- group, its members). Returns true if the principal was a member of the space. +-- (Space-scoped only; the principal row itself and any other spaces are left +-- untouched.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.remove_principal_from_space +( _space_id uuid +, _principal_id uuid +) +returns bool +as $func$ + with del_grants as + ( + delete from {{schema}}.tree_access + where space_id = _space_id + and principal_id = _principal_id + ) + , del_group_member as + ( + delete from {{schema}}.group_member + where space_id = _space_id + and (member_id = _principal_id or group_id = _principal_id) + ) + , del_membership as + ( + delete from {{schema}}.principal_space + where space_id = _space_id + and principal_id = _principal_id + returning 1 + ) + select exists (select 1 from del_membership) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- remove_group_member +-- Removes a member from a group within a space. Returns true if a row was removed. +------------------------------------------------------------------------------- +create or replace function {{schema}}.remove_group_member +( _space_id uuid +, _group_id uuid +, _member_id uuid +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.group_member + where space_id = _space_id + and group_id = _group_id + and member_id = _member_id + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_space_principals +-- Principals that belong to a space, deduplicated: either added directly +-- (principal_space) or reached through a group in the space (group_member) — +-- group membership confers space access, so both count. `direct` is true when +-- the principal has a direct membership row; `admin` is its direct-membership +-- admin flag (false for group-only members). Optional kind filter +-- ('u' | 'a' | 'g'); null returns all. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_space_principals +( _space_id uuid +, _kind text default null +) +returns table +( id uuid +, kind text +, name text +, owner_id uuid +, direct bool +, admin bool +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + with mem as + ( + -- directly added to the space + select ps.principal_id as id, true as direct, ps.admin as admin + from {{schema}}.principal_space ps + where ps.space_id = _space_id + union all + -- reached through a group belonging to the space + select gm.member_id as id, false as direct, false as admin + from {{schema}}.group_member gm + where gm.space_id = _space_id + ) + , agg as + ( + select id, bool_or(direct) as direct, bool_or(admin) as admin + from mem + group by id + ) + select p.id, p.kind, p.name::text, p.owner_id, agg.direct, agg.admin, p.created_at, p.updated_at + from agg + join {{schema}}.principal p on p.id = agg.id + where (_kind is null or p.kind = _kind) + order by p.kind, p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_group_members +-- Members (users / agents) of a group within a space, with the admin flag. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_group_members +( _space_id uuid +, _group_id uuid +) +returns table +( member_id uuid +, kind text +, name text +, admin bool +, created_at timestamptz +) +as $func$ + select gm.member_id, p.kind, p.name::text, gm.admin, gm.created_at + from {{schema}}.group_member gm + join {{schema}}.principal p on p.id = gm.member_id + where gm.space_id = _space_id + and gm.group_id = _group_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_groups_for_member +-- Groups within a space that a member (user / agent) belongs to, with the +-- admin flag. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_groups_for_member +( _space_id uuid +, _member_id uuid +) +returns table +( group_id uuid +, name text +, admin bool +, created_at timestamptz +) +as $func$ + select gm.group_id, p.name::text, gm.admin, gm.created_at + from {{schema}}.group_member gm + join {{schema}}.principal p on p.id = gm.group_id + where gm.space_id = _space_id + and gm.member_id = _member_id + order by p.name +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/007_grant.sql b/packages/database/core/migrate/idempotent/007_grant.sql new file mode 100644 index 0000000..6dbf5a0 --- /dev/null +++ b/packages/database/core/migrate/idempotent/007_grant.sql @@ -0,0 +1,77 @@ +------------------------------------------------------------------------------- +-- grant_tree_access +-- Grants (or updates) a principal's access at a tree path in a space. +-- access: 1 = read, 2 = write, 3 = owner. Access is purely additive (grants); +-- there are no deny entries. +------------------------------------------------------------------------------- +create or replace function {{schema}}.grant_tree_access +( _space_id uuid +, _principal_id uuid +, _tree_path ltree +, _access int +) +returns void +as $func$ + insert into {{schema}}.tree_access (space_id, principal_id, tree_path, access) + values (_space_id, _principal_id, _tree_path, _access) + on conflict (space_id, principal_id, tree_path) do update set + access = excluded.access -- updated_at maintained by the before-update trigger +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- remove_tree_access_grant +-- Removes a single grant. Returns true if a row was removed. (No deny perms, +-- so removing a grant simply drops that access.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.remove_tree_access_grant +( _space_id uuid +, _principal_id uuid +, _tree_path ltree +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.tree_access + where space_id = _space_id + and principal_id = _principal_id + and tree_path = _tree_path + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_tree_access_grants +-- The grant rows in a space, optionally filtered to a single principal and/or +-- to a subtree (_under: only grants at-or-below this path, i.e. _under @> path). +-- (Owner listing is this filtered to access = 3 by the caller.) Distinct from +-- build_tree_access, which resolves a member's *effective* access set; this +-- lists the raw grants. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_tree_access_grants +( _space_id uuid +, _principal_id uuid default null +, _under ltree default null +) +returns table +( principal_id uuid +, tree_path text +, access int +, created_at timestamptz +, updated_at timestamptz +) +as $func$ + select t.principal_id, t.tree_path::text, t.access, t.created_at, t.updated_at + from {{schema}}.tree_access t + where t.space_id = _space_id + and (_principal_id is null or t.principal_id = _principal_id) + and (_under is null or _under @> t.tree_path) + order by t.principal_id, t.tree_path +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/008_api_key.sql b/packages/database/core/migrate/idempotent/008_api_key.sql new file mode 100644 index 0000000..f01cdda --- /dev/null +++ b/packages/database/core/migrate/idempotent/008_api_key.sql @@ -0,0 +1,111 @@ +------------------------------------------------------------------------------- +-- create_api_key +-- The caller generates the key (lookup_id + secret) and passes the *hashed* +-- secret; we never store the plaintext. Scoped to a member (user or agent). +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_api_key +( _member_id uuid +, _lookup_id text +, _secret text -- already hashed by the caller +, _name text +, _expires_at timestamptz default null +) +returns uuid +as $func$ + insert into {{schema}}.api_key (member_id, lookup_id, secret, name, expires_at) + values (_member_id, _lookup_id, _secret, _name, _expires_at) + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- validate_api_key +-- Looks a key up by lookup_id, compares the hashed secret, and enforces expiry. +-- Returns the member_id + api_key id when valid; no rows otherwise. +------------------------------------------------------------------------------- +create or replace function {{schema}}.validate_api_key +( _lookup_id text +, _secret text -- hashed +) +returns table +( member_id uuid +, api_key_id uuid +) +as $func$ + select k.member_id, k.id + from {{schema}}.api_key k + where k.lookup_id = _lookup_id + and k.secret = _secret + and (k.expires_at is null or k.expires_at > now()) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- get_api_key +-- Key metadata by id (never the secret). +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_api_key +( _id uuid +) +returns table +( id uuid +, member_id uuid +, lookup_id text +, name text +, created_at timestamptz +, expires_at timestamptz +) +as $func$ + select k.id, k.member_id, k.lookup_id, k.name, k.created_at, k.expires_at + from {{schema}}.api_key k + where k.id = _id +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_api_keys +-- A member's keys (never the secret), newest first. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_api_keys +( _member_id uuid +) +returns table +( id uuid +, member_id uuid +, lookup_id text +, name text +, created_at timestamptz +, expires_at timestamptz +) +as $func$ + select k.id, k.member_id, k.lookup_id, k.name, k.created_at, k.expires_at + from {{schema}}.api_key k + where k.member_id = _member_id + order by k.created_at desc +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete_api_key +-- Hard-delete a key by id. Returns true if a row was deleted. (There is no +-- soft-revoke state; revoke and delete are the same operation.) +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_api_key +( _id uuid +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.api_key + where id = _id + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/009_invitation.sql b/packages/database/core/migrate/idempotent/009_invitation.sql new file mode 100644 index 0000000..55868d7 --- /dev/null +++ b/packages/database/core/migrate/idempotent/009_invitation.sql @@ -0,0 +1,123 @@ +------------------------------------------------------------------------------- +-- create_space_invitation +-- Issue (or update, if one is already pending) an invitation to a space, keyed +-- by invitee email. _share_access null means no share grant; otherwise it is +-- the level (1/2/3) granted at the shared root on redemption. Returns the id. +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_space_invitation +( _space_id uuid +, _email citext +, _admin bool +, _share_access int +, _invited_by uuid +) +returns uuid +as $func$ + insert into {{schema}}.space_invitation (space_id, email, admin, share_access, invited_by) + values (_space_id, _email, _admin, _share_access, _invited_by) + on conflict (space_id, email) where accepted_at is null do update set + admin = excluded.admin + , share_access = excluded.share_access + , invited_by = excluded.invited_by -- updated_at maintained by the before-update trigger + returning id +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_space_invitations +-- Pending invitations for a space (accepted ones are history), with the +-- inviter's display name when still resolvable. +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_space_invitations +( _space_id uuid +) +returns table +( id uuid +, email text +, admin bool +, share_access int +, invited_by uuid +, invited_by_name text +, created_at timestamptz +) +as $func$ + select i.id, i.email::text, i.admin, i.share_access, i.invited_by, p.name::text, i.created_at + from {{schema}}.space_invitation i + left join {{schema}}.principal p on p.id = i.invited_by + where i.space_id = _space_id + and i.accepted_at is null + order by i.created_at +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- revoke_space_invitation +-- Delete a pending invitation by email. Returns true if one was removed. An +-- already-accepted invitation is not revocable here (the user is a member; +-- use remove_principal_from_space). +------------------------------------------------------------------------------- +create or replace function {{schema}}.revoke_space_invitation +( _space_id uuid +, _email citext +) +returns bool +as $func$ + with d as + ( + delete from {{schema}}.space_invitation + where space_id = _space_id + and email = _email + and accepted_at is null + returning 1 + ) + select exists (select 1 from d) +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- redeem_space_invitations +-- Redeem every pending invitation for a (now-registered, verified) email: join +-- the user to each space (add_principal_to_space also grants owner@home), grant +-- access at the shared root 'share' when share_access is set, and stamp +-- accepted_at. Idempotent — a second call finds nothing pending. The user must +-- already exist as a core principal. Returns one row per space joined. +------------------------------------------------------------------------------- +create or replace function {{schema}}.redeem_space_invitations +( _user_id uuid +, _email citext +) +returns table +( space_id uuid +, slug text +, name text +, admin bool +, share_access int +) +as $func$ +declare + inv record; +begin + for inv in + select i.id, i.space_id, i.admin, i.share_access + from {{schema}}.space_invitation i + where i.email = _email + and i.accepted_at is null + for update + loop + perform {{schema}}.add_principal_to_space(inv.space_id, _user_id, inv.admin); + if inv.share_access is not null then + perform {{schema}}.grant_tree_access(inv.space_id, _user_id, 'share'::ltree, inv.share_access); + end if; + update {{schema}}.space_invitation set accepted_at = pg_catalog.now() where id = inv.id; + return query + select s.id, s.slug, s.name::text, inv.admin, inv.share_access + from {{schema}}.space s + where s.id = inv.space_id; + end loop; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/core/migrate/idempotent/sql.d.ts b/packages/database/core/migrate/idempotent/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/core/migrate/idempotent/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/core/migrate/incremental/001_space.sql b/packages/database/core/migrate/incremental/001_space.sql new file mode 100644 index 0000000..a8ea560 --- /dev/null +++ b/packages/database/core/migrate/incremental/001_space.sql @@ -0,0 +1,12 @@ +------------------------------------------------------------------------------- +-- space +------------------------------------------------------------------------------- +create table {{schema}}.space +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, slug text not null unique check (slug ~ '^[a-z0-9]{12}$') +, name citext not null +, language text not null default 'english' check (language ~ '^[a-z_]+$') +-- we likely need columns for embedding provider, model, dimensions +, created_at timestamptz not null default now() +, updated_at timestamptz +); diff --git a/packages/database/core/migrate/incremental/002_principal.sql b/packages/database/core/migrate/incremental/002_principal.sql new file mode 100644 index 0000000..b5c590e --- /dev/null +++ b/packages/database/core/migrate/incremental/002_principal.sql @@ -0,0 +1,35 @@ +------------------------------------------------------------------------------- +-- principal +------------------------------------------------------------------------------- +create table {{schema}}.principal +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid unique nulls distinct generated always as (case when kind = 'u' then id else null end) stored +, group_id uuid unique nulls distinct generated always as (case when kind = 'g' then id else null end) stored +, agent_id uuid unique nulls distinct generated always as (case when kind = 'a' then id else null end) stored +, member_id uuid unique nulls distinct generated always as (case when kind in ('u', 'a') then id else null end) stored +, owner_id uuid references {{schema}}.principal (user_id) on delete cascade -- points to agent's owner +, space_id uuid references {{schema}}.space (id) on delete cascade +, kind text not null check (kind in ('g', 'u', 'a')) -- group, user, agent +, name citext not null check (name::text !~ '/') -- agent names are displayed as / +, created_at timestamptz not null default now() +, updated_at timestamptz +, check + ( + (kind = 'a' and owner_id is not null) -- agents are owned by a user + or + (kind != 'a' and owner_id is null) -- users and groups have no owner + ) +, check + ( + (kind = 'g' and space_id is not null) -- groups belong to a single space + or + (kind != 'g' and space_id is null) -- users and agents are global + ) +); + +-- users must have a globally unique name +create unique index on {{schema}}.principal (name) include (user_id) where user_id is not null; +-- each user's agents must have a unique name (per that user) +create unique index on {{schema}}.principal (owner_id, name) where agent_id is not null; +-- each space's groups must have a unique name (per that space) +create unique index on {{schema}}.principal (space_id, name) where group_id is not null; diff --git a/packages/database/core/migrate/incremental/003_principal_space.sql b/packages/database/core/migrate/incremental/003_principal_space.sql new file mode 100644 index 0000000..ede33e0 --- /dev/null +++ b/packages/database/core/migrate/incremental/003_principal_space.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- principal_space +------------------------------------------------------------------------------- +create table {{schema}}.principal_space +( space_id uuid not null references {{schema}}.space (id) on delete cascade +, principal_id uuid not null references {{schema}}.principal (id) on delete cascade -- can be users, agents, or groups +, admin bool not null default false +, created_at timestamptz not null default now() +, updated_at timestamptz +, unique (principal_id, space_id) include (admin) +); + +create index on {{schema}}.principal_space (space_id, principal_id) include (admin); diff --git a/packages/database/core/migrate/incremental/004_group_member.sql b/packages/database/core/migrate/incremental/004_group_member.sql new file mode 100644 index 0000000..c041afc --- /dev/null +++ b/packages/database/core/migrate/incremental/004_group_member.sql @@ -0,0 +1,19 @@ +------------------------------------------------------------------------------- +-- group_member +------------------------------------------------------------------------------- +create table {{schema}}.group_member +( space_id uuid not null references {{schema}}.space (id) on delete cascade +, group_id uuid not null references {{schema}}.principal (group_id) on delete cascade -- can only be groups +, member_id uuid not null references {{schema}}.principal (member_id) on delete cascade -- can be users or agents, but not groups +, admin bool not null default false +, created_at timestamptz not null default now() +, updated_at timestamptz +, unique (space_id, member_id, group_id) include (admin) +); + +-- index for listing groups in a space and/or members of a group +create index on {{schema}}.group_member (space_id, group_id, member_id) include (admin); + +-- index for finding a member's spaces/groups across the whole space set +-- (e.g. list_spaces_for_member's member_id lookup, with no space_id filter) +create index on {{schema}}.group_member (member_id, space_id); diff --git a/packages/database/core/migrate/incremental/005_tree_access.sql b/packages/database/core/migrate/incremental/005_tree_access.sql new file mode 100644 index 0000000..2bf4114 --- /dev/null +++ b/packages/database/core/migrate/incremental/005_tree_access.sql @@ -0,0 +1,12 @@ +------------------------------------------------------------------------------- +-- tree_access +------------------------------------------------------------------------------- +create table {{schema}}.tree_access +( space_id uuid not null references {{schema}}.space (id) on delete cascade +, principal_id uuid not null references {{schema}}.principal (id) on delete cascade -- can be users, agents, or groups +, tree_path ltree not null +, access int not null check (access in (1, 2, 3)) -- 1 = read, 2 = write, 3 = owner +, created_at timestamptz not null default now() +, updated_at timestamptz +, unique (space_id, principal_id, tree_path) include (access) +); diff --git a/packages/database/core/migrate/incremental/006_api_key.sql b/packages/database/core/migrate/incremental/006_api_key.sql new file mode 100644 index 0000000..8d6418f --- /dev/null +++ b/packages/database/core/migrate/incremental/006_api_key.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- api_key +------------------------------------------------------------------------------- +create table {{schema}}.api_key +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, member_id uuid not null references {{schema}}.principal (member_id) on delete cascade -- may be users or agents, not groups +, lookup_id text unique not null check (lookup_id ~ '^[A-Za-z0-9_-]{16}$') +, secret text not null -- hashed secret +, name text not null +, created_at timestamptz not null default now() +, expires_at timestamptz +, unique (member_id, name) +); diff --git a/packages/database/core/migrate/incremental/007_space_invitation.sql b/packages/database/core/migrate/incremental/007_space_invitation.sql new file mode 100644 index 0000000..0c33a89 --- /dev/null +++ b/packages/database/core/migrate/incremental/007_space_invitation.sql @@ -0,0 +1,27 @@ +------------------------------------------------------------------------------- +-- space_invitation +-- Invitations to a space, keyed by invitee email so an invite can be issued +-- before the user registers. Redeemed at login (against the user's verified +-- email) by redeem_space_invitations, which joins the space (owner@home via +-- add_principal_to_space), optionally grants access at the shared root, and +-- stamps accepted_at. A *pending* invite is one with accepted_at is null; +-- accepted rows are kept as history. +------------------------------------------------------------------------------- +create table {{schema}}.space_invitation +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, space_id uuid not null references {{schema}}.space (id) on delete cascade +, email citext not null -- invitee (the key; may not be a user yet) +, admin bool not null default false -- make the user a space admin on redemption +, share_access int check (share_access in (1, 2, 3)) -- null = no share grant; else read/write/owner at 'share' +, invited_by uuid references {{schema}}.principal (id) on delete set null -- who issued it (audit) +, created_at timestamptz not null default now() +, updated_at timestamptz -- maintained by the before-update trigger +, accepted_at timestamptz -- null = pending; set on redemption +); + +-- at most one pending invite per (space, email); accepted rows are kept as +-- history, so the uniqueness is partial. email is citext, so the dedup is +-- case-insensitive. +create unique index space_invitation_pending_uq + on {{schema}}.space_invitation (space_id, email) + where accepted_at is null; diff --git a/packages/database/core/migrate/incremental/sql.d.ts b/packages/database/core/migrate/incremental/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/core/migrate/incremental/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/core/migrate/migrate.integration.test.ts b/packages/database/core/migrate/migrate.integration.test.ts new file mode 100644 index 0000000..c0f116d --- /dev/null +++ b/packages/database/core/migrate/migrate.integration.test.ts @@ -0,0 +1,819 @@ +// Integration tests for the `core` control-plane migrations (migrateCore). +// +// The core migrations are templated, so each test targets its own throwaway +// `core_test_` schema — never the real `core`. That makes these tests +// isolated and safe to run against any database (including a shared dev one). +// Read-only shape assertions share one canonical core provisioned in beforeAll; +// the few behavior tests provision their own. Tests run serially within the +// file; cross-suite parallelism comes from `bun run test:db` (separate +// processes for core and space). +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Sql as SQL } from "postgres"; +import { CORE_SCHEMA_VERSION } from "../version"; +import { migrateCore } from "./migrate"; +import { + appliedMigrations, + connect, + expectReject, + extensionInstalled, + getSchemaVersion, + listFunctions, + listTables, + listTriggers, + randomCoreSchema, + schemaExists, + TestCore, + tableExists, + withTestCore, +} from "./test-utils"; + +const EXPECTED_TABLES = [ + "api_key", + "group_member", + "migration", + "principal", + "principal_space", + "space", + "space_invitation", + "tree_access", + "version", +]; + +const EXPECTED_MIGRATIONS = [ + "001_space", + "002_principal", + "003_principal_space", + "004_group_member", + "005_tree_access", + "006_api_key", + "007_space_invitation", +]; + +const EXPECTED_FUNCTIONS = [ + "agent_tree_access", + "is_principal_in_space", + "is_principal_space_admin", + "member_groups", + "member_tree_access", + "update_updated_at", + "user_tree_access", +]; + +const REQUIRED_EXTENSIONS = ["citext", "ltree", "vector", "pg_textsearch"]; + +/** A valid space slug: 12 lowercase alphanumerics (see space.slug check). */ +function randomSlug(): string { + const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let slug = ""; + for (const b of bytes) slug += alphabet[b % 36]; + return slug; +} + +let sql: SQL; +// One migrated core shared by all read-only shape/function assertions. +let canonical: TestCore; + +beforeAll(async () => { + sql = connect(12); + canonical = await TestCore.create(sql); // migrateCore installs extensions itself +}); + +afterAll(async () => { + await canonical?.drop(); + await sql.end(); +}); + +describe("provisioned core schema", () => { + test("provisions into the requested (templated) schema", async () => { + expect(canonical.schema).toMatch(/^core_test_/); + expect(await schemaExists(sql, canonical.schema)).toBe(true); + }); + + test("creates infrastructure and domain tables", async () => { + const tables = await listTables(sql, canonical.schema); + for (const table of EXPECTED_TABLES) { + expect(tables).toContain(table); + } + }); + + test("records every incremental migration exactly once", async () => { + expect(await appliedMigrations(sql, canonical.schema)).toEqual( + EXPECTED_MIGRATIONS, + ); + }); + + test("stamps the schema version", async () => { + expect(await getSchemaVersion(sql, canonical.schema)).toBe( + CORE_SCHEMA_VERSION, + ); + }); + + test("installs all required extensions", async () => { + for (const ext of REQUIRED_EXTENSIONS) { + expect(await extensionInstalled(sql, ext)).toBe(true); + } + }); + + test("creates the access-control functions in the schema", async () => { + const functions = await listFunctions(sql, canonical.schema); + for (const fn of EXPECTED_FUNCTIONS) { + expect(functions).toContain(fn); + } + }); + + test("installs updated_at triggers on mutable tables", async () => { + for (const table of [ + "space", + "principal", + "principal_space", + "group_member", + "tree_access", + "space_invitation", + ]) { + const triggers = await listTriggers(sql, canonical.schema, table); + expect(triggers).toContain(`${table}_before_update_trg`); + } + }); +}); + +describe("schema constraints enforce", () => { + test("principal.kind is restricted to g/u/a", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.principal (kind, name) values ('x', 'bad-kind')`, + ), + ); + }); + + test("principal ids must be UUIDv7", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.principal (id, kind, name) + values ('00000000-0000-4000-8000-000000000000', 'u', 'v4-id')`, + ), + ); + }); + + test("space.slug must be 12 lowercase alphanumerics", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.space (slug, name) values ('BAD', 'x')`, + ), + ); + }); + + test("user names are globally unique", async () => { + const name = `smoke_unique_${crypto.randomUUID().slice(0, 8)}`; + await sql.unsafe( + `insert into ${canonical.schema}.principal (kind, name) values ('u', '${name}')`, + ); + try { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.principal (kind, name) values ('u', '${name}')`, + ), + ); + } finally { + await sql.unsafe( + `delete from ${canonical.schema}.principal where name = '${name}'`, + ); + } + }); +}); + +describe("access-control functions are callable", () => { + // Catches functions that "exist" but reference missing columns/types: a bad + // body only errors when executed, not when created. + const dummy = "00000000-0000-7000-8000-000000000000"; + + test("access functions execute against empty data", async () => { + const s = canonical.schema; + await sql.unsafe( + `select * from ${s}.user_tree_access('${dummy}', '${dummy}')`, + ); + await sql.unsafe( + `select * from ${s}.agent_tree_access('${dummy}', '${dummy}')`, + ); + await sql.unsafe( + `select * from ${s}.member_tree_access('${dummy}', '${dummy}')`, + ); + await sql.unsafe( + `select * from ${s}.member_groups('${dummy}', '${dummy}')`, + ); + }); + + test("predicate functions return false for unknown principals", async () => { + const s = canonical.schema; + const [a] = await sql.unsafe( + `select ${s}.is_principal_in_space('${dummy}', '${dummy}') as v`, + ); + expect(a?.v).toBe(false); + const [b] = await sql.unsafe( + `select ${s}.is_principal_space_admin('${dummy}', '${dummy}') as v`, + ); + expect(b?.v).toBe(false); + }); +}); + +describe("agent_tree_access clamps agent access to its owner", () => { + // Regression for the `max(x.access)` + `group by tree_path` at the end of + // agent_tree_access (idempotent/003_tree_access.sql). Setup: + // + // agent grants: foo = owner(3), foo.bar = read(1) <- foo.bar is redundant, + // owner grants: foo.bar = write(2) already covered by foo=3 + // + // The inner UNION then emits foo.bar twice with different access levels: + // * arm 1 keeps the agent's (foo.bar, read) — the owner's foo.bar covers it + // * arm 2 keeps the owner's (foo.bar, write) — the agent's foo covers it + // Without the trailing max/group-by, agent_tree_access would return foo.bar + // twice; the effective access is the highest surviving row, (foo.bar, write). + // `foo` itself never surfaces — the owner grants nothing at or above it. + test("collapses the two clamp directions into one row per path", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + + const [space] = await sql.unsafe( + `insert into ${s}.space (slug, name) values ($1, $2) returning id`, + [randomSlug(), "clamp"], + ); + const spaceId = space?.id as string; + + const [owner] = await sql.unsafe( + `insert into ${s}.principal (kind, name) values ('u', 'owner') returning id`, + ); + const ownerId = owner?.id as string; + + const [agent] = await sql.unsafe( + `insert into ${s}.principal (kind, name, owner_id) values ('a', 'agent', $1) returning id`, + [ownerId], + ); + const agentId = agent?.id as string; + + // both principals must belong to the space for the access functions to see them + await sql.unsafe( + `insert into ${s}.principal_space (space_id, principal_id) values ($1, $2), ($1, $3)`, + [spaceId, ownerId, agentId], + ); + + await sql.unsafe( + `insert into ${s}.tree_access (space_id, principal_id, tree_path, access) values + ($1, $2, 'foo', 3), + ($1, $2, 'foo.bar', 1), + ($1, $3, 'foo.bar', 2)`, + [spaceId, agentId, ownerId], + ); + + const rows = await sql.unsafe( + `select tree_path::text as tree_path, access + from ${s}.agent_tree_access($1, $2) + order by tree_path`, + [agentId, spaceId], + ); + const result = rows.map((r) => ({ + tree_path: r.tree_path as string, + access: r.access as number, + })); + + // One clamped row, access collapsed to the max of the two union arms. + expect(result).toEqual([{ tree_path: "foo.bar", access: 2 }]); + }); + }); +}); + +describe("control-plane functions", () => { + /** A fresh uuidv7 from the database (principal.id requires version 7). */ + async function v7(): Promise { + const [row] = await sql.unsafe(`select uuidv7() as id`); + return row?.id as string; + } + + /** A principal's canonical home path (mirrors space/path.ts homePrefix). */ + const homePath = (id: string) => `home.${id.replace(/-/g, "")}`; + + type Grant = { tree_path: string; access: number }; + + test("create_space + create_user + grant → build_tree_access returns the search_memory jsonb shape", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Test Space", + ]); + const spaceId = sp?.id as string; + + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + userId, + "alice", + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + true, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + userId, + "work.projects", + 2, + ]); + + const [row] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + const ta = row?.ta as Grant[]; + // add_principal_to_space also grants the user owner@home; the explicit + // grant adds to it. + expect(ta).toContainEqual({ tree_path: "work.projects", access: 2 }); + expect(ta).toContainEqual({ tree_path: homePath(userId), access: 3 }); + expect(ta).toHaveLength(2); + }); + }); + + test("add_principal_to_space grants owner@home to users, idempotently; not agents/groups", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Homes", + ]); + const spaceId = sp?.id as string; + + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "homer"]); + const agentId = await v7(); + await sql.unsafe(`select ${s}.create_agent($1, $2, $3)`, [ + userId, // owner + `agent_${randomSlug()}`, // name + agentId, // id + ]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + `grp_${randomSlug()}`, + ]); + const groupId = grp?.id as string; + + // add each twice to prove the home grant is idempotent + for (const id of [userId, agentId, groupId]) { + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + id, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + id, + false, + ]); + } + + const grants = async (id: string): Promise => { + const rows = await sql.unsafe( + `select tree_path::text, access from ${s}.tree_access + where space_id = $1 and principal_id = $2`, + [spaceId, id], + ); + return rows as unknown as Grant[]; + }; + // the user gets exactly one owner@home grant (not duplicated by re-add) + expect(await grants(userId)).toEqual([ + { tree_path: homePath(userId), access: 3 }, + ]); + // agents are excluded (would be clamped to the owner) and groups have no home + expect(await grants(agentId)).toEqual([]); + expect(await grants(groupId)).toEqual([]); + }); + }); + + test("build_tree_access includes access granted via a group", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Team Space", + ]); + const spaceId = sp?.id as string; + + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2) as id`, [ + userId, + "bob", + ]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + "engineering", + ]); + const groupId = grp?.id as string; + + // both the user and the group must be members of the space + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + groupId, + false, + ]); + await sql.unsafe(`select ${s}.add_group_member($1, $2, $3, $4)`, [ + spaceId, + groupId, + userId, + false, + ]); + // grant to the GROUP, not the user + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + groupId, + "shared.docs", + 1, + ]); + + const [row] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + const ta = row?.ta as Grant[]; + expect(ta).toContainEqual({ tree_path: "shared.docs", access: 1 }); + }); + }); + + test("remove_group_member revokes group-inherited access", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Team", + ]); + const spaceId = sp?.id as string; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "erin"]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + "ops", + ]); + const groupId = grp?.id as string; + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + groupId, + false, + ]); + await sql.unsafe(`select ${s}.add_group_member($1, $2, $3, $4)`, [ + spaceId, + groupId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + groupId, + "team.notes", + 2, + ]); + + // sanity: access is inherited via the group + const [before] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + expect(before?.ta as Grant[]).toContainEqual({ + tree_path: "team.notes", + access: 2, + }); + + const [removed] = await sql.unsafe( + `select ${s}.remove_group_member($1, $2, $3) as removed`, + [spaceId, groupId, userId], + ); + expect(removed?.removed).toBe(true); + + const [after] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + // still a space member (only left the group): the group grant is gone, + // but the user keeps its own home. + expect(after?.ta).toEqual([{ tree_path: homePath(userId), access: 3 }]); + + // second remove is a no-op + const [again] = await sql.unsafe( + `select ${s}.remove_group_member($1, $2, $3) as removed`, + [spaceId, groupId, userId], + ); + expect(again?.removed).toBe(false); + }); + }); + + test("remove_principal_from_space cascades grants + group memberships (space-scoped)", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Cascade", + ]); + const spaceId = sp?.id as string; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "frank"]); + const [grp] = await sql.unsafe(`select ${s}.create_group($1, $2) as id`, [ + spaceId, + "team", + ]); + const groupId = grp?.id as string; + + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + groupId, + false, + ]); + await sql.unsafe(`select ${s}.add_group_member($1, $2, $3, $4)`, [ + spaceId, + groupId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + userId, + "direct", + 2, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + groupId, + "shared", + 1, + ]); + + const [removed] = await sql.unsafe( + `select ${s}.remove_principal_from_space($1, $2) as removed`, + [spaceId, userId], + ); + expect(removed?.removed).toBe(true); + + const count = async (table: string, col: string, id: string) => { + const [r] = await sql.unsafe( + `select count(*)::int as n from ${s}.${table} where space_id=$1 and ${col}=$2`, + [spaceId, id], + ); + return Number(r?.n); + }; + // the user's membership, direct grant, and group membership are all gone + expect(await count("principal_space", "principal_id", userId)).toBe(0); + expect(await count("tree_access", "principal_id", userId)).toBe(0); + expect(await count("group_member", "member_id", userId)).toBe(0); + // the group itself and its own grant are untouched + expect(await count("principal_space", "principal_id", groupId)).toBe(1); + expect(await count("tree_access", "principal_id", groupId)).toBe(1); + }); + }); + + test("remove_tree_access_grant drops the grant", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Sp", + ]); + const spaceId = sp?.id as string; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "carol"]); + await sql.unsafe(`select ${s}.add_principal_to_space($1, $2, $3)`, [ + spaceId, + userId, + false, + ]); + await sql.unsafe(`select ${s}.grant_tree_access($1, $2, $3::ltree, $4)`, [ + spaceId, + userId, + "a.b", + 3, + ]); + + const [first] = await sql.unsafe( + `select ${s}.remove_tree_access_grant($1, $2, $3::ltree) as removed`, + [spaceId, userId, "a.b"], + ); + expect(first?.removed).toBe(true); + const [second] = await sql.unsafe( + `select ${s}.remove_tree_access_grant($1, $2, $3::ltree) as removed`, + [spaceId, userId, "a.b"], + ); + expect(second?.removed).toBe(false); + + const [row] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + // the explicit a.b grant is gone; the user keeps its own home. + expect(row?.ta).toEqual([{ tree_path: homePath(userId), access: 3 }]); + }); + }); + + test("space invitations: create (upsert) / list / redeem (join + home + share) / revoke", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const [sp] = await sql.unsafe(`select ${s}.create_space($1, $2) as id`, [ + randomSlug(), + "Invites", + ]); + const spaceId = sp?.id as string; + + // inviter must exist as a principal (invited_by FK) + const inviterId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [ + inviterId, + "inviter@example.com", + ]); + + const email = "invitee@example.com"; + const create = (admin: boolean, share: number | null) => + sql.unsafe( + `select ${s}.create_space_invitation($1, $2, $3, $4, $5) as id`, + [spaceId, email, admin, share, inviterId], + ); + + // create (read share, not admin), then re-create promotes the SAME pending + // row to admin + owner share (upsert, not a duplicate) + const [c1] = await create(false, 1); + const inviteId = c1?.id as string; + expect(inviteId).toBeTruthy(); + const [c2] = await create(true, 3); + expect(c2?.id).toBe(inviteId); + + // list: one pending invite with the updated fields + the inviter's name + const listed = await sql.unsafe( + `select * from ${s}.list_space_invitations($1)`, + [spaceId], + ); + expect(listed).toHaveLength(1); + expect(listed[0]?.email).toBe(email); + expect(listed[0]?.admin).toBe(true); + expect(listed[0]?.share_access).toBe(3); + expect(listed[0]?.invited_by_name).toBe("inviter@example.com"); + + // the invitee registers, then redeems (email match is case-insensitive) + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, email]); + const redeemed = await sql.unsafe( + `select * from ${s}.redeem_space_invitations($1, $2)`, + [userId, "INVITEE@EXAMPLE.COM"], + ); + expect(redeemed).toHaveLength(1); + expect(redeemed[0]?.space_id).toBe(spaceId); + expect(redeemed[0]?.admin).toBe(true); + expect(redeemed[0]?.share_access).toBe(3); + + // joined as admin, with owner@home (add_principal_to_space) + owner@share + const [ps] = await sql.unsafe( + `select admin from ${s}.principal_space where space_id=$1 and principal_id=$2`, + [spaceId, userId], + ); + expect(ps?.admin).toBe(true); + const [taRow] = await sql.unsafe( + `select ${s}.build_tree_access($1, $2) as ta`, + [userId, spaceId], + ); + const ta = taRow?.ta as Grant[]; + expect(ta).toContainEqual({ tree_path: homePath(userId), access: 3 }); + expect(ta).toContainEqual({ tree_path: "share", access: 3 }); + + // accepted: gone from the pending list, and re-redeem is a no-op + expect( + await sql.unsafe(`select * from ${s}.list_space_invitations($1)`, [ + spaceId, + ]), + ).toHaveLength(0); + expect( + await sql.unsafe( + `select * from ${s}.redeem_space_invitations($1, $2)`, + [userId, email], + ), + ).toHaveLength(0); + + // revoke: a fresh pending invite is revocable once + await create(false, null); // re-invite the same email (now allowed: prior is accepted) + const [r1] = await sql.unsafe( + `select ${s}.revoke_space_invitation($1, $2) as ok`, + [spaceId, email], + ); + expect(r1?.ok).toBe(true); + const [r2] = await sql.unsafe( + `select ${s}.revoke_space_invitation($1, $2) as ok`, + [spaceId, email], + ); + expect(r2?.ok).toBe(false); + }); + }); + + test("create_api_key + validate_api_key (good, wrong-secret, expired)", async () => { + await withTestCore(sql, {}, async (core) => { + const s = core.schema; + const userId = await v7(); + await sql.unsafe(`select ${s}.create_user($1, $2)`, [userId, "dave"]); + + const lookup = "abcdEFGH12345678"; // 16 chars, matches lookup_id check + await sql.unsafe(`select ${s}.create_api_key($1, $2, $3, $4)`, [ + userId, + lookup, + "hashed-secret", + "default", + ]); + + const valid = await sql.unsafe( + `select member_id from ${s}.validate_api_key($1, $2)`, + [lookup, "hashed-secret"], + ); + expect(valid.length).toBe(1); + expect(valid[0]?.member_id).toBe(userId); + + const wrong = await sql.unsafe( + `select member_id from ${s}.validate_api_key($1, $2)`, + [lookup, "nope"], + ); + expect(wrong.length).toBe(0); + + // expired key + const lookup2 = "ZYXW9876_-abcdef"; + await sql.unsafe( + `select ${s}.create_api_key($1, $2, $3, $4, $5::timestamptz)`, + [userId, lookup2, "h2", "expired", "2000-01-01T00:00:00Z"], + ); + const expired = await sql.unsafe( + `select member_id from ${s}.validate_api_key($1, $2)`, + [lookup2, "h2"], + ); + expect(expired.length).toBe(0); + }); + }); +}); + +describe("migration behavior", () => { + test("is idempotent: re-running changes no migration rows or version", async () => { + await withTestCore(sql, {}, async (core) => { + const before = await appliedMigrations(sql, core.schema); + await migrateCore(sql, { schema: core.schema }); + expect(await appliedMigrations(sql, core.schema)).toEqual(before); + expect(await getSchemaVersion(sql, core.schema)).toBe( + CORE_SCHEMA_VERSION, + ); + }); + }); + + test("rejects a downgrade (db version newer than app)", async () => { + await withTestCore(sql, {}, async (core) => { + await sql.unsafe(`update ${core.schema}.version set version = '99.0.0'`); + await expect(migrateCore(sql, { schema: core.schema })).rejects.toThrow( + /older than database version/, + ); + }); + }); + + test("rejects invalid schema names", async () => { + for (const schema of ["Bad-Schema", "1core", "core test", "core;drop"]) { + await expect(migrateCore(sql, { schema })).rejects.toThrow( + /Invalid core schema name/, + ); + } + }); + + test("concurrent migrateCore on one schema is serialized safely", async () => { + // The advisory lock serializes writers. A loser may exhaust its retry + // budget and throw "Unable to acquire lock" — expected, not corruption. + // What must hold: at least one succeeds and the schema stays valid. + const schema = randomCoreSchema(); + try { + const results = await Promise.allSettled([ + migrateCore(sql, { schema }), + migrateCore(sql, { schema }), + migrateCore(sql, { schema }), + ]); + + expect(results.some((r) => r.status === "fulfilled")).toBe(true); + for (const r of results) { + if (r.status === "rejected") { + expect(String((r.reason as Error)?.message ?? r.reason)).toContain( + "Unable to acquire lock", + ); + } + } + + expect(await getSchemaVersion(sql, schema)).toBe(CORE_SCHEMA_VERSION); + expect(await tableExists(sql, schema, "principal")).toBe(true); + } finally { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + } + }); +}); diff --git a/packages/database/core/migrate/migrate.ts b/packages/database/core/migrate/migrate.ts new file mode 100644 index 0000000..a9c3181 --- /dev/null +++ b/packages/database/core/migrate/migrate.ts @@ -0,0 +1,279 @@ +import { info, reportError, span } from "@pydantic/logfire-node"; +import { semver } from "bun"; +import type { Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + doesSchemaExist, + ensurePostgresVersion, + ensureRequiredExtensions, + executeSqlFile, + isValidSchemaName, + type Migration, + REQUIRED_EXTENSIONS, + runSchemaMigrations, + template, +} from "../../migrate/kit"; +import { CORE_SCHEMA_VERSION } from "../version"; +import idempotent000 from "./idempotent/000_update.sql" with { type: "text" }; +import idempotent001 from "./idempotent/001_principal_space.sql" with { + type: "text", +}; +import idempotent002 from "./idempotent/002_group_member.sql" with { + type: "text", +}; +import idempotent003 from "./idempotent/003_tree_access.sql" with { + type: "text", +}; +import idempotent004 from "./idempotent/004_space.sql" with { type: "text" }; +import idempotent005 from "./idempotent/005_principal.sql" with { + type: "text", +}; +import idempotent006 from "./idempotent/006_membership.sql" with { + type: "text", +}; +import idempotent007 from "./idempotent/007_grant.sql" with { type: "text" }; +import idempotent008 from "./idempotent/008_api_key.sql" with { type: "text" }; +import idempotent009 from "./idempotent/009_invitation.sql" with { + type: "text", +}; +import incremental001 from "./incremental/001_space.sql" with { type: "text" }; +import incremental002 from "./incremental/002_principal.sql" with { + type: "text", +}; +import incremental003 from "./incremental/003_principal_space.sql" with { + type: "text", +}; +import incremental004 from "./incremental/004_group_member.sql" with { + type: "text", +}; +import incremental005 from "./incremental/005_tree_access.sql" with { + type: "text", +}; +import incremental006 from "./incremental/006_api_key.sql" with { + type: "text", +}; +import incremental007 from "./incremental/007_space_invitation.sql" with { + type: "text", +}; +import provisionSql from "./provision.sql" with { type: "text" }; + +const DIR = "packages/database/core/migrate"; + +const incrementals: Migration[] = [ + { name: "001_space", file: "incremental/001_space.sql", sql: incremental001 }, + { + name: "002_principal", + file: "incremental/002_principal.sql", + sql: incremental002, + }, + { + name: "003_principal_space", + file: "incremental/003_principal_space.sql", + sql: incremental003, + }, + { + name: "004_group_member", + file: "incremental/004_group_member.sql", + sql: incremental004, + }, + { + name: "005_tree_access", + file: "incremental/005_tree_access.sql", + sql: incremental005, + }, + { + name: "006_api_key", + file: "incremental/006_api_key.sql", + sql: incremental006, + }, + { + name: "007_space_invitation", + file: "incremental/007_space_invitation.sql", + sql: incremental007, + }, +]; + +const idempotents: Migration[] = [ + { name: "000_update", file: "idempotent/000_update.sql", sql: idempotent000 }, + { + name: "001_principal_space", + file: "idempotent/001_principal_space.sql", + sql: idempotent001, + }, + { + name: "002_group_member", + file: "idempotent/002_group_member.sql", + sql: idempotent002, + }, + { + name: "003_tree_access", + file: "idempotent/003_tree_access.sql", + sql: idempotent003, + }, + { name: "004_space", file: "idempotent/004_space.sql", sql: idempotent004 }, + { + name: "005_principal", + file: "idempotent/005_principal.sql", + sql: idempotent005, + }, + { + name: "006_membership", + file: "idempotent/006_membership.sql", + sql: idempotent006, + }, + { name: "007_grant", file: "idempotent/007_grant.sql", sql: idempotent007 }, + { + name: "008_api_key", + file: "idempotent/008_api_key.sql", + sql: idempotent008, + }, + { + name: "009_invitation", + file: "idempotent/009_invitation.sql", + sql: idempotent009, + }, +]; + +/** + * The core control-plane schema name. Production always uses "core"; the name + * is a parameter so tests can provision throwaway, isolated cores (and so the + * SQL is templated symmetrically with the per-space migrations). Reference this + * constant rather than hardcoding "core" elsewhere. + */ +export const CORE_SCHEMA = "core"; + +export interface MigrateCoreOptions { + schema?: string; + logSqlFiles?: boolean; + statementTimeout?: string; + lockTimeout?: string; + transactionTimeout?: string; + idleInTransactionSessionTimeout?: string; +} + +interface NormalizedMigrateCoreOptions { + schema: string; + logSqlFiles: boolean; + schemaVersion: string; + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function migrateCore( + sql: SQL, + options: MigrateCoreOptions = {}, +): Promise { + const opts = normalizeMigrateCoreOptions(options); + const attributes = migrateAttributes(opts); + + await span("core.migrate", { + attributes, + callback: async () => { + try { + if (!isValidSchemaName(opts.schema)) { + throw new Error( + `Invalid core schema name: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, + ); + } + if (!semver.satisfies(opts.schemaVersion, "*")) { + throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); + } + const [key1, key2] = advisoryLockKey( + `memory-core:schema:${opts.schema}`, + ); + + await sql.begin(async (tx) => { + await applySessionTimeouts(tx, opts); + const acquired = await span("core.migrate.acquire_lock", { + attributes, + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error("Unable to acquire lock for core migrations."); + } + + await ensurePostgresVersion(tx); + await ensureRequiredExtensions(tx, "core.migrate"); + + if (!(await doesSchemaExist(tx, opts.schema))) { + await span("core.migrate.provision", { + attributes: { + ...attributes, + "core.migration_file": "provision.sql", + "core.migration_type": "provision", + }, + callback: () => + executeSqlFile( + tx, + template(provisionSql, { schema: opts.schema }), + { + logSqlFiles: opts.logSqlFiles, + label: "core", + schema: opts.schema, + type: "provision", + dir: DIR, + file: "provision.sql", + }, + ), + }); + info("Core schema provisioned", attributes); + } + await span("core.migrate.run", { + attributes, + callback: () => + runSchemaMigrations(tx, { + schema: opts.schema, + schemaVersion: opts.schemaVersion, + incrementals, + idempotents, + templateVars: { schema: opts.schema }, + label: "core", + dir: DIR, + logSqlFiles: opts.logSqlFiles, + }), + }); + }); + info("Core migrations completed", attributes); + } catch (error) { + reportError("Core migration failed", error as Error, attributes); + throw error; + } + }, + }); +} + +function migrateAttributes( + options: NormalizedMigrateCoreOptions, +): Record { + return { + "db.schema": options.schema, + "core.schema_version": options.schemaVersion, + "core.required_extensions": REQUIRED_EXTENSIONS.map( + (extension) => `${extension.name}@>=${extension.minVersion}`, + ), + "db.statement_timeout": options.statementTimeout, + "db.lock_timeout": options.lockTimeout, + "db.transaction_timeout": options.transactionTimeout, + "db.idle_in_transaction_session_timeout": + options.idleInTransactionSessionTimeout, + }; +} + +function normalizeMigrateCoreOptions( + options: MigrateCoreOptions, +): NormalizedMigrateCoreOptions { + return { + schema: options.schema ?? CORE_SCHEMA, + logSqlFiles: options.logSqlFiles ?? false, + schemaVersion: CORE_SCHEMA_VERSION, + statementTimeout: options.statementTimeout ?? "20s", + lockTimeout: options.lockTimeout ?? "5s", + transactionTimeout: options.transactionTimeout ?? "1min", + idleInTransactionSessionTimeout: + options.idleInTransactionSessionTimeout ?? "5s", + }; +} diff --git a/packages/database/core/migrate/provision.sql b/packages/database/core/migrate/provision.sql new file mode 100644 index 0000000..e98b9d9 --- /dev/null +++ b/packages/database/core/migrate/provision.sql @@ -0,0 +1,15 @@ +create schema {{schema}}; + +create table {{schema}}.version +( version text not null +, at timestamptz not null default now() +); + +create unique index version_singleton_idx on {{schema}}.version ((true)); -- only ONE row allowed +insert into {{schema}}.version (version) values ('0.0.0'); + +create table {{schema}}.migration +( name text not null constraint migration_pkey primary key +, applied_at_version text not null +, applied_at timestamptz not null default pg_catalog.clock_timestamp() +); diff --git a/packages/database/core/migrate/sql.d.ts b/packages/database/core/migrate/sql.d.ts new file mode 100644 index 0000000..0e51813 --- /dev/null +++ b/packages/database/core/migrate/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const sql: string; + export default sql; +} diff --git a/packages/database/core/migrate/test-utils.ts b/packages/database/core/migrate/test-utils.ts new file mode 100644 index 0000000..e645f5f --- /dev/null +++ b/packages/database/core/migrate/test-utils.ts @@ -0,0 +1,64 @@ +import type { Sql as SQL } from "postgres"; +import { type MigrateCoreOptions, migrateCore } from "./migrate"; + +// Connection, failure assertions, and schema introspection are shared with the +// space suite. +export * from "../../migrate/test-utils"; + +const SCHEMA_SUFFIX_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; + +/** A unique, valid core schema name, e.g. "core_test_a1b2c3d4". */ +export function randomCoreSchema(): string { + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let suffix = ""; + for (const b of bytes) suffix += SCHEMA_SUFFIX_ALPHABET[b % 36]; + return `core_test_${suffix}`; +} + +// --------------------------------------------------------------------------- +// TestCore — a provisioned, isolated core schema +// --------------------------------------------------------------------------- +// +// The core migrations are templated (production uses the "core" schema; tests +// pass a unique throwaway name), so every test gets its own isolated core and +// they run concurrently without ever touching a real `core` schema. + +export class TestCore { + readonly schema: string; + private readonly sql: SQL; + + private constructor(sql: SQL, schema: string) { + this.sql = sql; + this.schema = schema; + } + + static async create( + sql: SQL, + options: Omit & { schema?: string } = {}, + ): Promise { + const schema = options.schema ?? randomCoreSchema(); + await migrateCore(sql, { ...options, schema }); + return new TestCore(sql, schema); + } + + async drop(): Promise { + await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); + } +} + +/** + * Provision a fresh core, run `fn` against it, and always drop it afterward. + * Safe to call from concurrent tests — each gets its own unique schema. + */ +export async function withTestCore( + sql: SQL, + options: Omit & { schema?: string }, + fn: (core: TestCore) => Promise, +): Promise { + const core = await TestCore.create(sql, options); + try { + return await fn(core); + } finally { + await core.drop(); + } +} diff --git a/packages/database/core/version.ts b/packages/database/core/version.ts new file mode 100644 index 0000000..9fd61d7 --- /dev/null +++ b/packages/database/core/version.ts @@ -0,0 +1 @@ +export const CORE_SCHEMA_VERSION = "0.0.1"; diff --git a/packages/database/index.ts b/packages/database/index.ts new file mode 100644 index 0000000..825a189 --- /dev/null +++ b/packages/database/index.ts @@ -0,0 +1,9 @@ +// Control plane (`core` schema) and data plane (per-space `me_` schemas) +// live in one package; they are co-located in a single database/deployment. +// Kept as separate `core/` and `space/` modules so the boundary stays clean +// (space must not import core) and the split is easy to undo if spaces are ever +// distributed across databases again. The `auth` schema (better-auth-shaped +// users/sessions/accounts) is its own module for the same reason. +export * from "./auth"; +export * from "./core"; +export * from "./space"; diff --git a/packages/database/migrate/kit.ts b/packages/database/migrate/kit.ts new file mode 100644 index 0000000..138b6d7 --- /dev/null +++ b/packages/database/migrate/kit.ts @@ -0,0 +1,430 @@ +import { createHash } from "node:crypto"; +import { info, span } from "@pydantic/logfire-node"; +import { semver } from "bun"; +import type { ISql } from "postgres"; + +// --------------------------------------------------------------------------- +// Shared migration machinery for the core (control plane) and space (data +// plane) migrators. migrateCore / migrateSpace / bootstrapSpaceDatabase are +// thin orchestrators over these helpers. `label` (e.g. "core" / "space") drives +// span names, telemetry attribute keys, and log messages so each migrator keeps +// its existing observability; `dir` is the on-disk path used in SQL-file logs. +// --------------------------------------------------------------------------- + +export interface Migration { + name: string; + file: string; + sql: string; +} + +export const REQUIRED_EXTENSIONS = [ + { name: "citext", minVersion: "1.6" }, + { name: "ltree", minVersion: "1.3" }, + { name: "vector", minVersion: "0.8.2" }, + { name: "pg_textsearch", minVersion: "1.1.0" }, +] as const; + +/** A valid lowercase SQL identifier usable as a schema name (<= 63 chars). */ +export function isValidSchemaName(schema: string): boolean { + return /^[a-z_][a-z0-9_]*$/.test(schema) && schema.length <= 63; +} + +// --------------------------------------------------------------------------- +// Advisory locking +// --------------------------------------------------------------------------- + +const MAX_LOCK_RETRIES = 5; +const BASE_DELAY_MS = 100; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Derive a stable (int4, int4) advisory-lock key pair from a name. */ +export function advisoryLockKey(name: string): [number, number] { + const digest = createHash("sha256").update(name).digest(); + return [digest.readInt32BE(0), digest.readInt32BE(4)]; +} + +/** Try to take a transaction-scoped advisory lock, with bounded backoff. */ +export async function acquireAdvisoryLock( + tx: ISql, + key1: number, + key2: number, +): Promise { + for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { + const [result] = await tx` + select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired + `; + if (result?.acquired) return true; + if (attempt < MAX_LOCK_RETRIES - 1) { + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } + return false; +} + +// --------------------------------------------------------------------------- +// Session / precondition helpers +// --------------------------------------------------------------------------- + +export interface SessionTimeouts { + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function applySessionTimeouts( + tx: ISql, + t: SessionTimeouts, +): Promise { + await tx`select set_config('statement_timeout', ${t.statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${t.lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${t.transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${t.idleInTransactionSessionTimeout}, true)`; +} + +export async function ensurePostgresVersion(tx: ISql): Promise { + const [row] = await tx` + select current_setting('server_version_num')::int as server_version_num + `; + const serverVersionNum = Number(row?.server_version_num); + if (serverVersionNum < 180000) { + throw new Error( + `PostgreSQL version 18 or higher is required (found ${serverVersionNum})`, + ); + } +} + +export async function ensureExtension( + tx: ISql, + name: string, + minVersion: string, +): Promise { + // Extensions are database-global, but each migrator (auth, core, the + // space bootstrap) serializes only against its own advisory key — two + // DIFFERENT migrators racing on a fresh database both pass the existence + // check below and the loser's `create extension` dies with a + // unique_violation (seen with parallel integration suites on a fresh CI + // container). One database-wide lock serializes every extension ensure; + // transaction-scoped, so a loser proceeds only after the winner's commit + // made the extension visible. Re-acquiring within the same transaction + // (one lock per extension in a migrator's loop) is immediate. + const [key1, key2] = advisoryLockKey("memory:extensions"); + await tx`select pg_advisory_xact_lock(${key1}, ${key2})`; + + const [installed] = await tx` + select x.extversion, n.nspname + from pg_extension x + inner join pg_namespace n on (x.extnamespace = n.oid) + where x.extname = ${name} + `; + + if (installed) { + if ( + installed.nspname === "public" && + semver.order(installed.extversion, minVersion) >= 0 + ) { + return; + } + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required in the "public" schema (found ${installed.extversion} installed in "${installed.nspname}")`, + ); + } + + const [available] = await tx` + select default_version + from pg_available_extensions + where name = ${name} + `; + + if (!available || semver.order(available.default_version, minVersion) < 0) { + const found = available + ? `found ${available.default_version} available` + : "not available"; + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required (${found})`, + ); + } + + await tx`create extension if not exists ${tx(name)} with schema public`; +} + +/** Ensure every REQUIRED_EXTENSIONS entry, each wrapped in a span. */ +export async function ensureRequiredExtensions( + tx: ISql, + spanPrefix: string, +): Promise { + for (const extension of REQUIRED_EXTENSIONS) { + await span(`${spanPrefix}.ensure_extension`, { + attributes: { + "db.extension": extension.name, + "db.extension_min_version": extension.minVersion, + }, + callback: () => ensureExtension(tx, extension.name, extension.minVersion), + }); + } +} + +export async function doesSchemaExist( + tx: ISql, + schema: string, +): Promise { + const [row] = await tx` + select exists ( + select 1 from pg_namespace n where n.nspname = ${schema} + ) as present + `; + return Boolean(row?.present); +} + +export async function assertSchemaOwnership( + tx: ISql, + schema: string, +): Promise { + const [result] = await tx` + select + n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner + from pg_catalog.pg_namespace n + where n.nspname = ${schema} + `; + + if (!result?.is_owner) { + throw new Error( + `Only the owner of the ${schema} schema can run database migrations`, + ); + } +} + +// --------------------------------------------------------------------------- +// Templating +// --------------------------------------------------------------------------- + +/** Substitute `{{name}}` placeholders; throws on an unknown placeholder. */ +export function template(sql: string, vars: Record): string { + return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { + if (!(key in vars)) { + throw new Error(`Missing template variable: ${key}`); + } + return String(vars[key]); + }); +} + +// --------------------------------------------------------------------------- +// SQL-file execution (with error-location logging) +// --------------------------------------------------------------------------- + +export interface SqlFileContext { + logSqlFiles: boolean; + label: string; + schema: string; + type: string; // "provision" | "incremental" | "idempotent" + dir: string; // e.g. "packages/database/core/migrate" + file: string; // e.g. "incremental/001_space.sql" +} + +export async function executeSqlFile( + tx: ISql, + sqlText: string, + ctx: SqlFileContext, +): Promise { + if (ctx.logSqlFiles) { + console.error( + `[migrate:db] ${ctx.label} ${ctx.schema} ${ctx.type} ${ctx.dir}/${ctx.file}`, + ); + } + try { + await tx.unsafe(sqlText); + } catch (error) { + if (ctx.logSqlFiles) { + console.error( + `[migrate:db] failed ${ctx.label} ${ctx.schema} ${ctx.type} ${ctx.dir}/${ctx.file}`, + ); + logPostgresSqlLocation(sqlText, error); + } + throw error; + } +} + +function logPostgresSqlLocation(sqlText: string, error: unknown): void { + // postgres-js sets `position` (1-based) on server errors; non-PG errors won't. + const position = Number((error as { position?: unknown })?.position); + if (!Number.isSafeInteger(position) || position < 1) return; + + const location = sqlLocation(sqlText, position); + if (!location) return; + console.error( + `[migrate:db] sql position ${position} -> line ${location.line}, column ${location.column}`, + ); + console.error(sqlContext(sqlText, location.line, location.column)); +} + +function sqlLocation( + sqlText: string, + position: number, +): { line: number; column: number } | undefined { + if (position > sqlText.length + 1) return undefined; + let line = 1; + let column = 1; + for (let i = 0; i < position - 1; i++) { + if (sqlText.charCodeAt(i) === 10) { + line++; + column = 1; + } else { + column++; + } + } + return { line, column }; +} + +function sqlContext(sqlText: string, line: number, column: number): string { + const lines = sqlText.split("\n"); + const start = Math.max(1, line - 2); + const end = Math.min(lines.length, line + 2); + const width = String(end).length; + const output = ["[migrate:db] sql context:"]; + + for (let n = start; n <= end; n++) { + const marker = n === line ? ">" : " "; + output.push(`${marker} ${String(n).padStart(width)} | ${lines[n - 1]}`); + if (n === line) { + output.push(` ${" ".repeat(width)} | ${" ".repeat(column - 1)}^`); + } + } + + return output.join("\n"); +} + +// --------------------------------------------------------------------------- +// The incremental-once / idempotent-always runner +// --------------------------------------------------------------------------- + +export interface RunMigrationsConfig { + schema: string; + schemaVersion: string; + incrementals: Migration[]; + idempotents: Migration[]; + /** Template vars applied to every migration's SQL (always includes `schema`). */ + templateVars: Record; + label: string; // "core" | "space" + dir: string; + logSqlFiles: boolean; +} + +function migrationAttributes( + label: string, + schema: string, + schemaVersion: string, + migration: Migration, + type: string, +): Record { + return { + "db.schema": schema, + [`${label}.migration`]: migration.name, + [`${label}.migration_file`]: migration.file, + [`${label}.migration_type`]: type, + [`${label}.schema_version`]: schemaVersion, + }; +} + +/** + * Assumes the schema's version + migration tracking tables exist (created by + * provision). Checks ownership, rejects downgrades, applies pending incremental + * migrations once (tracked), re-applies all idempotent migrations, and stamps + * the version. + */ +export async function runSchemaMigrations( + tx: ISql, + cfg: RunMigrationsConfig, +): Promise { + const { schema, schemaVersion, label, dir, logSqlFiles, templateVars } = cfg; + const Label = label.charAt(0).toUpperCase() + label.slice(1); + + await assertSchemaOwnership(tx, schema); + + const [versionRow] = await tx`select version from ${tx(schema)}.version`; + const dbVersion: string = versionRow?.version; + const cmp = semver.order(schemaVersion, dbVersion); + // abort if the application is older than the database + if (cmp < 0) { + throw new Error( + `Application version (${schemaVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the application.", + ); + } + /* run migrations regardless + if (cmp === 0) { + // version matches; no need to run migrations + info(`${Label} migration skipped, version current`, { + "db.schema": schema, + [`${label}.version`]: dbVersion, + [`${label}.schema_version`]: schemaVersion, + }); + return; + } + */ + + const incrementals = [...cfg.incrementals].sort((a, b) => + a.name.localeCompare(b.name), + ); + for (const migration of incrementals) { + const [existingRow] = await tx` + select exists ( + select 1 from ${tx(schema)}.migration where name = ${migration.name} + ) as existing + `; + if (existingRow?.existing) continue; + + const attributes = migrationAttributes( + label, + schema, + schemaVersion, + migration, + "incremental", + ); + await span(`${label}.migrate.incremental`, { + attributes, + callback: async () => { + await executeSqlFile(tx, template(migration.sql, templateVars), { + logSqlFiles, + label, + schema, + type: "incremental", + dir, + file: migration.file, + }); + await tx` + insert into ${tx(schema)}.migration (name, applied_at_version) + values (${migration.name}, ${schemaVersion})`; + }, + }); + info(`${Label} migration applied`, attributes); + } + + const idempotents = [...cfg.idempotents].sort((a, b) => + a.name.localeCompare(b.name), + ); + for (const migration of idempotents) { + await span(`${label}.migrate.idempotent`, { + attributes: migrationAttributes( + label, + schema, + schemaVersion, + migration, + "idempotent", + ), + callback: () => + executeSqlFile(tx, template(migration.sql, templateVars), { + logSqlFiles, + label, + schema, + type: "idempotent", + dir, + file: migration.file, + }), + }); + } + + await tx`update ${tx(schema)}.version set version = ${schemaVersion}, at = now()`; +} diff --git a/packages/database/migrate/test-utils.ts b/packages/database/migrate/test-utils.ts new file mode 100644 index 0000000..68dcbb5 --- /dev/null +++ b/packages/database/migrate/test-utils.ts @@ -0,0 +1,198 @@ +import type { Sql as SQL } from "postgres"; +import postgres from "postgres"; + +// --------------------------------------------------------------------------- +// Shared test helpers for the database package's migration suites +// --------------------------------------------------------------------------- +// +// Generic, schema-agnostic helpers used by both the core and space test-utils +// (which re-export this module and add their own provisioning). Tests run +// against a real PostgreSQL 18 with citext/ltree/vector/pg_textsearch; point +// `TEST_DATABASE_URL` at a ghost database (see CLAUDE.md → Database integration +// tests). Falls back to a local Postgres when unset. + +const DEFAULT_TEST_DATABASE_URL = + "postgresql://postgres@127.0.0.1:5432/postgres"; + +export function resolveTestDatabaseUrl(): string { + return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; +} + +export function connect(max = 10): SQL { + // onnotice silences the routine "… already exists, skipping" NOTICEs that the + // idempotent migrations emit (postgres-js prints them to the console by default). + return postgres(resolveTestDatabaseUrl(), { max, onnotice: () => {} }); +} + +/** + * Assert that a query rejects. bun:test's `expect(...).rejects` does not drive + * postgres-js's lazy `PendingQuery` (it only runs when truly awaited), so it + * hangs; awaiting inside try/catch executes the query and observes the error. + */ +export async function expectReject(fn: () => Promise): Promise { + let rejected = false; + try { + await fn(); + } catch { + rejected = true; + } + if (!rejected) { + throw new Error("expected the operation to reject, but it resolved"); + } +} + +// --------------------------------------------------------------------------- +// Schema introspection +// --------------------------------------------------------------------------- + +export async function schemaExists(sql: SQL, name: string): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${name} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function tableExists( + sql: SQL, + schema: string, + table: string, +): Promise { + const [row] = await sql` + select exists ( + select 1 from information_schema.tables + where table_schema = ${schema} and table_name = ${table} + ) as exists + `; + return Boolean(row?.exists); +} + +export async function listTables(sql: SQL, schema: string): Promise { + const rows = await sql` + select table_name + from information_schema.tables + where table_schema = ${schema} and table_type = 'BASE TABLE' + order by table_name + `; + return rows.map((r) => r.table_name as string); +} + +/** Distinct function names in a schema (overloads collapse to one entry). */ +export async function listFunctions( + sql: SQL, + schema: string, +): Promise { + const rows = await sql` + select distinct p.proname + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = ${schema} + order by p.proname + `; + return rows.map((r) => r.proname as string); +} + +export async function listTriggers( + sql: SQL, + schema: string, + table: string, +): Promise { + const rows = await sql` + select t.tgname + from pg_trigger t + join pg_class c on c.oid = t.tgrelid + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = ${schema} and c.relname = ${table} and not t.tgisinternal + order by t.tgname + `; + return rows.map((r) => r.tgname as string); +} + +/** Whether an extension is installed (in any schema). */ +export async function extensionInstalled( + sql: SQL, + name: string, +): Promise { + const [row] = await sql` + select exists ( + select 1 from pg_extension where extname = ${name} + ) as exists + `; + return Boolean(row?.exists); +} + +/** Fully-resolved type of a column, e.g. "halfvec(768)" or "uuid". */ +export async function columnType( + sql: SQL, + schema: string, + table: string, + column: string, +): Promise { + const [row] = await sql` + select format_type(a.atttypid, a.atttypmod) as type + from pg_attribute a + where a.attrelid = ${`${schema}.${table}`}::regclass + and a.attname = ${column} + and not a.attisdropped + `; + return row?.type ?? null; +} + +export async function listIndexes( + sql: SQL, + schema: string, + table: string, +): Promise { + const rows = await sql` + select indexname from pg_indexes + where schemaname = ${schema} and tablename = ${table} + order by indexname + `; + return rows.map((r) => r.indexname as string); +} + +export async function getIndexDef( + sql: SQL, + schema: string, + index: string, +): Promise { + const [row] = await sql` + select indexdef from pg_indexes + where schemaname = ${schema} and indexname = ${index} + `; + return row?.indexdef ?? null; +} + +/** Storage parameters of an index, e.g. ["m=8", "ef_construction=32"]. */ +export async function getIndexReloptions( + sql: SQL, + schema: string, + index: string, +): Promise { + const [row] = await sql` + select c.reloptions + from pg_class c + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = ${schema} and c.relname = ${index} + `; + return row?.reloptions ?? []; +} + +export async function appliedMigrations( + sql: SQL, + schema: string, +): Promise { + const rows = await sql.unsafe( + `select name from ${schema}.migration order by name`, + ); + return rows.map((r) => r.name as string); +} + +export async function getSchemaVersion( + sql: SQL, + schema: string, +): Promise { + const [row] = await sql.unsafe(`select version from ${schema}.version`); + return row?.version; +} diff --git a/packages/database/package.json b/packages/database/package.json new file mode 100644 index 0000000..97a304b --- /dev/null +++ b/packages/database/package.json @@ -0,0 +1,11 @@ +{ + "name": "@memory.build/database", + "version": "0.2.5", + "private": true, + "type": "module", + "dependencies": { + "@memory.build/protocol": "workspace:*", + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9" + } +} diff --git a/packages/database/space/index.ts b/packages/database/space/index.ts new file mode 100644 index 0000000..1181e10 --- /dev/null +++ b/packages/database/space/index.ts @@ -0,0 +1,26 @@ +export { bootstrapSpaceDatabase } from "./migrate/bootstrap"; +export { + type MigrateSpaceOptions, + migrateSpace, + provisionSpace, +} from "./migrate/migrate"; +export { + classifyTreeFilter, + denormalizeTreePath, + HOME_NAMESPACE, + homePrefix, + normalizeTreeFilter, + normalizeTreePath, + SHARE_NAMESPACE, + type TreeFilter, + TreePathError, + type TreePathOptions, +} from "./path"; +export { + generateSlug, + isValidSlug, + isValidSpaceSchema, + schemaToSlug, + slugToSchema, +} from "./slug"; +export { SPACE_SCHEMA_VERSION } from "./version"; diff --git a/packages/database/space/migrate/bootstrap.ts b/packages/database/space/migrate/bootstrap.ts new file mode 100644 index 0000000..b28eec1 --- /dev/null +++ b/packages/database/space/migrate/bootstrap.ts @@ -0,0 +1,63 @@ +import { info, reportError, span } from "@pydantic/logfire-node"; +import type { Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + ensurePostgresVersion, + ensureRequiredExtensions, + REQUIRED_EXTENSIONS, +} from "../../migrate/kit"; + +/** + * Prepare a physical database to host space schemas. + * + * This does not create or migrate an individual space. Spaces are still created + * on demand by migrateSpace(), which provisions a specific me_ schema. + */ +export async function bootstrapSpaceDatabase( + sql: SQL, + statementTimeout: string = "20s", + lockTimeout: string = "5s", + transactionTimeout: string = "30s", + idleInTransactionSessionTimeout: string = "30s", +): Promise { + const attributes = { + "db.statement_timeout": statementTimeout, + "db.lock_timeout": lockTimeout, + "db.transaction_timeout": transactionTimeout, + "db.idle_in_transaction_session_timeout": idleInTransactionSessionTimeout, + "space.required_extensions": REQUIRED_EXTENSIONS.map( + (extension) => `${extension.name}@>=${extension.minVersion}`, + ), + }; + + await span("space.bootstrap", { + attributes, + callback: async () => { + try { + const [key1, key2] = advisoryLockKey("memory-space:bootstrap"); + await sql.begin(async (tx) => { + await ensurePostgresVersion(tx); + const acquired = await span("space.bootstrap.acquire_lock", { + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error("Failed to acquire advisory lock"); + } + await applySessionTimeouts(tx, { + statementTimeout, + lockTimeout, + transactionTimeout, + idleInTransactionSessionTimeout, + }); + await ensureRequiredExtensions(tx, "space.bootstrap"); + }); + info("Space bootstrap completed", attributes); + } catch (error) { + reportError("Space bootstrap failed", error as Error, attributes); + throw error; + } + }, + }); +} diff --git a/packages/database/space/migrate/idempotent/001_memory.sql b/packages/database/space/migrate/idempotent/001_memory.sql new file mode 100644 index 0000000..7ebc54a --- /dev/null +++ b/packages/database/space/migrate/idempotent/001_memory.sql @@ -0,0 +1,723 @@ +------------------------------------------------------------------------------- +-- memory triggers +------------------------------------------------------------------------------- +create or replace function {{schema}}.memory_before_update() +returns trigger +as $func$ +begin + -- always update the timestamp + new.updated_at = pg_catalog.now(); + + -- content changed -> new embedding needs to be generated + if old.content is distinct from new.content + and old.embedding is not distinct from new.embedding + then + new.embedding = null; + new.embedding_version = old.embedding_version operator(pg_catalog.+) 1; + end if; + + return new; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp -- public required for pgvector's `is not distinct from` +; + +create or replace trigger memory_before_update_trg +before update on {{schema}}.memory +for each row +execute function {{schema}}.memory_before_update(); + +------------------------------------------------------------------------------- +-- tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.tree_access(_tree_access jsonb) +returns table +( tree_path ltree +, access int +) +as $func$ + select + x.tree_path + , x.access + from jsonb_to_recordset(_tree_access) x(tree_path ltree, access int) +$func$ language sql immutable strict security invoker +; + +------------------------------------------------------------------------------- +-- has_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.has_tree_access +( _tree_access jsonb +, _tree_path ltree +, _access int +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.tree_access(_tree_access) x + where x.tree_path @> _tree_path + and x.access >= _access + ) +$func$ language sql immutable strict security invoker +; + +------------------------------------------------------------------------------- +-- get memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_memory +( _tree_access jsonb +, _id uuid default null +) +returns table +( id uuid +, tree ltree +, meta jsonb +, temporal tstzrange +, content text +, created_at timestamptz +, updated_at timestamptz +, has_embedding bool +) +as $func$ + select + m.id + , m.tree + , m.meta + , m.temporal + , m.content + , m.created_at + , m.updated_at + , m.embedding is not null + from {{schema}}.memory m + where m.id = _id + and {{schema}}.has_tree_access(_tree_access, m.tree, 1) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- batch create memory +-- +-- The canonical memory insert: one set-based statement for a whole batch +-- (create_memory below is a one-row wrapper). Parallel arrays, aligned by +-- position, carry the rows. Per-row, on a duplicate explicit id the outcome +-- depends on _replace_if_meta_differs: +-- - null (default): skip — the existing row is left untouched. +-- - a meta key name: the existing row is REPLACED (tree/meta/temporal/ +-- content) when its meta->>key value differs from the new record's, and +-- skipped when it matches. Deterministic-id importers use this to push +-- re-renders by bumping a version value in meta (importer_version). +-- The replace arm additionally requires write access on the EXISTING +-- row's tree; without it the row is silently skipped (not raised, unlike +-- patch_memory) so one inaccessible row can't fail a whole batch. +-- +-- Returns one row (id, inserted) per insert/replace — inserted distinguishes +-- a fresh insert (true, xmax = 0) from a replace (false); skipped rows are +-- absent. The target-tree access check is all-or-nothing up front (one bad +-- row raises before anything is written), and an explicit id repeated WITHIN +-- the batch collapses to its first occurrence (a single INSERT cannot touch +-- the same row twice); later occurrences are skipped. +-- +-- Embedding columns are never set here: the update triggers invalidate and +-- re-enqueue the embedding only when content actually changed, so a +-- meta-only replace does not re-embed. +------------------------------------------------------------------------------- +create or replace function {{schema}}.batch_create_memory +( _tree_access jsonb +, _ids uuid[] -- null elements get a generated uuidv7 +, _trees ltree[] +, _contents text[] +, _metas jsonb -- json ARRAY of meta objects; null elements default to '{}' +, _temporals tstzrange[] +, _replace_if_meta_differs text default null +) +returns table (id uuid, inserted boolean) +as $func$ +-- The out columns (id, inserted) shadow table columns inside the body; the +-- body never reads them as variables, so resolve ambiguity to the columns. +#variable_conflict use_column +begin + -- _metas is one jsonb array (not jsonb[]): drivers pass json values + -- reliably (sql.json), where a jsonb[] parameter invites double-encoded + -- string scalars. Elements align with the arrays by position. + if jsonb_typeof(_metas) is distinct from 'array' + or cardinality(_ids) is distinct from cardinality(_trees) + or cardinality(_ids) is distinct from cardinality(_contents) + or cardinality(_ids) is distinct from jsonb_array_length(_metas) + or cardinality(_ids) is distinct from cardinality(_temporals) + then + raise exception 'batch arrays must have equal lengths' + using errcode = 'invalid_parameter_value'; + end if; + + if exists + ( + select 1 + from unnest(_trees) t(tree) + where not {{schema}}.has_tree_access(_tree_access, t.tree, 2) + ) then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + return query + with r as + ( + select + coalesce(u.id, uuidv7()) as id + , u.tree + , coalesce(nullif(e.meta, 'null'::jsonb), '{}'::jsonb) as meta + , u.temporal + , u.content + , u.ord + from unnest(_ids, _trees, _contents, _temporals) + with ordinality u(id, tree, content, temporal, ord) + join jsonb_array_elements(_metas) with ordinality e(meta, ord) + on e.ord = u.ord + ) + , d as + ( + -- First occurrence wins when a batch repeats an explicit id. + select distinct on (r.id) r.* + from r + order by r.id, r.ord + ) + insert into {{schema}}.memory as m + ( id + , tree + , meta + , temporal + , content + ) + select d.id, d.tree, d.meta, d.temporal, d.content + from d + on conflict (id) do update set + tree = excluded.tree + , meta = excluded.meta + , temporal = excluded.temporal + , content = excluded.content + where _replace_if_meta_differs is not null + and m.meta->>_replace_if_meta_differs + is distinct from excluded.meta->>_replace_if_meta_differs + and {{schema}}.has_tree_access(_tree_access, m.tree, 2) + returning m.id, (m.xmax = 0) + ; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- create memory +-- +-- One-row wrapper over batch_create_memory — see there for the conflict +-- semantics (insert / replace-if-meta-differs / skip) and the return shape. +-- +-- The drop covers the pre-upsert 6-arg signature — without it, create would +-- add an ambiguous overload (and the return type changed). No-op on re-runs. +------------------------------------------------------------------------------- +drop function if exists {{schema}}.create_memory(jsonb, ltree, text, uuid, jsonb, tstzrange); +create or replace function {{schema}}.create_memory +( _tree_access jsonb +, _tree ltree +, _content text +, _id uuid default null +, _meta jsonb default '{}' +, _temporal tstzrange default null +, _replace_if_meta_differs text default null +) +returns table (id uuid, inserted boolean) +as $func$ + select b.id, b.inserted + from {{schema}}.batch_create_memory( + _tree_access, + array[_id]::uuid[], + array[_tree], + array[_content], + jsonb_build_array(coalesce(_meta, '{}'::jsonb)), + array[_temporal], + _replace_if_meta_differs + ) b; +$func$ language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- patch memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.patch_memory +( _tree_access jsonb +, _id uuid +, _patch jsonb +) +returns bool +as $func$ +declare + _src ltree; + _dst ltree; + _ok bool; +begin + -- at least one valid field must be present + select count(*) filter (where k in ('meta', 'tree', 'temporal', 'content')) > 0 + into strict _ok + from jsonb_each(_patch) o(k, v) + ; + + if not _ok then + raise exception 'no valid patch fields found' + using errcode = 'invalid_parameter_value'; + end if; + + _dst = (_patch->>'tree')::ltree; + + -- cannot set tree to null + if _patch ? 'tree' and _dst is null then + raise exception 'tree cannot be set to null' + using errcode = 'invalid_parameter_value'; + end if; + + -- find the existing memory and get it's tree + select m.tree into _src + from {{schema}}.memory m + where m.id = _id + for update -- don't let anyone "move" the memory while we're working on it + ; + + if not found then + return false; + end if; + + with a as materialized + ( + select a.tree_path, a.access + from {{schema}}.tree_access(_tree_access) a + ) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 2 + ) + and + ( + _dst is null + or _src @> _dst + or exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + ) + into strict _ok + ; + + if not _ok then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + update {{schema}}.memory m set + tree = case when _patch ? 'tree' then (_patch->>'tree')::ltree else m.tree end + , meta = case when _patch ? 'meta' then _patch->'meta' else m.meta end + , temporal = case when _patch ? 'temporal' then (_patch->>'temporal')::tstzrange else m.temporal end + , content = case when _patch ? 'content' then _patch->>'content' else m.content end + where id = _id + returning id into _id + ; + + return _id is not null; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- move tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.move_tree +( _tree_access jsonb +, _src ltree +, _dst ltree +, _dry_run bool default false +) +returns bigint +as $func$ +declare + _has_src bool; + _has_dst bool; + _moved bigint; +begin + -- must have read/write on _src + -- must have read/write on _dst + with a as materialized + ( + select a.tree_path, a.access + from {{schema}}.tree_access(_tree_access) a + ) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 2 + ) + , exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + into strict _has_src, _has_dst + ; + + if not _has_src then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if not _has_dst then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + with x as + ( + select m.id + from {{schema}}.memory m + where _src @> m.tree + ) + , u as + ( + update {{schema}}.memory m + set tree = + case + when nlevel(m.tree) = nlevel(_src) then _dst + else _dst || subpath(m.tree, nlevel(_src), nlevel(m.tree) - nlevel(_src)) + end + from x + where m.id = x.id + and not _dry_run + ) + select count(*) into strict _moved + from x + ; + return _moved; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- copy tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.copy_tree +( _tree_access jsonb +, _src ltree +, _dst ltree +, _dry_run bool default false +) +returns bigint +as $func$ +declare + _has_src bool; + _has_dst bool; + _copied bigint; +begin + -- must have read on _src + -- must have read/write on _dst + with a as materialized + ( + select a.tree_path, a.access + from {{schema}}.tree_access(_tree_access) a + ) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 1 + ) + , exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + into strict _has_src, _has_dst + ; + + if not _has_src then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if not _has_dst then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + with m as + ( + select m.* + from {{schema}}.memory m + where _src @> m.tree + ) + , i as + ( + insert into {{schema}}.memory + ( meta + , tree + , temporal + , content + , embedding + , embedding_version + ) + select + m.meta + , case + when nlevel(m.tree) = nlevel(_src) then _dst + else _dst || subpath(m.tree, nlevel(_src), nlevel(m.tree) - nlevel(_src)) + end as dst + , m.temporal + , m.content + , m.embedding + , m.embedding_version + from m + where not _dry_run + ) + select count(*) into strict _copied + from m + ; + + return _copied; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_memory +( _tree_access jsonb +, _id uuid +) +returns bool +as $func$ +declare + _tree ltree; +begin + select m.tree into _tree + from {{schema}}.memory m + where m.id = _id + for update + ; + + if not found then + return false; + end if; + + if not {{schema}}.has_tree_access(_tree_access, _tree, 2) then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + delete from {{schema}}.memory + where id = _id + ; + return found; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_tree +( _tree_access jsonb +, _tree ltree +, _dry_run bool default false +) +returns bigint +as $func$ +declare + _has_access bool; + _deleted bigint; +begin + -- must have read/write on _tree + select exists + ( + select 1 + from {{schema}}.tree_access(_tree_access) a + where a.tree_path @> _tree + and a.access >= 2 + ) + into strict _has_access + ; + + if not _has_access then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if _dry_run then + select count(*) into strict _deleted + from {{schema}}.memory m + where _tree @> m.tree + ; + else + with d as + ( + delete from {{schema}}.memory m + where _tree @> m.tree + returning id + ) + select count(*) into strict _deleted + from d + ; + end if; + + return _deleted; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _tree_access jsonb +, _tree ltree +, _access int4 +) +returns bigint +as $func$ + with x as materialized + ( + select a.tree_path + from {{schema}}.tree_access(_tree_access) a + where a.access >= _access + ) + select count(*) + from {{schema}}.memory m + where _tree @> m.tree + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _tree_access jsonb +, _query lquery +, _access int4 +) +returns bigint +as $func$ + with x as materialized + ( + select a.tree_path + from {{schema}}.tree_access(_tree_access) a + where a.access >= _access + ) + select count(*) + from {{schema}}.memory m + where m.tree ~ _query + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _tree_access jsonb +, _query ltxtquery +, _access int4 +) +returns bigint +as $func$ + with x as materialized + ( + select a.tree_path + from {{schema}}.tree_access(_tree_access) a + where a.access >= _access + ) + select count(*) + from {{schema}}.memory m + where m.tree @ _query + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_tree +( _tree_access jsonb +, _query lquery +) +returns table +( tree ltree +, count bigint +) +as $func$ + with a as materialized + ( + select a.tree_path + from {{schema}}.tree_access(_tree_access) a + where a.access >= 1 + ) + , m as + ( + select distinct m.id, m.tree + from {{schema}}.memory m + where m.tree ~ _query + and exists + ( + select 1 + from a + where a.tree_path @> m.tree + ) + ) + select + subltree(m.tree, 0, i) as tree + , count(m.id) as count + from m + cross join lateral generate_series(1, nlevel(m.tree)) i + group by 1 + order by 1 +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/space/migrate/idempotent/002_search.sql b/packages/database/space/migrate/idempotent/002_search.sql new file mode 100644 index 0000000..d9504e7 --- /dev/null +++ b/packages/database/space/migrate/idempotent/002_search.sql @@ -0,0 +1,341 @@ +------------------------------------------------------------------------------- +-- search_memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.search_memory +( _tree_access jsonb +, _bm25 bm25query default null +, _vec halfvec({{embedding_dimensions}}) default null +, _max_vec_dist float8 default null +, _ltree ltree default null +, _lquery lquery default null +, _ltxtquery ltxtquery default null +, _meta_contains jsonb default null +, _temporal_within tstzrange default null +, _temporal_overlaps tstzrange default null +, _temporal_before timestamptz default null +, _temporal_after timestamptz default null +, _regexp text default null +, _limit bigint default 10 +, _order text default 'desc' -- unranked (filter-only) result order by id: 'desc' (newest first) | 'asc' +) +returns table +( id uuid +, meta jsonb +, tree ltree +, temporal tstzrange +, content text +, has_embedding bool +, created_at timestamptz +, updated_at timestamptz +, score float8 +) +as $func$ +declare + _filter_count int = 0; + _score text; + _filters text[] = '{}'::text; + _order_by text; + _sql text; +begin + -- _bm25 OR _vec but NOT BOTH + if _bm25 is not null and _vec is not null then + raise exception 'providing both _bm25 and _vec is not supported' + using errcode = 'invalid_parameter_value'; + end if; + + if _max_vec_dist is not null and _vec is null then + raise exception '_max_vec_dist provided but _vec was not provided' + using errcode = 'invalid_parameter_value'; + end if; + + -- min 1, max 1000, default 10 + _limit = greatest(least(coalesce(_limit, 10), 1000), 1); + + -- bm25 or semantic + -- score and order by + case + when _bm25 is not null then + _filter_count = _filter_count + 1; + -- <@> is negative bm25 score. smaller values means better match. order by this for index scans + -- negative score * -1 = score. higher score means better match + _score = format($sql$, (m.content <@> %L::bm25query) * -1 as score$sql$, _bm25); + _order_by = format($sql$order by m.content <@> %L::bm25query, m.id$sql$, _bm25); + when _vec is not null then + _filter_count = _filter_count + 1; + -- <=> is cosine distance. smaller distance means better match. order by this for index scans + -- distance * -1 = "score". higher score means better match + _score = format($sql$, (m.embedding <=> %L::halfvec({{embedding_dimensions}})) * -1 as score$sql$, _vec); + _order_by = format($sql$order by m.embedding <=> %L::halfvec({{embedding_dimensions}}), m.id$sql$, _vec); + _filters = array_append + ( _filters + , $sql$and m.embedding is not null$sql$ + ); + if _max_vec_dist is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and (m.embedding <=> %L::halfvec({{embedding_dimensions}})) <= %L::float8$sql$, _vec, _max_vec_dist) + ); + end if; + else + -- no ranking arm: constant score, typed float8 to match the return column + _score = $sql$, (-1)::float8 as score$sql$; + -- Order by id — a uuidv7, so creation-time-ordered (and message-time-ordered + -- for the importer's deterministic ids), i.e. a chronological browse. Default + -- desc (newest first). `_order` is whitelisted to asc|desc to keep this + -- interpolation injection-safe. + _order_by = format + ( $sql$order by m.id %s$sql$ + , case when lower(coalesce(_order, 'desc')) = 'asc' then 'asc' else 'desc' end + ); + end case; + + -- ltree + if _ltree is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::ltree @> m.tree$sql$, _ltree) + ); + end if; + + -- lquery + if _lquery is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.tree ~ %L::lquery$sql$, _lquery) + ); + end if; + + -- ltxtquery + if _ltxtquery is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.tree @ %L::ltxtquery$sql$, _ltxtquery) + ); + end if; + + -- meta_contains + if _meta_contains is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.meta @> %L::jsonb$sql$, _meta_contains) + ); + end if; + + -- temporal_within + if _temporal_within is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::tstzrange @> m.temporal$sql$, _temporal_within) + ); + end if; + + -- temporal_overlaps + if _temporal_overlaps is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::tstzrange && m.temporal$sql$, _temporal_overlaps) + ); + end if; + + -- temporal_before + if _temporal_before is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and tstzrange('-infinity'::timestamptz, %L::timestamptz, '[]') @> m.temporal$sql$, _temporal_before) + ); + end if; + + -- temporal_after + if _temporal_after is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and tstzrange(%L::timestamptz, 'infinity'::timestamptz, '[]') @> m.temporal$sql$, _temporal_after) + ); + end if; + + -- regexp + if _regexp is not null then + if _filter_count = 0 then + raise exception 'regexp must not be the only filter criteria' + using errcode = 'invalid_parameter_value'; + end if; + _filters = array_append + ( _filters + , format($sql$and m.content ~* %L::text$sql$, _regexp) + ); + end if; + + -- construct the query + _sql = format( + $sql$ + with x as materialized + ( + select x.tree_path + from jsonb_to_recordset($1) x(tree_path ltree, access int) + where x.access >= 1 + ) + select + m.id + , m.meta + , m.tree + , m.temporal + , m.content + , m.embedding is not null + , m.created_at + , m.updated_at + %s + from {{schema}}.memory m + where exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) + %s + %s + limit $2 + $sql$ + , _score + , coalesce + ( + ( + select string_agg(x, E'\n ') + from unnest(_filters) x + ) + , '' + ) + , _order_by + ); + + return query execute _sql using _tree_access, _limit; +end; +$func$ language plpgsql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- hybrid_search_memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.hybrid_search_memory +( _tree_access jsonb +, _bm25 bm25query +, _vec halfvec({{embedding_dimensions}}) +, _max_vec_dist float8 default null +, _ltree ltree default null +, _lquery lquery default null +, _ltxtquery ltxtquery default null +, _meta_contains jsonb default null +, _temporal_within tstzrange default null +, _temporal_overlaps tstzrange default null +, _temporal_before timestamptz default null +, _temporal_after timestamptz default null +, _regexp text default null +, _k float8 default 60.0 +, _candidate_limit bigint default 30 +, _fulltext_weight float8 default 1.0 +, _semantic_weight float8 default 1.0 +, _limit bigint default 10 +) +returns table +( id uuid +, meta jsonb +, tree ltree +, temporal tstzrange +, content text +, has_embedding bool +, created_at timestamptz +, updated_at timestamptz +, score float8 +) +as $func$ +declare +begin + if _bm25 is null then + raise exception '_bm25 must not be null' + using errcode = 'invalid_parameter_value'; + end if; + + if _vec is null then + raise exception '_vec must not be null' + using errcode = 'invalid_parameter_value'; + end if; + + _k = greatest(coalesce(_k, 60.0), 0.0); + _limit = greatest(least(coalesce(_limit, 10), 1000), 1); + _candidate_limit = greatest + ( least(coalesce(_candidate_limit, 30), 1000) + , _limit + ); + _fulltext_weight = greatest(least(coalesce(_fulltext_weight, 1.0), 1.0), 0.0); + _semantic_weight = greatest(least(coalesce(_semantic_weight, 1.0), 1.0), 0.0); + + -- reciprocal rank fusion + return query + select + coalesce(x1.id, x2.id) as id + , coalesce(x1.meta, x2.meta) as meta + , coalesce(x1.tree, x2.tree) as tree + , coalesce(x1.temporal, x2.temporal) as temporal + , coalesce(x1.content, x2.content) as content + , coalesce(x1.has_embedding, x2.has_embedding) as has_embedding + , coalesce(x1.created_at, x2.created_at) as created_at + , coalesce(x1.updated_at, x2.updated_at) as updated_at + , coalesce(_fulltext_weight / (_k + x1.rank), 0.0) + + coalesce(_semantic_weight / (_k + x2.rank), 0.0) as score + from + ( + select + row_number() over (order by m.score desc, m.id) as rank + , m.* + from {{schema}}.search_memory + ( _tree_access => _tree_access + , _bm25 => _bm25 + , _ltree => _ltree + , _lquery => _lquery + , _ltxtquery => _ltxtquery + , _meta_contains => _meta_contains + , _temporal_within => _temporal_within + , _temporal_overlaps => _temporal_overlaps + , _temporal_before => _temporal_before + , _temporal_after => _temporal_after + , _regexp => _regexp + , _limit => _candidate_limit + ) m + ) x1 + full outer join + ( + select + row_number() over (order by m.score desc, m.id) as rank + , m.* + from {{schema}}.search_memory + ( _tree_access => _tree_access + , _vec => _vec + , _max_vec_dist => _max_vec_dist + , _ltree => _ltree + , _lquery => _lquery + , _ltxtquery => _ltxtquery + , _meta_contains => _meta_contains + , _temporal_within => _temporal_within + , _temporal_overlaps => _temporal_overlaps + , _temporal_before => _temporal_before + , _temporal_after => _temporal_after + , _regexp => _regexp + , _limit => _candidate_limit + ) m + ) x2 on (x1.id = x2.id) + order by score desc, id + limit _limit + ; +end; +$func$ language plpgsql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/database/space/migrate/idempotent/003_embedding_queue.sql b/packages/database/space/migrate/idempotent/003_embedding_queue.sql new file mode 100644 index 0000000..048a083 --- /dev/null +++ b/packages/database/space/migrate/idempotent/003_embedding_queue.sql @@ -0,0 +1,250 @@ + +------------------------------------------------------------------------------- +-- enqueue_embedding +------------------------------------------------------------------------------- +create or replace function {{schema}}.enqueue_embedding() +returns trigger +as $func$ +begin + insert into {{schema}}.embedding_queue (memory_id, embedding_version) + values (new.id, new.embedding_version); + return new; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +create or replace trigger memory_enqueue_embedding_insert +after insert on {{schema}}.memory +for each row +when (new.embedding is null) -- it's possible to insert with an embedding +execute function {{schema}}.enqueue_embedding() +; + +create or replace trigger memory_enqueue_embedding_update +after update on {{schema}}.memory +for each row +when +( old.content is distinct from new.content + and new.embedding is null +) +execute function {{schema}}.enqueue_embedding() +; + +------------------------------------------------------------------------------- +-- claim_embedding_batch +------------------------------------------------------------------------------- +create or replace function {{schema}}.claim_embedding_batch +( _batch_size int default 10 +, _lock_duration interval default '5 minutes' +, _max_attempts int default 3 +) +returns table +( queue_id bigint +, memory_id uuid +, embedding_version int +, content text +) +as $func$ +declare + _rec record; + _mem record; + _claimed_count int = 0; +begin + -- bulk-cancel visible queue rows superseded by a newer row for the same memory + update {{schema}}.embedding_queue eq + set outcome = 'cancelled' + where eq.outcome is null + and eq.vt <= now() + and exists + ( + select 1 + from {{schema}}.embedding_queue newer + where newer.memory_id = eq.memory_id + and newer.embedding_version > eq.embedding_version + and newer.outcome is null + ); + + -- sweep: finalize exhausted rows orphaned by worker crash + -- (attempts reached max but outcome was never written back) + update {{schema}}.embedding_queue + set + outcome = 'failed' + , last_error = coalesce(last_error, 'exceeded max attempts') + where outcome is null + and vt <= now() + and attempts >= _max_attempts + ; + + for _rec in + ( + select + eq.id + , eq.memory_id + , eq.embedding_version + from {{schema}}.embedding_queue eq + where eq.outcome is null + and eq.vt <= now() + and eq.attempts < _max_attempts + order by eq.vt + for update skip locked + ) + loop + -- check memory still exists + current version + select m.content, m.embedding_version + into _mem + from {{schema}}.memory m + where m.id = _rec.memory_id + ; + + if not found or _mem.content is null then + -- memory deleted or empty → cancel queue row + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = _rec.id; + continue; + end if; + + if _rec.embedding_version != _mem.embedding_version then + -- stale version → cancel + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = _rec.id; + continue; + end if; + + -- claim this row + update {{schema}}.embedding_queue q set + vt = now() + _lock_duration + , attempts = q.attempts + 1 + where id = _rec.id; + + queue_id = _rec.id; + memory_id = _rec.memory_id; + embedding_version = _rec.embedding_version; + content = _mem.content; + return next; + + _claimed_count = _claimed_count + 1; + exit when _claimed_count >= _batch_size; + end loop; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- prune embedding queue +------------------------------------------------------------------------------- +-- prune terminal queue rows older than the retention window. +-- runs opportunistically from the worker on spaces that returned no +-- claimable work, so the queue table doesn't grow unbounded. +-- +-- relies on embedding_queue_archive_idx (created_at) where outcome is not null +-- from migration 005, so the no-op case is cheap. +create or replace function {{schema}}.prune_embedding_queue(_retention interval default '7 days') +returns bigint +as $func$ +declare + pruned bigint; +begin + delete from {{schema}}.embedding_queue + where outcome is not null + and created_at < now() - _retention + ; + get diagnostics pruned = row_count; + return pruned; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- write-back: complete_embedding / fail_embedding / release_embedding +-- The worker claims with claim_embedding_batch, generates embeddings out of +-- band, then finalizes each row through one of these (so the worker holds no +-- inline SQL). Claim and write-back are separate transactions; on a transient +-- failure the row keeps outcome NULL and becomes claimable again after its +-- visibility timeout. +------------------------------------------------------------------------------- + +-- Version-guarded write-back. Writes the embedding to the memory only if its +-- embedding_version still matches the claimed version, then finalizes the queue +-- row: 'completed' when written, 'cancelled' when the memory was superseded +-- (content changed → newer version) or deleted in the meantime. Atomic; returns +-- the outcome. +create or replace function {{schema}}.complete_embedding +( _queue_id bigint +, _memory_id uuid +, _embedding_version int +, _embedding halfvec +) +returns text +as $func$ +declare + _updated int; + _outcome text; +begin + update {{schema}}.memory + set embedding = _embedding + where id = _memory_id + and embedding_version = _embedding_version + ; + get diagnostics _updated = row_count; + + _outcome = case when _updated > 0 then 'completed' else 'cancelled' end; + + update {{schema}}.embedding_queue + set outcome = _outcome + where id = _queue_id + ; + return _outcome; +end; +$func$ +language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +-- Record a transient embedding error without finalizing: leaves outcome NULL so +-- the row retries (the claim sweep fails it once attempts are exhausted). No-op +-- when the row is already terminal or was CASCADE-deleted with its memory. +create or replace function {{schema}}.fail_embedding +( _queue_id bigint +, _error text +) +returns void +as $func$ + update {{schema}}.embedding_queue + set last_error = _error + where id = _queue_id + and outcome is null + ; +$func$ +language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +-- Undo a claim for a transient rate limit — the inverse of claim_embedding_batch: +-- decrement attempts (the rate limit must not consume the attempt budget) AND +-- reset the visibility timeout so the row is immediately claimable again. +-- Without resetting vt the row would sit out the full claim lock (~minutes) +-- before retrying; the worker's own rate-limit backoff (honoring Retry-After) +-- paces the actual retry. No-op once the row is terminal. +create or replace function {{schema}}.release_embedding +( _queue_id bigint +) +returns void +as $func$ + update {{schema}}.embedding_queue + set attempts = greatest(attempts - 1, 0) + , vt = now() + where id = _queue_id + and outcome is null + ; +$func$ +language sql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; diff --git a/packages/accounts/migrate/migrations/sql.d.ts b/packages/database/space/migrate/idempotent/sql.d.ts similarity index 100% rename from packages/accounts/migrate/migrations/sql.d.ts rename to packages/database/space/migrate/idempotent/sql.d.ts diff --git a/packages/engine/migrate/migrations/002_memory.sql b/packages/database/space/migrate/incremental/001_memory.sql similarity index 72% rename from packages/engine/migrate/migrations/002_memory.sql rename to packages/database/space/migrate/incremental/001_memory.sql index 7c8a4de..c66eea7 100644 --- a/packages/engine/migrate/migrations/002_memory.sql +++ b/packages/database/space/migrate/incremental/001_memory.sql @@ -1,21 +1,18 @@ +------------------------------------------------------------------------------- +-- memory +------------------------------------------------------------------------------- create table {{schema}}.memory ( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) -, meta jsonb not null default '{}' +, meta jsonb not null default '{}' check (jsonb_typeof(meta) = 'object') , tree ltree not null default ''::ltree , temporal tstzrange , content text not null , embedding halfvec({{embedding_dimensions}}) , embedding_version int4 not null default 1 -, embedding_attempts int4 not null default 0 -, embedding_last_error text , created_at timestamptz not null default now() -, created_by uuid , updated_at timestamptz ); -grant select on {{schema}}.memory to me_ro; -grant select, insert, update, delete on {{schema}}.memory to me_rw; - -- index for faceted search create index memory_meta_gin_idx on {{schema}}.memory using gin (meta); @@ -24,7 +21,7 @@ create index memory_temporal_gist_idx on {{schema}}.memory using gist (temporal) -- index for BM25 text search create index memory_content_bm25_idx on {{schema}}.memory using bm25 (content) -with (text_config = '{{bm25_text_config}}', k1 = {{bm25_k1}}, b = {{bm25_b}}); +with (text_config = {{bm25_text_config}}, k1 = {{bm25_k1}}, b = {{bm25_b}}); -- index for vector similarity search create index memory_embedding_hnsw_idx on {{schema}}.memory using hnsw (embedding halfvec_cosine_ops) @@ -33,12 +30,6 @@ with (m = {{hnsw_m}}, ef_construction = {{hnsw_ef_construction}}); -- index for hierarchical organization create index memory_tree_gist_idx on {{schema}}.memory using gist (tree); --- index for efficiently finding rows with null embeddings -create index memory_null_embedding_idx on {{schema}}.memory (created_at) where (embedding is null and embedding_attempts < 3); - --- make sure the metadata is an object -alter table {{schema}}.memory add check (jsonb_typeof(meta) = 'object'); - /* enforce consistent temporal range conventions: - point-in-time events: lower = upper with inclusive bounds '[same,same]' diff --git a/packages/database/space/migrate/incremental/002_embedding_queue.sql b/packages/database/space/migrate/incremental/002_embedding_queue.sql new file mode 100644 index 0000000..468fe45 --- /dev/null +++ b/packages/database/space/migrate/incremental/002_embedding_queue.sql @@ -0,0 +1,22 @@ +------------------------------------------------------------------------------- +-- embedding queue +------------------------------------------------------------------------------- +-- per-space embedding queue table +create table {{schema}}.embedding_queue +( id bigint generated always as identity primary key +, memory_id uuid not null references {{schema}}.memory(id) on delete cascade +, embedding_version int not null +, vt timestamptz not null default now() +, outcome text check (outcome is null or outcome in ('completed', 'failed', 'cancelled')) +, attempts int not null default 0 +, last_error text +, created_at timestamptz not null default now() +, updated_at timestamptz +); + +-- index to find items to claim +create index embedding_queue_claim_idx on {{schema}}.embedding_queue (vt) where outcome is null; +-- index also used in finding items to claim. used to ensure there aren't any items for the same memory with a newer version +create index embedding_queue_memory_idx on {{schema}}.embedding_queue (memory_id, embedding_version desc) where outcome is null; +-- index to find items that have resolved to an outcome. these can be pruned +create index embedding_queue_archive_idx on {{schema}}.embedding_queue (created_at) where outcome is not null; diff --git a/packages/engine/migrate/migrations/sql.d.ts b/packages/database/space/migrate/incremental/sql.d.ts similarity index 100% rename from packages/engine/migrate/migrations/sql.d.ts rename to packages/database/space/migrate/incremental/sql.d.ts diff --git a/packages/database/space/migrate/migrate.integration.test.ts b/packages/database/space/migrate/migrate.integration.test.ts new file mode 100644 index 0000000..09df1a1 --- /dev/null +++ b/packages/database/space/migrate/migrate.integration.test.ts @@ -0,0 +1,538 @@ +// Integration tests for per-space data-plane migrations (migrateSpace) and the +// shared database bootstrap (bootstrapSpaceDatabase). +// +// Provisioning a space is latency-bound (many sequential statements; ~seconds +// against a remote ghost db), so we provision a small fixed set of spaces once +// in beforeAll — concurrently, via Promise.all — and run fast read-only +// assertions against them. Only the handful of tests that need a private, +// mutable space provision their own. +// +// Tests are serial within the file (Bun 1.3's `test.concurrent` deadlocks when +// many heavy migration transactions overlap). Parallelism comes from two +// places instead: concurrent provisioning in beforeAll, and running the core +// and space suites as separate processes (`bun run test:db`). Spaces are +// isolated by unique `me_` schema, so those processes never collide. +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Sql as SQL } from "postgres"; +import { SPACE_SCHEMA_VERSION } from "../version"; +import { bootstrapSpaceDatabase } from "./bootstrap"; +import { migrateSpace, provisionSpace } from "./migrate"; +import { + appliedMigrations, + columnType, + connect, + expectReject, + getIndexReloptions, + getSchemaVersion, + listFunctions, + listIndexes, + listTables, + listTriggers, + randomSlug, + schemaExists, + TestSpace, + tableExists, + withTestSpace, +} from "./test-utils"; + +const EXPECTED_TABLES = ["embedding_queue", "memory", "migration", "version"]; + +const EXPECTED_MIGRATIONS = ["001_memory", "002_embedding_queue"]; + +const EXPECTED_MEMORY_FUNCTIONS = [ + "batch_create_memory", + "copy_tree", + "count_tree", + "create_memory", + "delete_memory", + "delete_tree", + "get_memory", + "has_tree_access", + "hybrid_search_memory", + "list_tree", + "move_tree", + "patch_memory", + "search_memory", + "tree_access", +]; + +const EXPECTED_QUEUE_FUNCTIONS = [ + "claim_embedding_batch", + "enqueue_embedding", + "prune_embedding_queue", +]; + +const EXPECTED_MEMORY_INDEXES = [ + "memory_content_bm25_idx", + "memory_embedding_hnsw_idx", + "memory_meta_gin_idx", + "memory_temporal_gist_idx", + "memory_tree_gist_idx", +]; + +let sql: SQL; +// Shared spaces, provisioned once. Read-only shape/functional assertions run +// against these; their schemas never change underneath each other. +let canonical: TestSpace; // default params; also used for functional smoke +let dim768: TestSpace; // custom embedding dimension +let customIdx: TestSpace; // custom HNSW + BM25 index parameters + +beforeAll(async () => { + sql = connect(12); + await bootstrapSpaceDatabase(sql); + [canonical, dim768, customIdx] = await Promise.all([ + TestSpace.create(sql), + TestSpace.create(sql, { embeddingDimensions: 768 }), + TestSpace.create(sql, { + hnswM: 8, + hnswEfConstruction: 32, + bm25K1: 1.5, + bm25B: 0.5, + }), + ]); +}); + +afterAll(async () => { + await Promise.all([canonical?.drop(), dim768?.drop(), customIdx?.drop()]); + await sql.end(); +}); + +describe("provisionSpace (caller-transaction, transactional DDL)", () => { + test("rolls back the whole schema when the caller's transaction aborts", async () => { + const slug = randomSlug(); + const schema = `metest_${slug}`; + await expect( + sql.begin(async (tx) => { + await provisionSpace(tx, { slug, schema, embeddingDimensions: 4 }); + // visible inside the transaction (schema + memory table created) + const [r] = + await tx`select to_regclass(${`${schema}.memory`}) is not null as present`; + expect(r?.present).toBe(true); + throw new Error("rollback"); + }), + ).rejects.toThrow("rollback"); + // schema + bm25/hnsw index DDL all rolled back — nothing left behind + expect(await schemaExists(sql, schema)).toBe(false); + }); + + test("commits a fully-migrated space when the transaction succeeds", async () => { + const slug = randomSlug(); + const schema = `metest_${slug}`; + try { + await sql.begin(async (tx) => { + await provisionSpace(tx, { slug, schema, embeddingDimensions: 4 }); + }); + expect(await schemaExists(sql, schema)).toBe(true); + expect(await tableExists(sql, schema, "memory")).toBe(true); + expect(await getSchemaVersion(sql, schema)).toBe(SPACE_SCHEMA_VERSION); + } finally { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + } + }); +}); + +describe("provisioned space schema", () => { + test("creates the space schema", async () => { + expect(await schemaExists(sql, canonical.schema)).toBe(true); + }); + + test("creates infrastructure and domain tables", async () => { + const tables = await listTables(sql, canonical.schema); + for (const table of EXPECTED_TABLES) { + expect(tables).toContain(table); + } + }); + + test("records every incremental migration exactly once", async () => { + expect(await appliedMigrations(sql, canonical.schema)).toEqual( + EXPECTED_MIGRATIONS, + ); + }); + + test("stamps the schema version", async () => { + expect(await getSchemaVersion(sql, canonical.schema)).toBe( + SPACE_SCHEMA_VERSION, + ); + }); + + test("creates the memory + queue functions", async () => { + const functions = await listFunctions(sql, canonical.schema); + for (const fn of [ + ...EXPECTED_MEMORY_FUNCTIONS, + ...EXPECTED_QUEUE_FUNCTIONS, + ]) { + expect(functions).toContain(fn); + } + }); + + test("creates all memory search indexes", async () => { + const indexes = await listIndexes(sql, canonical.schema, "memory"); + for (const idx of EXPECTED_MEMORY_INDEXES) { + expect(indexes).toContain(idx); + } + }); + + test("memory.embedding defaults to halfvec(1536)", async () => { + expect(await columnType(sql, canonical.schema, "memory", "embedding")).toBe( + "halfvec(1536)", + ); + }); + + test("installs the memory update trigger", async () => { + const triggers = await listTriggers(sql, canonical.schema, "memory"); + expect(triggers).toContain("memory_before_update_trg"); + }); +}); + +describe("migration templating", () => { + test("applies a custom embedding dimension to the table and search fn", async () => { + // The template var drives the column type ... + expect(await columnType(sql, dim768.schema, "memory", "embedding")).toBe( + "halfvec(768)", + ); + // ... and is baked into the search function body's vector casts. + const [row] = await sql.unsafe( + `select pg_get_functiondef(p.oid) as def + from pg_proc p join pg_namespace n on n.oid = p.pronamespace + where n.nspname = '${dim768.schema}' and p.proname = 'search_memory'`, + ); + expect(row?.def).toContain("halfvec(768)"); + }); + + test("applies custom HNSW index parameters", async () => { + const opts = await getIndexReloptions( + sql, + customIdx.schema, + "memory_embedding_hnsw_idx", + ); + expect(opts).toContain("m=8"); + expect(opts).toContain("ef_construction=32"); + }); + + test("applies custom BM25 index parameters", async () => { + const opts = await getIndexReloptions( + sql, + customIdx.schema, + "memory_content_bm25_idx", + ); + expect(opts).toContain("k1=1.5"); + expect(opts).toContain("b=0.5"); + }); +}); + +describe("migration behavior", () => { + test("is idempotent: re-running is safe", async () => { + await withTestSpace(sql, {}, async (space) => { + const before = await appliedMigrations(sql, space.schema); + await migrateSpace(sql, { slug: space.slug, schema: space.schema }); + expect(await appliedMigrations(sql, space.schema)).toEqual(before); + expect(await getSchemaVersion(sql, space.schema)).toBe( + SPACE_SCHEMA_VERSION, + ); + }); + }); + + test("rejects a downgrade (db version newer than app)", async () => { + await withTestSpace(sql, {}, async (space) => { + await sql.unsafe(`update ${space.schema}.version set version = '99.0.0'`); + await expect( + migrateSpace(sql, { slug: space.slug, schema: space.schema }), + ).rejects.toThrow(/older than database version/); + }); + }); + + test("rejects invalid slugs before touching the database", async () => { + for (const slug of ["BAD", "short", "way-too-long-slug", "has space12"]) { + await expect(migrateSpace(sql, { slug })).rejects.toThrow( + /Invalid space slug/, + ); + } + }); + + test("provisions distinct spaces independently and in parallel", async () => { + const [a, b] = await Promise.all([ + TestSpace.create(sql), + TestSpace.create(sql), + ]); + try { + expect(a.schema).not.toBe(b.schema); + expect(await schemaExists(sql, a.schema)).toBe(true); + expect(await schemaExists(sql, b.schema)).toBe(true); + // Dropping one leaves the other intact. + await a.drop(); + expect(await schemaExists(sql, a.schema)).toBe(false); + expect(await schemaExists(sql, b.schema)).toBe(true); + } finally { + await a.drop(); + await b.drop(); + } + }); +}); + +describe("provisioned schema is functional", () => { + // Shape assertions read the catalog, so sharing `canonical` with these write + // smoke tests is safe — inserted rows don't affect schema introspection. + test("accepts a memory and fires the update trigger", async () => { + const [row] = await sql.unsafe( + `insert into ${canonical.schema}.memory (content, tree) + values ('hello world', 'a.b') returning id, updated_at`, + ); + expect(row?.id).toBeDefined(); + expect(row?.updated_at).toBeNull(); + + const [updated] = await sql.unsafe( + `update ${canonical.schema}.memory set content = 'changed' + where id = '${row?.id}' returning updated_at`, + ); + expect(updated?.updated_at).not.toBeNull(); + }); + + // create_memory's conditional upsert: (treeAccess, tree, content, id, meta, + // temporal, replaceIfMetaDiffers) → zero rows (skip) or (id, inserted). + const OWNER = `'[{"tree_path": "", "access": 3}]'::jsonb`; + const createMemory = (args: string) => + sql.unsafe(`select * from ${canonical.schema}.create_memory(${args})`); + + test("create_memory skips a duplicate explicit id by default", async () => { + // Deterministic-id importers re-submit existing ids; with no replace key + // the second create must be a zero-row no-op leaving the row intact. + const id = "01941000-0000-7000-8000-000000000001"; + const [first] = await createMemory( + `${OWNER}, 'a.dup'::ltree, 'original', '${id}'::uuid`, + ); + expect(first?.id).toBe(id); + expect(first?.inserted).toBe(true); + + const second = await createMemory( + `${OWNER}, 'a.dup'::ltree, 'replacement', '${id}'::uuid`, + ); + expect(second.length).toBe(0); + + const [row] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("original"); + }); + + test("create_memory replaces a duplicate when the meta key differs, skips when it matches", async () => { + const id = "01941000-0000-7000-8000-000000000002"; + await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v1', '${id}'::uuid, '{"v": "1"}'::jsonb`, + ); + + // Same version → skip, content untouched. + const same = await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v1 again', '${id}'::uuid, '{"v": "1"}'::jsonb, null, 'v'`, + ); + expect(same.length).toBe(0); + + // Bumped version → replaced in place, inserted = false. + const [bumped] = await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v2', '${id}'::uuid, '{"v": "2"}'::jsonb, null, 'v'`, + ); + expect(bumped?.id).toBe(id); + expect(bumped?.inserted).toBe(false); + + const [row] = await sql.unsafe( + `select content, meta->>'v' as v, updated_at + from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("render v2"); + expect(row?.v).toBe("2"); + expect(row?.updated_at).not.toBeNull(); + + // A key absent on the stored row but present on the new record counts as + // "differs" (legacy rows written before the version key existed). + const [legacy] = await createMemory( + `${OWNER}, 'a.ver'::ltree, 'render v3', '${id}'::uuid, '{"v": "2", "legacy_v": "1"}'::jsonb, null, 'legacy_v'`, + ); + expect(legacy?.inserted).toBe(false); + const [afterLegacy] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(afterLegacy?.content).toBe("render v3"); + }); + + test("create_memory replace requires write access on the existing row's tree", async () => { + // Row lives under a.secret; the caller's grant covers only a.open — the + // insert-arm check passes (target a.open) but the replace arm must skip. + const id = "01941000-0000-7000-8000-000000000003"; + await createMemory( + `${OWNER}, 'a.secret'::ltree, 'guarded', '${id}'::uuid, '{"v": "1"}'::jsonb`, + ); + + const limited = `'[{"tree_path": "a.open", "access": 3}]'::jsonb`; + const res = await createMemory( + `${limited}, 'a.open'::ltree, 'hijack', '${id}'::uuid, '{"v": "2"}'::jsonb, null, 'v'`, + ); + expect(res.length).toBe(0); + + const [row] = await sql.unsafe( + `select content, tree::text as tree from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("guarded"); + expect(row?.tree).toBe("a.secret"); + }); + + test("batch_create_memory upserts a whole batch in one statement", async () => { + const stale = "01941000-0000-7000-8000-00000000b001"; + const fresh = "01941000-0000-7000-8000-00000000b002"; + await createMemory( + `${OWNER}, 'a.batch'::ltree, 'old render', '${stale}'::uuid, '{"v": "1"}'::jsonb`, + ); + await createMemory( + `${OWNER}, 'a.batch'::ltree, 'current', '${fresh}'::uuid, '{"v": "2"}'::jsonb`, + ); + + // One call carrying: a stale row (update), a current row (skip), a brand + // new row (insert), and a no-id row (insert with generated id). + const rows = await sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${OWNER}, + array['${stale}', '${fresh}', '01941000-0000-7000-8000-00000000b003', null]::uuid[], + array['a.batch', 'a.batch', 'a.batch', 'a.batch']::ltree[], + array['new render', 'untouched', 'added', 'generated']::text[], + '[{"v": "2"}, {"v": "2"}, {"v": "2"}, {"v": "2"}]'::jsonb, + array[null, null, null, null]::tstzrange[], + 'v' + )`, + ); + const byId = new Map(rows.map((r) => [r.id as string, r.inserted])); + expect(byId.get(stale)).toBe(false); // replaced + expect(byId.has(fresh)).toBe(false); // skipped → absent + expect(byId.get("01941000-0000-7000-8000-00000000b003")).toBe(true); + expect(rows).toHaveLength(3); // 2 inserts + 1 update + + const [updated] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${stale}'`, + ); + expect(updated?.content).toBe("new render"); + const [skipped] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${fresh}'`, + ); + expect(skipped?.content).toBe("current"); + }); + + test("batch_create_memory collapses an id repeated within the batch (first wins)", async () => { + const id = "01941000-0000-7000-8000-00000000b010"; + const rows = await sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${OWNER}, + array['${id}', '${id}']::uuid[], + array['a.batchdup', 'a.batchdup']::ltree[], + array['first', 'second']::text[], + '[{}, {}]'::jsonb, + array[null, null]::tstzrange[] + )`, + ); + expect(rows).toHaveLength(1); + const [row] = await sql.unsafe( + `select content from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(row?.content).toBe("first"); + }); + + test("batch_create_memory rejects misaligned arrays and bad target access", async () => { + await expectReject(() => + sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${OWNER}, + array[null]::uuid[], + array['a.x', 'a.y']::ltree[], + array['one']::text[], + '[{}]'::jsonb, + array[null]::tstzrange[] + )`, + ), + ); + + // One row outside the grant fails the whole batch before any write. + const limited = `'[{"tree_path": "a.open", "access": 3}]'::jsonb`; + await expectReject(() => + sql.unsafe( + `select * from ${canonical.schema}.batch_create_memory( + ${limited}, + array[null, null]::uuid[], + array['a.open', 'a.secret']::ltree[], + array['ok', 'denied']::text[], + '[{}, {}]'::jsonb, + array[null, null]::tstzrange[] + )`, + ), + ); + const [count] = await sql.unsafe( + `select count(*)::int as n from ${canonical.schema}.memory + where content in ('ok', 'denied')`, + ); + expect(count?.n).toBe(0); + }); + + test("create_memory replace re-embeds only when content changed", async () => { + const id = "01941000-0000-7000-8000-000000000004"; + await createMemory( + `${OWNER}, 'a.emb'::ltree, 'stable content', '${id}'::uuid, '{"v": "1"}'::jsonb`, + ); + // Simulate the worker: embedding present (default 1536 dims), queue drained. + await sql.unsafe( + `update ${canonical.schema}.memory + set embedding = ('[' || repeat('0,', 1535) || '0]')::halfvec + where id = '${id}'`, + ); + await sql.unsafe( + `delete from ${canonical.schema}.embedding_queue where memory_id = '${id}'`, + ); + + // Meta-only replace (identical content): embedding survives, no re-enqueue. + await createMemory( + `${OWNER}, 'a.emb'::ltree, 'stable content', '${id}'::uuid, '{"v": "2"}'::jsonb, null, 'v'`, + ); + const [afterMeta] = await sql.unsafe( + `select (embedding is not null) as has_embedding, + (select count(*)::int from ${canonical.schema}.embedding_queue + where memory_id = '${id}' and outcome is null) as queued + from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(afterMeta?.has_embedding).toBe(true); + expect(afterMeta?.queued).toBe(0); + + // Content replace: embedding invalidated and re-enqueued. + await createMemory( + `${OWNER}, 'a.emb'::ltree, 'new content', '${id}'::uuid, '{"v": "3"}'::jsonb, null, 'v'`, + ); + const [afterContent] = await sql.unsafe( + `select (embedding is null) as embedding_cleared, + (select count(*)::int from ${canonical.schema}.embedding_queue + where memory_id = '${id}' and outcome is null) as queued + from ${canonical.schema}.memory where id = '${id}'`, + ); + expect(afterContent?.embedding_cleared).toBe(true); + expect(afterContent?.queued).toBe(1); + }); + + test("enforces the meta-is-object constraint", async () => { + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.memory (content, meta) + values ('x', '[]'::jsonb)`, + ), + ); + }); + + test("enforces the temporal range convention", async () => { + // A closed [start,end] range with start < end violates the convention + // (ranges must be inclusive-exclusive); only point-in-time may close upper. + await expectReject(() => + sql.unsafe( + `insert into ${canonical.schema}.memory (content, temporal) + values ('x', '[2024-01-01, 2024-01-02]'::tstzrange)`, + ), + ); + }); +}); + +describe("bootstrapSpaceDatabase", () => { + test("is idempotent", async () => { + await bootstrapSpaceDatabase(sql); + await bootstrapSpaceDatabase(sql); + }); +}); diff --git a/packages/database/space/migrate/migrate.ts b/packages/database/space/migrate/migrate.ts new file mode 100644 index 0000000..4c7836d --- /dev/null +++ b/packages/database/space/migrate/migrate.ts @@ -0,0 +1,248 @@ +import { info, reportError, span } from "@pydantic/logfire-node"; +import { semver } from "bun"; +import type { ISql, Sql as SQL } from "postgres"; +import { + acquireAdvisoryLock, + advisoryLockKey, + applySessionTimeouts, + doesSchemaExist, + executeSqlFile, + isValidSchemaName, + type Migration, + runSchemaMigrations, + template, +} from "../../migrate/kit"; +import { isValidSlug, slugToSchema } from "../slug"; +import { SPACE_SCHEMA_VERSION } from "../version"; +import idempotent001 from "./idempotent/001_memory.sql" with { type: "text" }; +import idempotent002 from "./idempotent/002_search.sql" with { type: "text" }; +import idempotent003 from "./idempotent/003_embedding_queue.sql" with { + type: "text", +}; +import incremental001 from "./incremental/001_memory.sql" with { type: "text" }; +import incremental002 from "./incremental/002_embedding_queue.sql" with { + type: "text", +}; +import provisionSql from "./provision.sql" with { type: "text" }; + +const DIR = "packages/database/space/migrate"; + +const incrementals: Migration[] = [ + { + name: "001_memory", + file: "incremental/001_memory.sql", + sql: incremental001, + }, + { + name: "002_embedding_queue", + file: "incremental/002_embedding_queue.sql", + sql: incremental002, + }, +]; + +const idempotents: Migration[] = [ + { name: "001_memory", file: "idempotent/001_memory.sql", sql: idempotent001 }, + { name: "002_search", file: "idempotent/002_search.sql", sql: idempotent002 }, + { + name: "003_embedding_queue", + file: "idempotent/003_embedding_queue.sql", + sql: idempotent003, + }, +]; + +export interface MigrateSpaceOptions { + slug: string; + /** + * Override the target schema name. Defaults to `slugToSchema(slug)` (the + * production `me_`). Provided for tests, which provision under a + * `metest_` prefix so leftovers are trivially distinguishable from real + * spaces (see scripts/clean-test-schemas.ts). Must be a valid lowercase SQL + * identifier; the slug is still validated and used for locking/telemetry. + */ + schema?: string; + logSqlFiles?: boolean; + embeddingDimensions?: number; + bm25TextConfig?: string; + bm25K1?: number; + bm25B?: number; + hnswM?: number; + hnswEfConstruction?: number; + statementTimeout?: string; + lockTimeout?: string; + transactionTimeout?: string; + idleInTransactionSessionTimeout?: string; +} + +interface NormalizedMigrateSpaceOptions { + slug: string; + schema?: string; + logSqlFiles: boolean; + schemaVersion: string; + embeddingDimensions: number; + bm25TextConfig: string; + bm25K1: number; + bm25B: number; + hnswM: number; + hnswEfConstruction: number; + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +/** Validate options and resolve the target schema name. */ +function resolveSchema(opts: NormalizedMigrateSpaceOptions): string { + if (!isValidSlug(opts.slug)) { + throw new Error( + `Invalid space slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, + ); + } + if (opts.schema !== undefined && !isValidSchemaName(opts.schema)) { + throw new Error( + `Invalid schema override: "${opts.schema}" — must be a valid lowercase SQL identifier (<= 63 chars)`, + ); + } + if (!semver.satisfies(opts.schemaVersion, "*")) { + throw new Error(`Invalid schema version: "${opts.schemaVersion}"`); + } + return opts.schema ?? slugToSchema(opts.slug); +} + +/** Provision (if missing) then run all migrations, against a given transaction. */ +async function provisionAndRun( + tx: ISql, + schema: string, + opts: NormalizedMigrateSpaceOptions, +): Promise { + if (!(await doesSchemaExist(tx, schema))) { + await executeSqlFile(tx, template(provisionSql, { schema }), { + logSqlFiles: opts.logSqlFiles, + label: "space", + schema, + type: "provision", + dir: DIR, + file: "provision.sql", + }); + } + await runSchemaMigrations(tx, { + schema, + schemaVersion: opts.schemaVersion, + incrementals, + idempotents, + templateVars: templateVars(schema, opts), + label: "space", + dir: DIR, + logSqlFiles: opts.logSqlFiles, + }); +} + +/** + * Provision + migrate a space schema within the CALLER's transaction — no own + * transaction and no advisory lock. Because schema creation is transactional, + * the caller can compose this atomically with other writes (e.g. provisionUser + * creates the me_ schema + the core/auth rows in one transaction, so any + * failure rolls the schema back — no orphan, no cleanup). + * + * Use `migrateSpace` for the standalone re-migrate path: it owns its + * transaction + advisory lock to serialize concurrent migrators of an existing + * space. Skipping the lock here is safe — a freshly generated slug has no + * contender, and the schema-name / `space.slug` uniqueness makes any race + * fail-and-roll-back. + */ +export async function provisionSpace( + tx: ISql, + options: MigrateSpaceOptions, +): Promise { + const opts = normalizeMigrateSpaceOptions(options); + const schema = resolveSchema(opts); + await provisionAndRun(tx, schema, opts); +} + +export async function migrateSpace( + sql: SQL, + options: MigrateSpaceOptions, +): Promise { + const opts = normalizeMigrateSpaceOptions(options); + const attributes = migrateAttributes(opts); + + await span("space.migrate", { + attributes, + callback: async () => { + try { + const schema = resolveSchema(opts); + const schemaAttributes = { ...attributes, "db.schema": schema }; + const [key1, key2] = advisoryLockKey(`memory-space:schema:${schema}`); + + await sql.begin(async (tx) => { + await applySessionTimeouts(tx, opts); + const acquired = await span("space.migrate.acquire_lock", { + attributes: schemaAttributes, + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error( + `Unable to acquire lock for space slug ${opts.slug} migrations.`, + ); + } + await provisionAndRun(tx, schema, opts); + }); + info("Space migrations completed", schemaAttributes); + } catch (error) { + reportError("Space migration failed", error as Error, attributes); + throw error; + } + }, + }); +} + +function migrateAttributes( + options: NormalizedMigrateSpaceOptions, +): Record { + return { + "space.slug": options.slug, + "space.schema_version": options.schemaVersion, + "db.statement_timeout": options.statementTimeout, + "db.lock_timeout": options.lockTimeout, + "db.transaction_timeout": options.transactionTimeout, + "db.idle_in_transaction_session_timeout": + options.idleInTransactionSessionTimeout, + }; +} + +function normalizeMigrateSpaceOptions( + options: MigrateSpaceOptions, +): NormalizedMigrateSpaceOptions { + return { + slug: options.slug, + schema: options.schema, + logSqlFiles: options.logSqlFiles ?? false, + schemaVersion: SPACE_SCHEMA_VERSION, + embeddingDimensions: options.embeddingDimensions ?? 1536, + bm25TextConfig: options.bm25TextConfig ?? "english", + bm25K1: options.bm25K1 ?? 1.2, + bm25B: options.bm25B ?? 0.75, + hnswM: options.hnswM ?? 16, + hnswEfConstruction: options.hnswEfConstruction ?? 64, + statementTimeout: options.statementTimeout ?? "20s", + lockTimeout: options.lockTimeout ?? "5s", + transactionTimeout: options.transactionTimeout ?? "1min", + idleInTransactionSessionTimeout: + options.idleInTransactionSessionTimeout ?? "5s", + }; +} + +function templateVars( + schema: string, + options: NormalizedMigrateSpaceOptions, +): Record { + return { + ...options, + schema, + embedding_dimensions: options.embeddingDimensions, + bm25_text_config: options.bm25TextConfig, + bm25_k1: options.bm25K1, + bm25_b: options.bm25B, + hnsw_m: options.hnswM, + hnsw_ef_construction: options.hnswEfConstruction, + }; +} diff --git a/packages/database/space/migrate/provision.sql b/packages/database/space/migrate/provision.sql new file mode 100644 index 0000000..e98b9d9 --- /dev/null +++ b/packages/database/space/migrate/provision.sql @@ -0,0 +1,15 @@ +create schema {{schema}}; + +create table {{schema}}.version +( version text not null +, at timestamptz not null default now() +); + +create unique index version_singleton_idx on {{schema}}.version ((true)); -- only ONE row allowed +insert into {{schema}}.version (version) values ('0.0.0'); + +create table {{schema}}.migration +( name text not null constraint migration_pkey primary key +, applied_at_version text not null +, applied_at timestamptz not null default pg_catalog.clock_timestamp() +); diff --git a/packages/database/space/migrate/sql.d.ts b/packages/database/space/migrate/sql.d.ts new file mode 100644 index 0000000..89b092e --- /dev/null +++ b/packages/database/space/migrate/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const content: string; + export default content; +} diff --git a/packages/database/space/migrate/test-utils.ts b/packages/database/space/migrate/test-utils.ts new file mode 100644 index 0000000..88dbf41 --- /dev/null +++ b/packages/database/space/migrate/test-utils.ts @@ -0,0 +1,85 @@ +import type { Sql as SQL } from "postgres"; +import { type MigrateSpaceOptions, migrateSpace } from "./migrate"; + +// Connection, failure assertions, and schema introspection are shared with the +// core suite. +export * from "../../migrate/test-utils"; + +/** + * Test spaces provision under a `metest_` schema instead of the production + * `me_`, so leftover test schemas are distinguishable from real spaces by + * name alone: scripts/clean-test-schemas.ts sweeps `metest_*` (and + * `core_test_*`) and can never touch a production `me_*` space. `metest_` + * deliberately does not start with the `me_` engine-schema prefix. + */ +const TEST_SCHEMA_PREFIX = "metest_"; + +/** The throwaway schema name a test space provisions under (`metest_`). */ +export function testSchema(slug: string): string { + return `${TEST_SCHEMA_PREFIX}${slug}`; +} + +const SLUG_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; + +/** A random valid space slug: 12 lowercase alphanumeric chars. */ +export function randomSlug(): string { + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let slug = ""; + for (const b of bytes) slug += SLUG_ALPHABET[b % SLUG_ALPHABET.length]; + return slug; +} + +// --------------------------------------------------------------------------- +// TestSpace — a provisioned, isolated space schema +// --------------------------------------------------------------------------- + +/** + * A migrated space schema in the shared test database. Assumes + * bootstrapSpaceDatabase() has already run (extensions installed) — do that + * once per file in beforeAll. + */ +export class TestSpace { + readonly slug: string; + readonly schema: string; + private readonly sql: SQL; + + private constructor(sql: SQL, slug: string) { + this.sql = sql; + this.slug = slug; + this.schema = testSchema(slug); + } + + static async create( + sql: SQL, + options: Omit & { + slug?: string; + } = {}, + ): Promise { + const slug = options.slug ?? randomSlug(); + // Provision under metest_ so leftovers are name-distinguishable from + // production me_ spaces (see clean-test-schemas.ts). + await migrateSpace(sql, { ...options, slug, schema: testSchema(slug) }); + return new TestSpace(sql, slug); + } + + async drop(): Promise { + await this.sql.unsafe(`drop schema if exists ${this.schema} cascade`); + } +} + +/** + * Provision a fresh space, run `fn` against it, and always drop it afterward. + * Safe to call from concurrent tests — each gets its own unique schema. + */ +export async function withTestSpace( + sql: SQL, + options: Omit & { slug?: string }, + fn: (space: TestSpace) => Promise, +): Promise { + const space = await TestSpace.create(sql, options); + try { + return await fn(space); + } finally { + await space.drop(); + } +} diff --git a/packages/database/space/path.test.ts b/packages/database/space/path.test.ts new file mode 100644 index 0000000..2755253 --- /dev/null +++ b/packages/database/space/path.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, test } from "bun:test"; +import { + classifyTreeFilter, + denormalizeTreePath, + homePrefix, + normalizeTreeFilter, + normalizeTreePath, + TreePathError, +} from "./path"; + +const ID = "0199c2a4-f8e1-7b3c-9d2e-5a6f08b4c1d7"; +const HOME = "home.0199c2a4f8e17b3c9d2e5a6f08b4c1d7"; + +describe("normalizeTreePath", () => { + test("root forms collapse to the empty path", () => { + for (const root of ["", "/", ".", "///", "..", "/.//"]) { + expect(normalizeTreePath(root)).toBe(""); + } + }); + + test("slash and dot separators are interchangeable; runs collapse; ends trim", () => { + expect(normalizeTreePath("foo")).toBe("foo"); + expect(normalizeTreePath("foo/bar")).toBe("foo.bar"); + expect(normalizeTreePath("foo.bar")).toBe("foo.bar"); + expect(normalizeTreePath("/foo/bar/")).toBe("foo.bar"); + expect(normalizeTreePath("foo//bar")).toBe("foo.bar"); + expect(normalizeTreePath("a/b.c")).toBe("a.b.c"); + }); + + test("hyphen and underscore labels are valid", () => { + expect(normalizeTreePath("my-project/notes_2")).toBe("my-project.notes_2"); + }); + + test("rejects illegal label characters", () => { + expect(() => normalizeTreePath("foo bar")).toThrow(TreePathError); + expect(() => normalizeTreePath("foo@bar")).toThrow(TreePathError); + expect(() => normalizeTreePath("a/b!c")).toThrow(TreePathError); + }); + + test("leading ~ expands to the caller's home", () => { + expect(normalizeTreePath("~", { home: ID })).toBe(HOME); + expect(normalizeTreePath("~/bar", { home: ID })).toBe(`${HOME}.bar`); + expect(normalizeTreePath("~.bar", { home: ID })).toBe(`${HOME}.bar`); + expect(normalizeTreePath("~/a/b", { home: ID })).toBe(`${HOME}.a.b`); + }); + + test("~ requires a home and is only valid as the first segment", () => { + expect(() => normalizeTreePath("~/bar")).toThrow(TreePathError); + expect(() => normalizeTreePath("foo/~/bar", { home: ID })).toThrow( + TreePathError, + ); + }); + + test("a literal 'home' path is not special (only ~ injects the id)", () => { + expect(normalizeTreePath("home/bar", { home: ID })).toBe("home.bar"); + }); +}); + +describe("normalizeTreeFilter", () => { + test("passes lquery / ltxtquery syntax through unvalidated", () => { + expect(normalizeTreeFilter("*")).toBe("*"); + expect(normalizeTreeFilter("*.api.*")).toBe("*.api.*"); + expect(normalizeTreeFilter("foo & bar")).toBe("foo & bar"); + }); + + test("normalizes separators and trims", () => { + expect(normalizeTreeFilter("")).toBe(""); + expect(normalizeTreeFilter("/foo/bar/")).toBe("foo.bar"); + expect(normalizeTreeFilter("foo//bar")).toBe("foo.bar"); + }); + + test("expands a leading ~ but keeps the wildcard remainder", () => { + expect(normalizeTreeFilter("~", { home: ID })).toBe(HOME); + expect(normalizeTreeFilter("~/proj.*", { home: ID })).toBe( + `${HOME}.proj.*`, + ); + expect(normalizeTreeFilter("~.*", { home: ID })).toBe(`${HOME}.*`); + }); +}); + +describe("classifyTreeFilter", () => { + test("empty input is no filter", () => { + expect(classifyTreeFilter("")).toBeNull(); + expect(classifyTreeFilter("/")).toBeNull(); + expect(classifyTreeFilter(" ")).toBeNull(); + }); + + test("a bare path classifies as ltree (containment)", () => { + expect(classifyTreeFilter("share")).toEqual({ + kind: "ltree", + value: "share", + }); + expect(classifyTreeFilter("/share/projects/")).toEqual({ + kind: "ltree", + value: "share.projects", + }); + expect(classifyTreeFilter("my-proj.notes_2")).toEqual({ + kind: "ltree", + value: "my-proj.notes_2", + }); + }); + + test("a wildcard classifies as lquery", () => { + expect(classifyTreeFilter("share.projects.*")).toEqual({ + kind: "lquery", + value: "share.projects.*", + }); + expect(classifyTreeFilter("*.api.*")).toEqual({ + kind: "lquery", + value: "*.api.*", + }); + // `|` and `!` are lquery label operators, not ltxtquery here. + expect(classifyTreeFilter("foo|bar.baz")).toEqual({ + kind: "lquery", + value: "foo|bar.baz", + }); + }); + + test("an `&` boolean classifies as ltxtquery", () => { + expect(classifyTreeFilter("api & v2")).toEqual({ + kind: "ltxtquery", + value: "api & v2", + }); + }); + + test("a leading ~ expands before classification", () => { + expect(classifyTreeFilter("~.*", { home: ID })).toEqual({ + kind: "lquery", + value: `${HOME}.*`, + }); + expect(classifyTreeFilter("~/notes", { home: ID })).toEqual({ + kind: "ltree", + value: `${HOME}.notes`, + }); + }); +}); + +describe("homePrefix", () => { + test("strips hyphens from the principal id", () => { + expect(homePrefix(ID)).toBe(HOME); + }); +}); + +describe("denormalizeTreePath", () => { + test("reverse-maps the caller's home to ~ with the canonical dot separator", () => { + expect(denormalizeTreePath(HOME, { home: ID })).toBe("~"); + expect(denormalizeTreePath(`${HOME}.bar`, { home: ID })).toBe("~.bar"); + expect(denormalizeTreePath(`${HOME}.a.b`, { home: ID })).toBe("~.a.b"); + }); + + test("leaves non-home paths (and other principals' homes) unchanged", () => { + expect(denormalizeTreePath("work.projects", { home: ID })).toBe( + "work.projects", + ); + expect(denormalizeTreePath("home.deadbeef.x", { home: ID })).toBe( + "home.deadbeef.x", + ); + expect(denormalizeTreePath(`${HOME}.bar`)).toBe(`${HOME}.bar`); // no home opt + }); + + test("round-trips with normalizeTreePath", () => { + const display = denormalizeTreePath(`${HOME}.a.b`, { home: ID }); // ~.a.b + expect(normalizeTreePath(display, { home: ID })).toBe(`${HOME}.a.b`); + }); +}); diff --git a/packages/database/space/path.ts b/packages/database/space/path.ts new file mode 100644 index 0000000..44c7802 --- /dev/null +++ b/packages/database/space/path.ts @@ -0,0 +1,195 @@ +/** + * User-facing tree-path normalization. + * + * Memories live under an ltree `tree` path (dot-separated; the root is the empty + * path). At the user-facing boundary (RPC handlers, CLI, MCP) we accept lenient + * input and normalize it once to the canonical ltree form. The store layer and + * SQL functions stay ltree-native and only ever see canonical paths. + * + * Conventions: + * - **Separators**: `/` and `.` are interchangeable; runs collapse; leading and + * trailing separators are dropped. `/a/b`, `a/b`, `a.b` → `a.b`. + * - **Root**: ``, `/`, `.` → `` (the empty ltree path). + * - **Home**: a leading `~` segment expands to `home.`, where + * `` is the caller's principal id with hyphens stripped (a valid + * ltree label). `~` → `home.`, `~/bar` → `home..bar`. + * `~` is only meaningful as the first segment. + * - **Labels** (concrete paths): each segment must be a legal ltree label + * (`[A-Za-z0-9_-]+`, PG16+); anything else throws `TreePathError`. + * + * Two entry points: + * - `normalizeTreePath` — a concrete path (create/update/move/grant/…). Strict + * label validation. + * - `normalizeTreeFilter` — a search filter, which may be an ltree `lquery` + * (`*.api.*`) or `ltxtquery`. Expands `~` and slashes but does NOT validate + * labels, so wildcard/query syntax passes through untouched. + */ + +/** The reserved top-level namespace for per-principal home directories. */ +export const HOME_NAMESPACE = "home"; + +/** + * The reserved top-level namespace for a space's shared tree. Unlike `home`, + * this is a single shared root (not per-principal) and carries no input sugar — + * `share/x` normalizes like any other path. It exists as a named constant + * because membership/invitations grant a configurable level (read/write/owner) + * at this root; see core `redeem_space_invitations`. + * + * Canonically defined in `@memory.build/protocol` (the wire contract) and + * re-exported here so the database/server side keeps a single source of truth. + */ +export { SHARE_NAMESPACE } from "@memory.build/protocol"; + +/** A legal ltree label (PostgreSQL 16+): letters, digits, underscore, hyphen. */ +const LTREE_LABEL = /^[A-Za-z0-9_-]+$/; + +/** Thrown on malformed user input (mapped to a validation error at the boundary). */ +export class TreePathError extends Error { + constructor(message: string) { + super(message); + this.name = "TreePathError"; + } +} + +export interface TreePathOptions { + /** + * The principal id whose home a leading `~` expands to. Required to use `~`; + * omitting it makes a `~` segment an error. + */ + home?: string; +} + +/** The canonical ltree prefix for a principal's home: `home.`. */ +export function homePrefix(principalId: string): string { + const id = principalId.replace(/-/g, ""); + if (!LTREE_LABEL.test(id)) { + throw new TreePathError( + `invalid home principal id: ${JSON.stringify(principalId)}`, + ); + } + return `${HOME_NAMESPACE}.${id}`; +} + +/** Split on runs of `/` or `.`, dropping empty segments. */ +function splitSegments(input: string): string[] { + return input.split(/[/.]+/).filter((s) => s.length > 0); +} + +/** + * Normalize a concrete tree path to canonical ltree. Lenient on separators and + * a leading `~`; strict on labels. Returns `""` for the root. + */ +export function normalizeTreePath( + input: string, + opts: TreePathOptions = {}, +): string { + const segments = splitSegments(input); + if (segments.length === 0) return ""; + + const out: string[] = []; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i] as string; + if (seg === "~") { + if (i !== 0) { + throw new TreePathError("'~' is only valid as the first path segment"); + } + if (opts.home === undefined) { + throw new TreePathError("'~' (home) is not available here"); + } + out.push(homePrefix(opts.home)); // already a valid `home.` ltree + continue; + } + if (!LTREE_LABEL.test(seg)) { + throw new TreePathError( + `invalid tree path segment: ${JSON.stringify(seg)}`, + ); + } + out.push(seg); + } + return out.join("."); +} + +/** + * Normalize a search filter (lquery / ltxtquery): expand a leading `~`, treat + * `/` as a separator, collapse and trim separators — but pass wildcard/query + * syntax through unvalidated. Returns `""` when there is no filter. + */ +export function normalizeTreeFilter( + input: string, + opts: TreePathOptions = {}, +): string { + let s = input.trim(); + if (s === "") return ""; + + // Leading `~` home expansion (only as the first segment). + if (s === "~" || s.startsWith("~/") || s.startsWith("~.")) { + if (opts.home === undefined) { + throw new TreePathError("'~' (home) is not available here"); + } + s = homePrefix(opts.home) + s.slice(1); // "~/foo" → "home./foo" + } + + // Slash → dot, collapse separator runs, trim ends. + s = s + .replace(/\//g, ".") + .replace(/\.{2,}/g, ".") + .replace(/^\.+|\.+$/g, ""); + return s; +} + +/** A bare ltree path: dot-separated `[A-Za-z0-9_-]` labels, no query operators. */ +const LTREE_PATH = /^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$/; + +/** + * A classified, normalized search tree filter, tagged by which ltree + * pattern type it is so the caller can bind it to the matching SQL parameter + * (`ltree` → `@>` containment, `lquery` → `~`, `ltxtquery` → `@`). + */ +export type TreeFilter = + | { kind: "ltree"; value: string } + | { kind: "lquery"; value: string } + | { kind: "ltxtquery"; value: string }; + +/** + * Normalize a search tree filter (via `normalizeTreeFilter`) and classify it as + * an exact ltree path, an `lquery` pattern, or an `ltxtquery` label search. + * `normalizeTreeFilter` only expands `~`/slashes — it does not pick a type — so + * without this the caller can't know which SQL parameter to bind, and casting a + * wildcard like `foo.*` to `::ltree` throws. Returns `null` for empty input (no + * filter). + * + * Classification (the input has already had `~`/slashes normalized): + * - bare ltree path (only `[A-Za-z0-9_-]` labels + `.`) → `ltree` (containment) + * - contains `&` (ltxtquery's boolean AND — never valid in lquery) → `ltxtquery` + * - anything else (wildcards `*`, `|`, `!`, `{n}`, …) → `lquery` + */ +export function classifyTreeFilter( + input: string, + opts: TreePathOptions = {}, +): TreeFilter | null { + const s = normalizeTreeFilter(input, opts); + if (s === "") return null; + if (LTREE_PATH.test(s)) return { kind: "ltree", value: s }; + if (s.includes("&")) return { kind: "ltxtquery", value: s }; + return { kind: "lquery", value: s }; +} + +/** + * Reverse of the home expansion, for display. A path under the given + * principal's home is shown with a leading `~`, keeping the canonical dot + * separator (`home.` → `~`, `home..a.b` → `~.a.b`); everything else + * (including other principals' homes) is returned unchanged. Dot is the + * canonical output separator throughout. + */ +export function denormalizeTreePath( + path: string, + opts: TreePathOptions = {}, +): string { + if (opts.home === undefined) return path; + const prefix = homePrefix(opts.home); + if (path === prefix) return "~"; + if (path.startsWith(`${prefix}.`)) { + return `~${path.slice(prefix.length)}`; // home..a.b → ~.a.b + } + return path; +} diff --git a/packages/database/space/slug.test.ts b/packages/database/space/slug.test.ts new file mode 100644 index 0000000..615f86e --- /dev/null +++ b/packages/database/space/slug.test.ts @@ -0,0 +1,102 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { randomSlug } from "./migrate/test-utils"; +import { + isValidSlug, + isValidSpaceSchema, + schemaToSlug, + slugToSchema, +} from "./slug"; + +describe("isValidSlug", () => { + test("accepts 12 lowercase alphanumeric chars", () => { + expect(isValidSlug("abcdef012345")).toBe(true); + expect(isValidSlug("000000000000")).toBe(true); + expect(isValidSlug("zzzzzzzzzzzz")).toBe(true); + }); + + test("rejects wrong length", () => { + expect(isValidSlug("abc")).toBe(false); + expect(isValidSlug("abcdef01234")).toBe(false); // 11 + expect(isValidSlug("abcdef0123456")).toBe(false); // 13 + expect(isValidSlug("")).toBe(false); + }); + + test("rejects uppercase and special characters", () => { + expect(isValidSlug("ABCDEF012345")).toBe(false); + expect(isValidSlug("abcdef-12345")).toBe(false); + expect(isValidSlug("abcdef_12345")).toBe(false); + expect(isValidSlug("abcde 012345")).toBe(false); + }); +}); + +describe("isValidSpaceSchema", () => { + test("accepts me_ prefix + valid slug", () => { + expect(isValidSpaceSchema("me_abcdef012345")).toBe(true); + }); + + test("rejects missing prefix or wrong shape", () => { + expect(isValidSpaceSchema("abcdef012345")).toBe(false); + expect(isValidSpaceSchema("me_ABCDEF012345")).toBe(false); + expect(isValidSpaceSchema("me_abc")).toBe(false); + expect(isValidSpaceSchema("core")).toBe(false); + expect(isValidSpaceSchema("xx_abcdef012345")).toBe(false); + }); +}); + +describe("slugToSchema / schemaToSlug", () => { + test("slugToSchema prepends me_", () => { + expect(slugToSchema("abcdef012345")).toBe("me_abcdef012345"); + }); + + test("schemaToSlug strips the me_ prefix", () => { + expect(schemaToSlug("me_abcdef012345")).toBe("abcdef012345"); + }); + + test("round-trips", () => { + const slug = "0a1b2c3d4e5f"; + expect(schemaToSlug(slugToSchema(slug))).toBe(slug); + expect(isValidSpaceSchema(slugToSchema(slug))).toBe(true); + }); +}); + +describe("SPACE_SCHEMA_PREFIX override", () => { + const ORIGINAL = process.env.SPACE_SCHEMA_PREFIX; + afterEach(() => { + if (ORIGINAL === undefined) delete process.env.SPACE_SCHEMA_PREFIX; + else process.env.SPACE_SCHEMA_PREFIX = ORIGINAL; + }); + + test("slugToSchema / schemaToSlug honor a non-default prefix", () => { + process.env.SPACE_SCHEMA_PREFIX = "metest_"; + expect(slugToSchema("abcdef012345")).toBe("metest_abcdef012345"); + expect(schemaToSlug("metest_abcdef012345")).toBe("abcdef012345"); + expect(schemaToSlug(slugToSchema("0a1b2c3d4e5f"))).toBe("0a1b2c3d4e5f"); + }); + + test("isValidSpaceSchema matches the configured prefix, not me_", () => { + process.env.SPACE_SCHEMA_PREFIX = "metest_"; + expect(isValidSpaceSchema("metest_abcdef012345")).toBe(true); + expect(isValidSpaceSchema("me_abcdef012345")).toBe(false); + }); + + test("rejects an invalid prefix", () => { + process.env.SPACE_SCHEMA_PREFIX = "9bad"; + expect(() => slugToSchema("abcdef012345")).toThrow(); + }); +}); + +describe("randomSlug", () => { + test("always produces a valid, schema-safe slug", () => { + for (let i = 0; i < 1000; i++) { + const slug = randomSlug(); + expect(isValidSlug(slug)).toBe(true); + expect(isValidSpaceSchema(slugToSchema(slug))).toBe(true); + } + }); + + test("is effectively unique across many draws", () => { + const seen = new Set(); + for (let i = 0; i < 10_000; i++) seen.add(randomSlug()); + expect(seen.size).toBe(10_000); + }); +}); diff --git a/packages/database/space/slug.ts b/packages/database/space/slug.ts new file mode 100644 index 0000000..7dc4200 --- /dev/null +++ b/packages/database/space/slug.ts @@ -0,0 +1,44 @@ +const SLUG_RE = /^[a-z0-9]{12}$/; +const SLUG_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +const DEFAULT_SPACE_PREFIX = "me_"; + +// The space-schema prefix is configurable via SPACE_SCHEMA_PREFIX (default +// "me_"), mirroring AUTH_SCHEMA/CORE_SCHEMA. It is read lazily (per call) rather +// than at module load so a test can set the env before the first call despite +// import hoisting (the e2e harness sets "metest_" so its spaces are swept by the +// existing schema reclaimer). Production leaves it unset → "me_". +function spacePrefix(): string { + const p = process.env.SPACE_SCHEMA_PREFIX ?? DEFAULT_SPACE_PREFIX; + // Must be a SQL-identifier-safe prefix: lowercase letters/digits/underscore, + // starting with a letter and ending with "_". + if (!/^[a-z][a-z0-9_]*_$/.test(p)) { + throw new Error( + `Invalid SPACE_SCHEMA_PREFIX: "${p}" — must be lowercase [a-z0-9_], start with a letter, and end with "_"`, + ); + } + return p; +} + +/** Generate a random 12-char lowercase-alphanumeric space slug. */ +export function generateSlug(): string { + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let slug = ""; + for (const b of bytes) slug += SLUG_ALPHABET[b % 36]; + return slug; +} + +export function isValidSpaceSchema(name: string): boolean { + return new RegExp(`^${spacePrefix()}[a-z0-9]{12}$`).test(name); +} + +export function isValidSlug(slug: string): boolean { + return SLUG_RE.test(slug); +} + +export function slugToSchema(slug: string): string { + return `${spacePrefix()}${slug}`; +} + +export function schemaToSlug(schema: string): string { + return schema.slice(spacePrefix().length); +} diff --git a/packages/database/space/version.ts b/packages/database/space/version.ts new file mode 100644 index 0000000..1b92647 --- /dev/null +++ b/packages/database/space/version.ts @@ -0,0 +1 @@ +export const SPACE_SCHEMA_VERSION = "0.0.1"; diff --git a/packages/database/tsconfig.json b/packages/database/tsconfig.json new file mode 100644 index 0000000..23b1d27 --- /dev/null +++ b/packages/database/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["**/*.ts", "**/*.d.ts"] +} diff --git a/packages/docs-site/lib/nav.ts b/packages/docs-site/lib/nav.ts index 9be813c..d2a3d9a 100644 --- a/packages/docs-site/lib/nav.ts +++ b/packages/docs-site/lib/nav.ts @@ -32,8 +32,9 @@ export const NAV: NavSection[] = [ { label: "me login", slug: "cli/me-login" }, { label: "me logout", slug: "cli/me-logout" }, { label: "me whoami", slug: "cli/me-whoami" }, - { label: "me engine", slug: "cli/me-engine" }, + { label: "me space", slug: "cli/me-space" }, { label: "me memory", slug: "cli/me-memory" }, + { label: "me import", slug: "cli/me-import" }, { label: "me mcp", slug: "cli/me-mcp" }, { label: "me claude", slug: "cli/me-claude" }, { label: "me codex", slug: "cli/me-codex" }, @@ -41,13 +42,10 @@ export const NAV: NavSection[] = [ { label: "me opencode", slug: "cli/me-opencode" }, { label: "me serve", slug: "cli/me-serve" }, { label: "Agent session imports", slug: "cli/agent-session-imports" }, - { label: "me user", slug: "cli/me-user" }, - { label: "me role", slug: "cli/me-role" }, - { label: "me grant", slug: "cli/me-grant" }, - { label: "me owner", slug: "cli/me-owner" }, - { label: "me org", slug: "cli/me-org" }, - { label: "me invitation", slug: "cli/me-invitation" }, + { label: "me agent", slug: "cli/me-agent" }, { label: "me apikey", slug: "cli/me-apikey" }, + { label: "me group", slug: "cli/me-group" }, + { label: "me access", slug: "cli/me-access" }, { label: "me pack", slug: "cli/me-pack" }, { label: "me completions", slug: "cli/me-completions" }, ], diff --git a/packages/embedding/generate.test.ts b/packages/embedding/generate.test.ts index c55e4b8..5076afd 100644 --- a/packages/embedding/generate.test.ts +++ b/packages/embedding/generate.test.ts @@ -7,79 +7,6 @@ import { } from "./generate"; import type { EmbeddingConfig } from "./types"; -// ============================================================================= -// Integration Tests (conditional) -// ============================================================================= - -const RUN_INTEGRATION = process.env.RUN_EMBEDDING_INTEGRATION === "1"; -const OLLAMA_URL = process.env.OLLAMA_URL ?? "http://localhost:11434"; - -const ollamaConfig: EmbeddingConfig = { - provider: "ollama", - model: "nomic-embed-text", - dimensions: 768, - baseUrl: OLLAMA_URL, -}; - -describe.skipIf(!RUN_INTEGRATION)("embedding integration (ollama)", () => { - test("generateEmbedding returns correct dimensions", async () => { - const result = await generateEmbedding("test text", ollamaConfig); - - expect(result.embedding).toBeInstanceOf(Array); - expect(result.embedding.length).toBe(768); - expect(typeof result.embedding[0]).toBe("number"); - expect(result.tokens).toBeGreaterThan(0); - }); - - test("generateEmbedding handles long text", async () => { - const longText = "word ".repeat(10000); // Very long text - const configWithTruncation: EmbeddingConfig = { - ...ollamaConfig, - options: { maxTokens: 8000 }, - }; - - const result = await generateEmbedding(longText, configWithTruncation); - expect(result.embedding.length).toBe(768); - expect(result.tokens).toBeGreaterThan(0); - }); - - test("generateEmbeddings returns results for batch", async () => { - const rows = [ - { id: "1", content: "first document" }, - { id: "2", content: "second document" }, - { id: "3", content: "third document" }, - ]; - - const results = await generateEmbeddings(rows, ollamaConfig); - - expect(results.length).toBe(3); - for (const result of results) { - expect(result.embedding.length).toBe(768); - expect(result.error).toBeUndefined(); - } - }); - - test("generateEmbeddings returns empty array for empty input", async () => { - const results = await generateEmbeddings([], ollamaConfig); - expect(results).toEqual([]); - }); - - test("validateConfig succeeds with valid config", async () => { - await expect(validateConfig(ollamaConfig)).resolves.toBeUndefined(); - }); - - test("validateConfig throws on dimension mismatch", async () => { - const badConfig: EmbeddingConfig = { - ...ollamaConfig, - dimensions: 512, // Wrong dimension - }; - - await expect(validateConfig(badConfig)).rejects.toThrow( - "Dimension mismatch", - ); - }); -}); - // ============================================================================= // OpenAI Integration Tests (conditional) // ============================================================================= @@ -92,7 +19,10 @@ const openaiConfig: EmbeddingConfig = { dimensions: 1536, }; -describe.skipIf(!RUN_OPENAI_INTEGRATION)( +// TEST_CI disables conditional skips: in CI this suite always runs (missing +// credentials fail loudly as test errors, never as a silent skip). Locally +// it stays opt-in via RUN_OPENAI_INTEGRATION=1. +describe.skipIf(!process.env.TEST_CI && !RUN_OPENAI_INTEGRATION)( "embedding integration (openai)", () => { test("generateEmbedding returns correct dimensions", async () => { diff --git a/packages/engine/core/api-key.test.ts b/packages/engine/core/api-key.test.ts new file mode 100644 index 0000000..f6c5ea8 --- /dev/null +++ b/packages/engine/core/api-key.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, test } from "bun:test"; +import { + formatApiKey, + generateLookupId, + generateSecret, + hashApiKeySecret, + isLegacyApiKey, + parseApiKey, +} from "./api-key"; + +describe("generateLookupId", () => { + test("generates a 16-char string", () => { + expect(generateLookupId()).toHaveLength(16); + }); + + test("only contains valid lookup_id characters", () => { + expect(generateLookupId()).toMatch(/^[A-Za-z0-9_-]{16}$/); + }); + + test("generates unique values", () => { + expect(generateLookupId()).not.toBe(generateLookupId()); + }); +}); + +describe("generateSecret", () => { + test("generates a 32-char string", () => { + expect(generateSecret()).toHaveLength(32); + }); + + test("only contains base64url characters", () => { + expect(generateSecret()).toMatch(/^[A-Za-z0-9_-]{32}$/); + }); + + test("generates unique values", () => { + expect(generateSecret()).not.toBe(generateSecret()); + }); +}); + +describe("hashApiKeySecret", () => { + test("is a stable hex sha256 digest", () => { + const h = hashApiKeySecret("a-secret"); + expect(h).toMatch(/^[0-9a-f]{64}$/); + expect(hashApiKeySecret("a-secret")).toBe(h); + }); + + test("different secrets produce different hashes", () => { + expect(hashApiKeySecret("secret-a")).not.toBe(hashApiKeySecret("secret-b")); + }); +}); + +describe("formatApiKey", () => { + test("formats key with all parts", () => { + expect(formatApiKey("lookupid12345678", "s".repeat(32))).toBe( + `me.lookupid12345678.${"s".repeat(32)}`, + ); + }); +}); + +describe("parseApiKey", () => { + const valid = `me.lookupid12345678.${"s".repeat(32)}`; + + test("parses a valid key (round-trips with formatApiKey)", () => { + const parsed = parseApiKey(valid); + expect(parsed).toEqual({ + lookupId: "lookupid12345678", + secret: "s".repeat(32), + }); + if (parsed) { + expect(formatApiKey(parsed.lookupId, parsed.secret)).toBe(valid); + } + }); + + test("returns null for the wrong prefix", () => { + expect(parseApiKey(`x.lookupid12345678.${"s".repeat(32)}`)).toBeNull(); + }); + + test("returns null for an invalid lookupId", () => { + expect(parseApiKey(`me.short.${"s".repeat(32)}`)).toBeNull(); + }); + + test("returns null for the wrong secret length", () => { + expect(parseApiKey("me.lookupid12345678.tooshort")).toBeNull(); + }); + + test("returns null for the wrong number of parts", () => { + expect(parseApiKey("me.lookupid12345678")).toBeNull(); + }); + + test("rejects a legacy 4-part key (with space slug)", () => { + expect( + parseApiKey(`me.abc123def456.lookupid12345678.${"s".repeat(32)}`), + ).toBeNull(); + }); +}); + +describe("isLegacyApiKey", () => { + const legacy = `me.abc123def456.lookupid12345678.${"s".repeat(32)}`; + + test("true for a 4-part legacy (space-scoped) key", () => { + expect(isLegacyApiKey(legacy)).toBe(true); + }); + + test("false for a current 3-part key", () => { + expect(isLegacyApiKey(`me.lookupid12345678.${"s".repeat(32)}`)).toBe(false); + }); + + test("false for an opaque session-like token", () => { + expect(isLegacyApiKey("a".repeat(43))).toBe(false); + }); + + test("false for a 4-part token with a malformed slug", () => { + expect( + isLegacyApiKey(`me.BADSLUG78901.lookupid12345678.${"s".repeat(32)}`), + ).toBe(false); + }); +}); diff --git a/packages/engine/core/api-key.ts b/packages/engine/core/api-key.ts new file mode 100644 index 0000000..07dbbce --- /dev/null +++ b/packages/engine/core/api-key.ts @@ -0,0 +1,83 @@ +/** + * API key helpers for the core control plane. + * + * Key format: me.{lookupId}.{secret} + * - me fixed prefix + * - lookupId 16-char id for the indexed db lookup + * - secret 32-char base64url random secret + * + * Keys are global per-principal credentials, not space-bound: the same key + * authenticates into any space the owning principal has been admitted to (the + * space is selected by the X-Me-Space header, gated by core.build_tree_access). + * + * The secret is high-entropy, so we store sha256(secret) and validate by + * equality in SQL (core.validate_api_key) — no per-request argon2 verify. This + * matches how session tokens are handled (see packages/accounts/util/hash.ts). + */ + +const LOOKUP_ID_LENGTH = 16; +const SECRET_LENGTH = 32; +const LOOKUP_ID_CHARSET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-"; + +/** Generate a random 16-char lookup id (matches the lookup_id check). */ +export function generateLookupId(): string { + const bytes = crypto.getRandomValues(new Uint8Array(LOOKUP_ID_LENGTH)); + let result = ""; + for (const byte of bytes) { + result += LOOKUP_ID_CHARSET[byte % LOOKUP_ID_CHARSET.length]; + } + return result; +} + +/** Generate a random 32-char base64url secret. */ +export function generateSecret(): string { + const bytes = crypto.getRandomValues(new Uint8Array(SECRET_LENGTH)); + return btoa(String.fromCharCode(...bytes)) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=/g, "") + .slice(0, SECRET_LENGTH); +} + +/** Hash a secret for storage / comparison: sha256, hex-encoded. */ +export function hashApiKeySecret(secret: string): string { + return new Bun.CryptoHasher("sha256").update(secret).digest("hex"); +} + +/** Assemble a full API key string from its parts. */ +export function formatApiKey(lookupId: string, secret: string): string { + return `me.${lookupId}.${secret}`; +} + +/** Parse an API key into its components; null if malformed. */ +export function parseApiKey( + key: string, +): { lookupId: string; secret: string } | null { + const parts = key.split("."); + if (parts.length !== 3) { + return null; + } + const [prefix, lookupId, secret] = parts; + if (prefix !== "me") return null; + if (!lookupId || !/^[A-Za-z0-9_-]{16}$/.test(lookupId)) return null; + if (!secret || secret.length !== SECRET_LENGTH) return null; + return { lookupId, secret }; +} + +/** + * True if the token is a legacy **space-scoped** api key + * (`me...`, the pre-global 4-part format). These no + * longer authenticate — callers use this to return a clear "recreate your key" + * error instead of a generic 401. New keys are 3-part (`parseApiKey`). + */ +export function isLegacyApiKey(token: string): boolean { + const parts = token.split("."); + return ( + parts.length === 4 && + parts[0] === "me" && + /^[a-z0-9]{12}$/.test(parts[1] ?? "") && + /^[A-Za-z0-9_-]{16}$/.test(parts[2] ?? "") && + (parts[3]?.length ?? 0) === SECRET_LENGTH + ); +} diff --git a/packages/engine/core/core.integration.test.ts b/packages/engine/core/core.integration.test.ts new file mode 100644 index 0000000..b786634 --- /dev/null +++ b/packages/engine/core/core.integration.test.ts @@ -0,0 +1,357 @@ +// Integration test for the core control-plane store additions (4C-2a): +// principal listing/rename/delete, group membership listing, grant listing, +// and api-key read/delete. Runs against a real core schema. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/engine/core/core.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { migrateCore } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { type CoreStore, coreStore } from "./db"; +import { ACCESS } from "./types"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let coreSchema: string; +let core: CoreStore; + +// Fresh space + owner user per test. +let spaceId: string; +let userId: string; +let userName: string; + +async function v7(): Promise { + const [row] = await sql`select uuidv7() as id`; + return row?.id as string; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + await migrateCore(sql, { schema: coreSchema }); + core = coreStore(sql, coreSchema); +}); + +afterAll(async () => { + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + spaceId = await core.createSpace(rand(12), "Test Space"); + userId = await v7(); + userName = `user_${rand(8)}@example.com`; + await core.createUser(userId, userName); + await core.addPrincipalToSpace(spaceId, userId, true); +}); + +test("getUserByName resolves a global user", async () => { + const u = await core.getUserByName(userName); + expect(u?.id).toBe(userId); + expect(u?.kind).toBe("u"); + expect(await core.getUserByName("nobody@example.com")).toBeNull(); +}); + +test("renamePrincipal refuses to rename users", async () => { + expect(await core.renamePrincipal(userId, "new@example.com")).toBe(false); + // the user's name is unchanged + expect((await core.getPrincipal(userId))?.name).toBe(userName); +}); + +test("listSpacePrincipals lists direct principals with admin flag and kind filter", async () => { + const all = await core.listSpacePrincipals(spaceId); + expect(all).toHaveLength(1); + expect(all[0]?.id).toBe(userId); + expect(all[0]?.direct).toBe(true); + expect(all[0]?.admin).toBe(true); + + expect(await core.listSpacePrincipals(spaceId, "u")).toHaveLength(1); + expect(await core.listSpacePrincipals(spaceId, "g")).toHaveLength(0); +}); + +test("listSpacePrincipals includes group-only principals (flagged direct=false)", async () => { + // a second user who is NOT added to the space directly, only via a group + const groupOnlyId = await v7(); + await core.createUser(groupOnlyId, `grouponly_${rand(8)}@example.com`); + const groupId = await core.createGroup(spaceId, "team"); + await core.addGroupMember(spaceId, groupId, groupOnlyId); + + const members = await core.listSpacePrincipals(spaceId, "u"); + const byId = Object.fromEntries(members.map((m) => [m.id, m])); + // owner is a direct member + expect(byId[userId]?.direct).toBe(true); + // the group-only user shows up as a member, flagged direct=false + expect(byId[groupOnlyId]).toBeDefined(); + expect(byId[groupOnlyId]?.direct).toBe(false); + expect(byId[groupOnlyId]?.admin).toBe(false); +}); + +test("agents appear as space members of kind 'a'", async () => { + const agentId = await core.createAgent(userId, `agent-${rand(6)}`); + await core.addPrincipalToSpace(spaceId, agentId); + const agents = await core.listSpacePrincipals(spaceId, "a"); + expect(agents).toHaveLength(1); + expect(agents[0]?.id).toBe(agentId); + expect(agents[0]?.ownerId).toBe(userId); +}); + +test("groups: create, list, rename, members, delete", async () => { + const groupId = await core.createGroup(spaceId, "eng"); + let groups = await core.listSpaceGroups(spaceId); + expect(groups.map((g) => g.name)).toContain("eng"); + + expect(await core.renamePrincipal(groupId, "engineering")).toBe(true); + groups = await core.listSpaceGroups(spaceId); + expect(groups.find((g) => g.id === groupId)?.name).toBe("engineering"); + + await core.addGroupMember(spaceId, groupId, userId, true); + const members = await core.listGroupMembers(spaceId, groupId); + expect(members).toHaveLength(1); + expect(members[0]?.memberId).toBe(userId); + expect(members[0]?.admin).toBe(true); + + const forMember = await core.listGroupsForMember(spaceId, userId); + expect(forMember.map((g) => g.groupId)).toContain(groupId); + + expect(await core.removeGroupMember(spaceId, groupId, userId)).toBe(true); + expect(await core.listGroupMembers(spaceId, groupId)).toHaveLength(0); + + expect(await core.deletePrincipal(groupId)).toBe(true); + expect(await core.listSpaceGroups(spaceId)).toHaveLength(0); +}); + +test("space admin transfers through an admin group", async () => { + const groupId = await core.createGroup(spaceId, `admins_${rand(6)}`); + // designate the group itself as an admin member of the space + await core.addPrincipalToSpace(spaceId, groupId, true); + + // a user added only to that group inherits space-admin transitively + const member = await v7(); + await core.createUser(member, `m_${rand(8)}@example.com`); + await core.addGroupMember(spaceId, groupId, member); + expect(await core.isSpaceAdmin(member, spaceId)).toBe(true); + + // a non-member is not an admin + const stranger = await v7(); + await core.createUser(stranger, `s_${rand(8)}@example.com`); + expect(await core.isSpaceAdmin(stranger, spaceId)).toBe(false); +}); + +test("group grants are inherited transitively (Model 2)", async () => { + // a user who is ONLY a group member (no direct principal_space row, no direct + // grant) inherits the group's grant via build_tree_access. + const groupOnly = await v7(); + await core.createUser(groupOnly, `go_${rand(8)}@example.com`); + const groupId = await core.createGroup(spaceId, `grp_${rand(6)}`); + await core.addGroupMember(spaceId, groupId, groupOnly); + await core.grantTreeAccess(spaceId, groupId, "shared", ACCESS.write); + + const ta = await core.buildTreeAccess(groupOnly, spaceId); + expect(ta).toContainEqual({ tree_path: "shared", access: ACCESS.write }); +}); + +test("listTreeAccessGrants returns grants; filterable by principal", async () => { + await core.grantTreeAccess(spaceId, userId, "a.b", ACCESS.write); + await core.grantTreeAccess(spaceId, userId, "c", ACCESS.owner); + + // the owner also has owner@home, granted when it joined the space (beforeEach) + const home = `home.${userId.replace(/-/g, "")}`; + + const all = await core.listTreeAccessGrants(spaceId); + const paths = all.map((g) => g.treePath).sort(); + expect(paths).toEqual([home, "a.b", "c"].sort()); + expect(all.find((g) => g.treePath === "c")?.access).toBe(ACCESS.owner); + + const forUser = await core.listTreeAccessGrants(spaceId, userId); + expect(forUser).toHaveLength(3); + + expect(await core.removeTreeAccessGrant(spaceId, userId, "a.b")).toBe(true); + expect(await core.listTreeAccessGrants(spaceId)).toHaveLength(2); +}); + +test("space invitations: create / list / redeem / revoke via the store", async () => { + // spaceId + the owner userId come from beforeEach; the owner is the inviter + const email = `invitee_${rand(8)}@example.com`; + const inviteId = await core.createSpaceInvitation(spaceId, email, { + admin: true, + shareAccess: ACCESS.write, + invitedBy: userId, + }); + expect(inviteId).toBeTruthy(); + + const pending = await core.listSpaceInvitations(spaceId); + expect(pending).toHaveLength(1); + expect(pending[0]?.email).toBe(email); + expect(pending[0]?.admin).toBe(true); + expect(pending[0]?.shareAccess).toBe(ACCESS.write); + expect(pending[0]?.invitedBy).toBe(userId); + expect(pending[0]?.invitedByName).toBe(userName); + + // the invitee registers and redeems + const inviteeId = await v7(); + await core.createUser(inviteeId, email); + const joined = await core.redeemSpaceInvitations(inviteeId, email); + expect(joined).toHaveLength(1); + expect(joined[0]?.spaceId).toBe(spaceId); + expect(joined[0]?.slug).toBeTruthy(); + expect(joined[0]?.admin).toBe(true); + expect(joined[0]?.shareAccess).toBe(ACCESS.write); + + // effective access: owner@home (from joining) + write@share + const ta = await core.buildTreeAccess(inviteeId, spaceId); + expect(ta).toContainEqual({ + tree_path: `home.${inviteeId.replace(/-/g, "")}`, + access: ACCESS.owner, + }); + expect(ta).toContainEqual({ tree_path: "share", access: ACCESS.write }); + + // accepted → no longer pending; re-redeem is a no-op + expect(await core.listSpaceInvitations(spaceId)).toHaveLength(0); + expect(await core.redeemSpaceInvitations(inviteeId, email)).toHaveLength(0); + + // a fresh invite (with no share grant) is revocable once + await core.createSpaceInvitation(spaceId, email, { + admin: false, + shareAccess: null, + invitedBy: userId, + }); + expect(await core.revokeSpaceInvitation(spaceId, email)).toBe(true); + expect(await core.revokeSpaceInvitation(spaceId, email)).toBe(false); +}); + +test("api keys: create, get, list, delete (no secret leaked)", async () => { + const key = await core.createApiKey(userId, "ci"); + expect(key.secret).toBeTruthy(); + + const got = await core.getApiKey(key.id); + expect(got?.id).toBe(key.id); + expect(got?.memberId).toBe(userId); + expect(got?.lookupId).toBe(key.lookupId); + expect(got?.name).toBe("ci"); + // metadata only — no secret field on ApiKeyInfo + expect((got as unknown as Record).secret).toBeUndefined(); + + const list = await core.listApiKeys(userId); + expect(list.map((k) => k.id)).toContain(key.id); + + expect(await core.deleteApiKey(key.id)).toBe(true); + expect(await core.getApiKey(key.id)).toBeNull(); + expect(await core.listApiKeys(userId)).toHaveLength(0); +}); + +// --------------------------------------------------------------------------- +// last-admin safeguard (enforce_last_admin trigger on principal_space) +// --------------------------------------------------------------------------- + +/** Assert a promise rejects with the last-admin guard's SQLSTATE (ME001). */ +async function expectLastAdmin(p: Promise) { + try { + await p; + throw new Error("expected a last-admin (ME001) rejection, but it resolved"); + } catch (e) { + expect((e as { code?: string }).code).toBe("ME001"); + } +} + +test("removing the last admin is rejected (ME001)", async () => { + // beforeEach made userId the space's sole admin. + await expectLastAdmin(core.removePrincipalFromSpace(spaceId, userId)); + // rolled back — the admin is still a member + const all = await core.listSpacePrincipals(spaceId); + expect(all.find((p) => p.id === userId)?.admin).toBe(true); +}); + +test("demoting the last admin is rejected (ME001)", async () => { + await expectLastAdmin(core.addPrincipalToSpace(spaceId, userId, false)); + const all = await core.listSpacePrincipals(spaceId); + expect(all.find((p) => p.id === userId)?.admin).toBe(true); +}); + +test("removing a non-last admin succeeds (another admin remains)", async () => { + const user2 = await v7(); + await core.createUser(user2, `admin2_${rand(8)}@example.com`); + await core.addPrincipalToSpace(spaceId, user2, true); // 2nd admin + + expect(await core.removePrincipalFromSpace(spaceId, userId)).toBe(true); + const admins = (await core.listSpacePrincipals(spaceId)).filter( + (p) => p.admin, + ); + expect(admins.map((p) => p.id)).toEqual([user2]); +}); + +test("deleting a group that is the space's only admin is rejected (ME001)", async () => { + // a fresh space whose sole effective admin is a user via an admin group + const sid = await core.createSpace(rand(12), "Group-admin Space"); + const groupId = await core.createGroup(sid, `admins_${rand(6)}`); + await core.addPrincipalToSpace(sid, groupId, true); + const member = await v7(); + await core.createUser(member, `gm_${rand(8)}@example.com`); + await core.addGroupMember(sid, groupId, member); // effective admin via the group + + await expectLastAdmin(core.deletePrincipal(groupId)); + // rolled back — the group is still the space's admin + const admins = (await core.listSpacePrincipals(sid)).filter((p) => p.admin); + expect(admins.map((p) => p.id)).toContain(groupId); +}); + +test("removing the last member of the sole admin group is rejected (ME001)", async () => { + const sid = await core.createSpace(rand(12), "Group-admin Space"); + const groupId = await core.createGroup(sid, `admins_${rand(6)}`); + await core.addPrincipalToSpace(sid, groupId, true); // group holds space-admin + const member = await v7(); + await core.createUser(member, `gm_${rand(8)}@example.com`); + await core.addGroupMember(sid, groupId, member); // sole effective admin + + // emptying the admin group leaves no effective admin + await expectLastAdmin(core.removeGroupMember(sid, groupId, member)); + expect( + (await core.listGroupMembers(sid, groupId)).map((m) => m.memberId), + ).toEqual([member]); + + // with a direct admin also present, removing the group member is fine + const direct = await v7(); + await core.createUser(direct, `direct_${rand(8)}@example.com`); + await core.addPrincipalToSpace(sid, direct, true); + expect(await core.removeGroupMember(sid, groupId, member)).toBe(true); +}); + +test("an empty admin group is not an effective admin (the brick is closed)", async () => { + const sid = await core.createSpace(rand(12), "Brick Space"); + const direct = await v7(); + await core.createUser(direct, `creator_${rand(8)}@example.com`); + await core.addPrincipalToSpace(sid, direct, true); // the only real admin + const emptyGroup = await core.createGroup(sid, `empties_${rand(6)}`); + await core.addPrincipalToSpace(sid, emptyGroup, true); // admin flag, no members + + // the empty admin group confers admin on nobody, so removing the direct admin + // would leave the space ungoverned — rejected. + await expectLastAdmin(core.removePrincipalFromSpace(sid, direct)); + const admins = (await core.listSpacePrincipals(sid)) + .filter((p) => p.admin) + .map((p) => p.id) + .sort(); + expect(admins).toEqual([direct, emptyGroup].sort()); +}); + +test("deleting the whole space is exempt from the guard (teardown)", async () => { + // a fresh space with a single admin — deleting the space drops the roster via + // FK cascade, which must NOT trip the last-admin guard. + const slug = rand(12); + const sid = await core.createSpace(slug, "Doomed Space"); + const admin = await v7(); + await core.createUser(admin, `doomed_${rand(8)}@example.com`); + await core.addPrincipalToSpace(sid, admin, true); + + expect(await core.deleteSpace(slug)).toBe(true); + expect(await core.getSpace(slug)).toBeNull(); +}); diff --git a/packages/engine/core/db.integration.test.ts b/packages/engine/core/db.integration.test.ts new file mode 100644 index 0000000..ea2929c --- /dev/null +++ b/packages/engine/core/db.integration.test.ts @@ -0,0 +1,150 @@ +// Integration tests for the core control-plane TS layer (coreStore). +// +// Provisions a throwaway `core_test_` schema via migrateCore and exercises +// the thin wrappers against the real SQL functions. Run with a database, e.g.: +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/engine/core/db.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateCore } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { formatApiKey, parseApiKey } from "./api-key"; +import { type CoreStore, coreStore } from "./db"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +function randomFrom(n: number): string { + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += ALPHABET[b % 36]; + return s; +} +const randomCoreSchema = () => `core_test_${randomFrom(8)}`; +const randomSlug = () => randomFrom(12); + +let sql: Sql; +let schema: string; +let db: CoreStore; + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + schema = randomCoreSchema(); + await migrateCore(sql, { schema }); + db = coreStore(sql, schema); +}); + +afterAll(async () => { + if (schema) await sql.unsafe(`drop schema if exists ${schema} cascade`); + await sql.end(); +}); + +/** A fresh uuidv7 (principal.id requires version 7, = the future auth.users.id). */ +async function newUserId(): Promise { + const [row] = await sql`select uuidv7() as id`; + return row?.id as string; +} + +test("createSpace + getSpace round-trips", async () => { + const slug = randomSlug(); + const id = await db.createSpace(slug, "My Space"); + expect(id).toBeTruthy(); + + const space = await db.getSpace(slug); + expect(space?.id).toBe(id); + expect(space?.name).toBe("My Space"); + expect(space?.language).toBe("english"); + + expect(await db.getSpace(randomSlug())).toBeNull(); +}); + +test("createUser + getPrincipal", async () => { + const userId = await newUserId(); + await db.createUser(userId, `alice_${userId.slice(0, 8)}`); + + const p = await db.getPrincipal(userId); + expect(p?.id).toBe(userId); + expect(p?.kind).toBe("u"); + expect(p?.ownerId).toBeNull(); + expect(p?.spaceId).toBeNull(); +}); + +test("grant + buildTreeAccess returns the search_memory jsonb shape", async () => { + const spaceId = await db.createSpace(randomSlug(), "S"); + const userId = await newUserId(); + await db.createUser(userId, `bob_${userId.slice(0, 8)}`); + await db.addPrincipalToSpace(spaceId, userId, true); + await db.grantTreeAccess(spaceId, userId, "work.projects", 2); + + const ta = await db.buildTreeAccess(userId, spaceId); + // addPrincipalToSpace also grants the user owner@home. + expect(ta).toContainEqual({ tree_path: "work.projects", access: 2 }); + expect(ta).toContainEqual({ + tree_path: `home.${userId.replace(/-/g, "")}`, + access: 3, + }); + expect(ta).toHaveLength(2); +}); + +test("group access flows through buildTreeAccess; removeGroupMember revokes it", async () => { + const spaceId = await db.createSpace(randomSlug(), "T"); + const userId = await newUserId(); + await db.createUser(userId, `carol_${userId.slice(0, 8)}`); + const groupId = await db.createGroup(spaceId, "eng"); + + await db.addPrincipalToSpace(spaceId, userId); + await db.addPrincipalToSpace(spaceId, groupId); + await db.addGroupMember(spaceId, groupId, userId); + await db.grantTreeAccess(spaceId, groupId, "shared", 1); + + expect(await db.buildTreeAccess(userId, spaceId)).toContainEqual({ + tree_path: "shared", + access: 1, + }); + + expect(await db.removeGroupMember(spaceId, groupId, userId)).toBe(true); + // still a space member: the group grant is gone, the user keeps its home. + expect(await db.buildTreeAccess(userId, spaceId)).toEqual([ + { tree_path: `home.${userId.replace(/-/g, "")}`, access: 3 }, + ]); +}); + +test("createApiKey + validateApiKey (good / wrong secret)", async () => { + const userId = await newUserId(); + await db.createUser(userId, `dave_${userId.slice(0, 8)}`); + + const key = await db.createApiKey(userId, "default"); + expect(key.lookupId).toMatch(/^[A-Za-z0-9_-]{16}$/); + expect(key.secret.length).toBe(32); + + const valid = await db.validateApiKey(key.lookupId, key.secret); + expect(valid?.memberId).toBe(userId); + expect(valid?.apiKeyId).toBe(key.id); + + expect(await db.validateApiKey(key.lookupId, "wrong-secret")).toBeNull(); +}); + +test("api key string format round-trips with parseApiKey", async () => { + const userId = await newUserId(); + await db.createUser(userId, `erin_${userId.slice(0, 8)}`); + const key = await db.createApiKey(userId, "fmt"); + + const str = formatApiKey(key.lookupId, key.secret); + expect(parseApiKey(str)).toEqual({ + lookupId: key.lookupId, + secret: key.secret, + }); +}); + +test("withTransaction rolls back on error", async () => { + const slug = randomSlug(); + await expect( + db.withTransaction(async (tx) => { + await tx.createSpace(slug, "Tx Space"); + throw new Error("boom"); + }), + ).rejects.toThrow("boom"); + // rolled back — the space was never committed + expect(await db.getSpace(slug)).toBeNull(); +}); diff --git a/packages/engine/core/db.ts b/packages/engine/core/db.ts new file mode 100644 index 0000000..cf2eaf2 --- /dev/null +++ b/packages/engine/core/db.ts @@ -0,0 +1,534 @@ +import { CORE_SCHEMA } from "@memory.build/database"; +import type { Sql } from "postgres"; +import { generateLookupId, generateSecret, hashApiKeySecret } from "./api-key"; +import type { + AccessLevel, + ApiKeyInfo, + CreatedApiKey, + Group, + GroupMember, + GroupMembership, + MemberSpace, + Principal, + PrincipalKind, + RedeemedInvitation, + Space, + SpaceInvitation, + SpacePrincipal, + TreeAccess, + TreeGrant, + ValidatedApiKey, +} from "./types"; + +/** + * The core control-plane data layer. + * + * Thin wrappers over the core SQL functions — every method calls a function in + * packages/database/core/migrate/idempotent/*.sql; none query core tables + * directly. Access enforcement and multi-table logic live in the SQL. + */ +export interface CoreStore { + createSpace(slug: string, name: string, language?: string): Promise; + getSpace(slug: string): Promise; + /** All spaces (e.g. for the embedding worker to discover me_ schemas). */ + listSpaces(): Promise; + /** Spaces a member belongs to (directly or via a group), with admin flag. */ + listSpacesForMember(memberId: string): Promise; + /** Rename a space (by slug). Returns true if it existed. */ + renameSpace(slug: string, name: string): Promise; + /** + * Delete a space's core row (cascades memberships/groups/grants). The + * me_ data schema must be dropped separately. Returns true if it existed. + */ + deleteSpace(slug: string): Promise; + + createUser(id: string, name: string): Promise; + createAgent(ownerId: string, name: string, id?: string): Promise; + createGroup(spaceId: string, name: string, id?: string): Promise; + getPrincipal(id: string): Promise; + /** Resolve a global user (kind 'u') by name (email). */ + getUserByName(name: string): Promise; + /** Rename an agent or group (never a user — its name is its identity email). */ + renamePrincipal(id: string, name: string): Promise; + deletePrincipal(id: string): Promise; + + /** Principals in a space — directly or via a group (each flagged `direct`). */ + listSpacePrincipals( + spaceId: string, + kind?: PrincipalKind, + ): Promise; + /** Whether a principal is an admin of a space (agents are never admins). */ + isSpaceAdmin(principalId: string, spaceId: string): Promise; + /** Whether a member is an admin of a group (agents are never group admins). */ + isGroupAdmin( + memberId: string, + groupId: string, + spaceId: string, + ): Promise; + /** Groups belonging to a space. */ + listSpaceGroups(spaceId: string): Promise; + /** A user's agents (global; agents are owned by a user, not a space). */ + listAgents(ownerId: string): Promise; + + addPrincipalToSpace( + spaceId: string, + principalId: string, + admin?: boolean, + ): Promise; + removePrincipalFromSpace( + spaceId: string, + principalId: string, + ): Promise; + addGroupMember( + spaceId: string, + groupId: string, + memberId: string, + admin?: boolean, + ): Promise; + removeGroupMember( + spaceId: string, + groupId: string, + memberId: string, + ): Promise; + /** Members (users / agents) of a group within a space. */ + listGroupMembers(spaceId: string, groupId: string): Promise; + /** Groups within a space that a member belongs to. */ + listGroupsForMember( + spaceId: string, + memberId: string, + ): Promise; + + grantTreeAccess( + spaceId: string, + principalId: string, + treePath: string, + access: AccessLevel, + ): Promise; + removeTreeAccessGrant( + spaceId: string, + principalId: string, + treePath: string, + ): Promise; + /** + * The raw grant rows in a space, optionally for a single principal and/or + * restricted to a subtree (`under`: grants at-or-below this path). Distinct + * from buildTreeAccess, which resolves a member's *effective* access set. + */ + listTreeAccessGrants( + spaceId: string, + principalId?: string, + under?: string, + ): Promise; + + /** Resolve a member's effective grants in a space (for the space functions). */ + buildTreeAccess(memberId: string, spaceId: string): Promise; + + /** Mint an api key for a member; returns the one-time plaintext secret. */ + createApiKey( + memberId: string, + name: string, + opts?: { expiresAt?: Date }, + ): Promise; + validateApiKey( + lookupId: string, + secret: string, + ): Promise; + getApiKey(id: string): Promise; + listApiKeys(memberId: string): Promise; + /** Hard-delete a key (revoke ≡ delete; there is no soft-revoke state). */ + deleteApiKey(id: string): Promise; + + /** + * Issue (or update, if one is already pending) an invitation to a space, + * keyed by invitee email — so it can be issued before the user registers. + * `shareAccess` null means no share grant. Returns the invitation id. + */ + createSpaceInvitation( + spaceId: string, + email: string, + opts: { + admin: boolean; + shareAccess: AccessLevel | null; + invitedBy: string; + }, + ): Promise; + /** Pending invitations for a space (accepted ones are history). */ + listSpaceInvitations(spaceId: string): Promise; + /** Revoke a pending invitation by email. Returns true if one was removed. */ + revokeSpaceInvitation(spaceId: string, email: string): Promise; + /** + * Redeem all pending invitations for a (now-registered, verified) email: + * join each space (owner@home), grant share access where set, mark accepted. + * Idempotent; the user must already exist as a core principal. Returns the + * spaces joined. + */ + redeemSpaceInvitations( + userId: string, + email: string, + ): Promise; + + /** Run operations atomically against the same transaction. */ + withTransaction(fn: (db: CoreStore) => Promise): Promise; +} + +function mapSpace(row: Record): Space { + return { + id: row.id as string, + slug: row.slug as string, + name: row.name as string, + language: row.language as string, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapPrincipal(row: Record): Principal { + return { + id: row.id as string, + kind: row.kind as PrincipalKind, + name: row.name as string, + ownerId: (row.owner_id as string | null) ?? null, + spaceId: (row.space_id as string | null) ?? null, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapSpacePrincipal(row: Record): SpacePrincipal { + return { + id: row.id as string, + kind: row.kind as PrincipalKind, + name: row.name as string, + ownerId: (row.owner_id as string | null) ?? null, + direct: Boolean(row.direct), + admin: Boolean(row.admin), + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapGroup(row: Record): Group { + return { + id: row.id as string, + name: row.name as string, + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapApiKeyInfo(row: Record): ApiKeyInfo { + return { + id: row.id as string, + memberId: row.member_id as string, + lookupId: row.lookup_id as string, + name: row.name as string, + createdAt: row.created_at as Date, + expiresAt: (row.expires_at as Date | null) ?? null, + }; +} + +export function coreStore(sql: Sql, schema: string = CORE_SCHEMA): CoreStore { + const sch = sql(schema); // escaped schema identifier reused across queries + + const db: CoreStore = { + async createSpace(slug, name, language) { + const [row] = await sql` + select ${sch}.create_space(${slug}, ${name}, ${language ?? null}) as id + `; + if (!row) throw new Error("create_space returned no row"); + return row.id as string; + }, + + async getSpace(slug) { + const [row] = await sql`select * from ${sch}.get_space(${slug})`; + return row ? mapSpace(row) : null; + }, + + async listSpaces() { + const rows = await sql`select * from ${sch}.list_spaces()`; + return rows.map(mapSpace); + }, + + async listSpacesForMember(memberId) { + const rows = await sql` + select * from ${sch}.list_spaces_for_member(${memberId}) + `; + return rows.map( + (r): MemberSpace => ({ ...mapSpace(r), admin: Boolean(r.admin) }), + ); + }, + + async renameSpace(slug, name) { + const [row] = + await sql`select ${sch}.rename_space(${slug}, ${name}) as ok`; + return Boolean(row?.ok); + }, + + async deleteSpace(slug) { + const [row] = await sql`select ${sch}.delete_space(${slug}) as ok`; + return Boolean(row?.ok); + }, + + async createUser(id, name) { + const [row] = await sql`select ${sch}.create_user(${id}, ${name}) as id`; + if (!row) throw new Error("create_user returned no row"); + return row.id as string; + }, + + async createAgent(ownerId, name, id) { + const [row] = await sql` + select ${sch}.create_agent(${ownerId}, ${name}, ${id ?? null}) as id + `; + if (!row) throw new Error("create_agent returned no row"); + return row.id as string; + }, + + async createGroup(spaceId, name, id) { + const [row] = await sql` + select ${sch}.create_group(${spaceId}, ${name}, ${id ?? null}) as id + `; + if (!row) throw new Error("create_group returned no row"); + return row.id as string; + }, + + async getPrincipal(id) { + const [row] = await sql`select * from ${sch}.get_principal(${id})`; + return row ? mapPrincipal(row) : null; + }, + + async getUserByName(name) { + const [row] = await sql`select * from ${sch}.get_user_by_name(${name})`; + return row ? mapPrincipal(row) : null; + }, + + async renamePrincipal(id, name) { + const [row] = await sql` + select ${sch}.rename_principal(${id}, ${name}) as ok + `; + return Boolean(row?.ok); + }, + + async deletePrincipal(id) { + const [row] = await sql`select ${sch}.delete_principal(${id}) as ok`; + return Boolean(row?.ok); + }, + + async listSpacePrincipals(spaceId, kind) { + const rows = await sql` + select * from ${sch}.list_space_principals(${spaceId}, ${kind ?? null}) + `; + return rows.map(mapSpacePrincipal); + }, + + async listSpaceGroups(spaceId) { + const rows = + await sql`select * from ${sch}.list_space_groups(${spaceId})`; + return rows.map(mapGroup); + }, + + async listAgents(ownerId) { + const rows = await sql`select * from ${sch}.list_agents(${ownerId})`; + return rows.map(mapPrincipal); + }, + + async isSpaceAdmin(principalId, spaceId) { + const [row] = await sql` + select ${sch}.is_principal_space_admin(${principalId}, ${spaceId}) as ok + `; + return Boolean(row?.ok); + }, + + async isGroupAdmin(memberId, groupId, spaceId) { + const [row] = await sql` + select ${sch}.is_group_admin(${memberId}, ${groupId}, ${spaceId}) as ok + `; + return Boolean(row?.ok); + }, + + async addPrincipalToSpace(spaceId, principalId, admin = false) { + await sql`select ${sch}.add_principal_to_space(${spaceId}, ${principalId}, ${admin})`; + }, + + async removePrincipalFromSpace(spaceId, principalId) { + const [row] = await sql` + select ${sch}.remove_principal_from_space(${spaceId}, ${principalId}) as removed + `; + return Boolean(row?.removed); + }, + + async addGroupMember(spaceId, groupId, memberId, admin = false) { + await sql`select ${sch}.add_group_member(${spaceId}, ${groupId}, ${memberId}, ${admin})`; + }, + + async removeGroupMember(spaceId, groupId, memberId) { + const [row] = await sql` + select ${sch}.remove_group_member(${spaceId}, ${groupId}, ${memberId}) as removed + `; + return Boolean(row?.removed); + }, + + async listGroupMembers(spaceId, groupId) { + const rows = await sql` + select * from ${sch}.list_group_members(${spaceId}, ${groupId}) + `; + return rows.map( + (r): GroupMember => ({ + memberId: r.member_id as string, + kind: r.kind as PrincipalKind, + name: r.name as string, + admin: Boolean(r.admin), + createdAt: r.created_at as Date, + }), + ); + }, + + async listGroupsForMember(spaceId, memberId) { + const rows = await sql` + select * from ${sch}.list_groups_for_member(${spaceId}, ${memberId}) + `; + return rows.map( + (r): GroupMembership => ({ + groupId: r.group_id as string, + name: r.name as string, + admin: Boolean(r.admin), + createdAt: r.created_at as Date, + }), + ); + }, + + async grantTreeAccess(spaceId, principalId, treePath, access) { + await sql` + select ${sch}.grant_tree_access(${spaceId}, ${principalId}, ${treePath}::ltree, ${access}) + `; + }, + + async removeTreeAccessGrant(spaceId, principalId, treePath) { + const [row] = await sql` + select ${sch}.remove_tree_access_grant(${spaceId}, ${principalId}, ${treePath}::ltree) as removed + `; + return Boolean(row?.removed); + }, + + async listTreeAccessGrants(spaceId, principalId, under) { + const rows = await sql` + select * from ${sch}.list_tree_access_grants( + ${spaceId}, ${principalId ?? null}, ${under ?? null}::ltree + ) + `; + return rows.map( + (r): TreeGrant => ({ + principalId: r.principal_id as string, + treePath: r.tree_path as string, + access: r.access as AccessLevel, + createdAt: r.created_at as Date, + updatedAt: (r.updated_at as Date | null) ?? null, + }), + ); + }, + + async buildTreeAccess(memberId, spaceId) { + const [row] = await sql` + select ${sch}.build_tree_access(${memberId}, ${spaceId}) as ta + `; + return (row?.ta as TreeAccess) ?? []; + }, + + async createApiKey(memberId, name, opts) { + const lookupId = generateLookupId(); + const secret = generateSecret(); + const secretHash = hashApiKeySecret(secret); + const [row] = await sql` + select ${sch}.create_api_key( + ${memberId}, ${lookupId}, ${secretHash}, ${name}, ${opts?.expiresAt ?? null} + ) as id + `; + if (!row) throw new Error("create_api_key returned no row"); + return { id: row.id as string, lookupId, secret }; + }, + + async validateApiKey(lookupId, secret) { + const secretHash = hashApiKeySecret(secret); + const [row] = await sql` + select member_id, api_key_id + from ${sch}.validate_api_key(${lookupId}, ${secretHash}) + `; + if (!row) return null; + return { + memberId: row.member_id as string, + apiKeyId: row.api_key_id as string, + }; + }, + + async getApiKey(id) { + const [row] = await sql`select * from ${sch}.get_api_key(${id})`; + return row ? mapApiKeyInfo(row) : null; + }, + + async listApiKeys(memberId) { + const rows = await sql`select * from ${sch}.list_api_keys(${memberId})`; + return rows.map(mapApiKeyInfo); + }, + + async deleteApiKey(id) { + const [row] = await sql`select ${sch}.delete_api_key(${id}) as ok`; + return Boolean(row?.ok); + }, + + async createSpaceInvitation(spaceId, email, opts) { + const [row] = await sql` + select ${sch}.create_space_invitation( + ${spaceId}, ${email}, ${opts.admin}, ${opts.shareAccess ?? null}, ${opts.invitedBy} + ) as id + `; + if (!row) throw new Error("create_space_invitation returned no row"); + return row.id as string; + }, + + async listSpaceInvitations(spaceId) { + const rows = await sql` + select * from ${sch}.list_space_invitations(${spaceId}) + `; + return rows.map( + (r): SpaceInvitation => ({ + id: r.id as string, + email: r.email as string, + admin: Boolean(r.admin), + shareAccess: (r.share_access as AccessLevel | null) ?? null, + invitedBy: (r.invited_by as string | null) ?? null, + invitedByName: (r.invited_by_name as string | null) ?? null, + createdAt: r.created_at as Date, + }), + ); + }, + + async revokeSpaceInvitation(spaceId, email) { + const [row] = await sql` + select ${sch}.revoke_space_invitation(${spaceId}, ${email}) as ok + `; + return Boolean(row?.ok); + }, + + async redeemSpaceInvitations(userId, email) { + const rows = await sql` + select * from ${sch}.redeem_space_invitations(${userId}, ${email}) + `; + return rows.map( + (r): RedeemedInvitation => ({ + spaceId: r.space_id as string, + slug: r.slug as string, + name: r.name as string, + admin: Boolean(r.admin), + shareAccess: (r.share_access as AccessLevel | null) ?? null, + }), + ); + }, + + async withTransaction(fn: (db: CoreStore) => Promise): Promise { + return sql.begin((tx) => + fn(coreStore(tx as unknown as Sql, schema)), + ) as Promise; + }, + }; + + return db; +} diff --git a/packages/engine/core/index.ts b/packages/engine/core/index.ts new file mode 100644 index 0000000..47956b5 --- /dev/null +++ b/packages/engine/core/index.ts @@ -0,0 +1,28 @@ +export { + formatApiKey, + generateLookupId, + generateSecret, + hashApiKeySecret, + isLegacyApiKey, + parseApiKey, +} from "./api-key"; +export { type CoreStore, coreStore } from "./db"; +export type { + AccessLevel, + ApiKeyInfo, + CreatedApiKey, + Group, + GroupMember, + GroupMembership, + MemberSpace, + Principal, + PrincipalKind, + RedeemedInvitation, + Space, + SpaceInvitation, + SpacePrincipal, + TreeAccess, + TreeGrant, + ValidatedApiKey, +} from "./types"; +export { ACCESS, ROOT_PATH } from "./types"; diff --git a/packages/engine/core/types.ts b/packages/engine/core/types.ts new file mode 100644 index 0000000..7628919 --- /dev/null +++ b/packages/engine/core/types.ts @@ -0,0 +1,163 @@ +/** + * Types for the core control-plane TS layer. + * + * This layer is intentionally thin: every method calls a core SQL function + * (see packages/database/core/migrate/idempotent/*.sql) and never queries the + * core tables directly. + */ + +export type PrincipalKind = "u" | "g" | "a"; + +/** Access levels stored in core.tree_access: 1 = read, 2 = write, 3 = owner. */ +export type AccessLevel = 1 | 2 | 3; + +/** Named tree-access levels — use instead of the raw 1/2/3. */ +export const ACCESS = { + read: 1, + write: 2, + owner: 3, +} as const satisfies Record; + +/** + * The root tree path: the empty ltree (`''`), which is the ancestor of every + * path — so a grant here covers the whole space. (ltree separates with `.`, not + * `/`, and its root is the empty path; `/` is not an ltree concept and is + * reserved for agent names like `user/agent`.) + */ +export const ROOT_PATH = ""; + +export interface Space { + id: string; + slug: string; + name: string; + language: string; + createdAt: Date; + updatedAt: Date | null; +} + +/** A space a principal belongs to, with the principal's direct-membership admin flag. */ +export interface MemberSpace extends Space { + admin: boolean; +} + +export interface Principal { + id: string; + kind: PrincipalKind; + name: string; + ownerId: string | null; + spaceId: string | null; + createdAt: Date; + updatedAt: Date | null; +} + +/** + * The effective access set for a member in a space, as produced by + * core.build_tree_access and consumed verbatim by the space data-plane + * functions (search_memory, get_memory, …). Kept in the on-the-wire snake_case + * shape because it is passed straight through to those functions as jsonb. + */ +export type TreeAccess = { tree_path: string; access: number }[]; + +export interface CreatedApiKey { + /** The api_key row id. */ + id: string; + /** The lookup id (goes in the key string, used for the indexed lookup). */ + lookupId: string; + /** The plaintext secret — returned once; only its sha256 hash is stored. */ + secret: string; +} + +export interface ValidatedApiKey { + /** The principal (user or agent) the key belongs to. */ + memberId: string; + /** The api_key row id. */ + apiKeyId: string; +} + +/** + * A principal that belongs to a space — directly or through a group. + * `direct` is true for a direct (principal_space) membership; `admin` is the + * direct-membership admin flag (false for group-only members). + */ +export interface SpacePrincipal { + id: string; + kind: PrincipalKind; + name: string; + ownerId: string | null; + direct: boolean; + admin: boolean; + createdAt: Date; + updatedAt: Date | null; +} + +/** A group (kind 'g') belonging to a space. */ +export interface Group { + id: string; + name: string; + createdAt: Date; + updatedAt: Date | null; +} + +/** A member (user / agent) of a group, with the group admin flag. */ +export interface GroupMember { + memberId: string; + kind: PrincipalKind; + name: string; + admin: boolean; + createdAt: Date; +} + +/** A group a member belongs to, with the group admin flag. */ +export interface GroupMembership { + groupId: string; + name: string; + admin: boolean; + createdAt: Date; +} + +/** A tree-access grant row. */ +export interface TreeGrant { + principalId: string; + treePath: string; + access: AccessLevel; + createdAt: Date; + updatedAt: Date | null; +} + +/** Api key metadata (never includes the secret). */ +export interface ApiKeyInfo { + id: string; + memberId: string; + lookupId: string; + name: string; + createdAt: Date; + expiresAt: Date | null; +} + +/** + * A pending invitation to a space, keyed by invitee email (so an invite can be + * issued before the user registers). Redeemed at login against the verified + * email; see CoreStore.redeemSpaceInvitations. + */ +export interface SpaceInvitation { + id: string; + email: string; + /** Make the user a space admin on redemption. */ + admin: boolean; + /** Access granted at the shared root on redemption; null = no share grant. */ + shareAccess: AccessLevel | null; + /** The principal who issued the invite (null if it has since been deleted). */ + invitedBy: string | null; + /** Display name of the inviter (a user's name is their email), if resolvable. */ + invitedByName: string | null; + createdAt: Date; +} + +/** A space joined by redeeming an invitation. */ +export interface RedeemedInvitation { + spaceId: string; + slug: string; + name: string; + admin: boolean; + shareAccess: AccessLevel | null; +} diff --git a/packages/engine/db.integration.test.ts b/packages/engine/db.integration.test.ts deleted file mode 100644 index 39eb37b..0000000 --- a/packages/engine/db.integration.test.ts +++ /dev/null @@ -1,1010 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { SQL } from "bun"; -import { createEngineDB } from "./db"; -import { bootstrap } from "./migrate/bootstrap"; -import { provisionEngine } from "./migrate/provision"; -import { TestDatabase } from "./migrate/test-utils"; - -const testDb = new TestDatabase(); -let connectionString: string; -let sql: SQL; -const schema = "me_testengine01"; - -beforeAll(async () => { - connectionString = await testDb.create(); - sql = new SQL(connectionString); - await bootstrap(sql); - await provisionEngine(sql, "testengine01", undefined, "0.1.0"); -}); - -afterAll(async () => { - await sql.close(); - await testDb.drop(); -}); - -// --------------------------------------------------------------------------- -// Principal Tests -// --------------------------------------------------------------------------- -describe("user ops", () => { - test("createUser creates a principal", async () => { - const db = createEngineDB(sql, schema); - const user = await db.createUser({ - name: "test-user", - }); - - expect(user.name).toBe("test-user"); - expect(user.superuser).toBe(false); - expect(user.id).toBeDefined(); - expect(user.createdAt).toBeInstanceOf(Date); - }); - - test("createUser with custom id", async () => { - const db = createEngineDB(sql, schema); - // Generate a UUIDv7 for testing - const customId = crypto.randomUUID(); - // Replace version nibble with 7 to make it UUIDv7 compatible - const uuidv7Id = customId.replace( - /^(.{8}-.{4}-)(.)/, - (_, prefix) => `${prefix}7`, - ); - - const user = await db.createUser({ - id: uuidv7Id, - name: "test-user-with-id", - }); - - expect(user.id).toBe(uuidv7Id); - expect(user.name).toBe("test-user-with-id"); - }); - - test("createSuperuser creates a superuser principal", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("admin"); - - expect(superuser.name).toBe("admin"); - expect(superuser.superuser).toBe(true); - }); - - test("createSuperuser with custom id", async () => { - const db = createEngineDB(sql, schema); - const customId = crypto - .randomUUID() - .replace(/^(.{8}-.{4}-)(.)/, (_, prefix) => `${prefix}7`); - const superuser = await db.createSuperuser("admin-with-id", customId); - - expect(superuser.id).toBe(customId); - expect(superuser.name).toBe("admin-with-id"); - expect(superuser.superuser).toBe(true); - }); - - test("createUser with identityId and canLogin", async () => { - const db = createEngineDB(sql, schema); - const identityId = crypto - .randomUUID() - .replace(/^(.{8}-.{4}-)(.)/, (_, prefix) => `${prefix}7`); - - const user = await db.createUser({ - name: "owned-user", - identityId: identityId, - canLogin: true, - }); - - expect(user.name).toBe("owned-user"); - expect(user.identityId).toBe(identityId); - expect(user.canLogin).toBe(true); - }); - - test("createRole creates a user with canLogin=false", async () => { - const db = createEngineDB(sql, schema); - const role = await db.createRole("test-role"); - - expect(role.name).toBe("test-role"); - expect(role.canLogin).toBe(false); - expect(role.superuser).toBe(false); - }); - - test("getUser returns user by ID", async () => { - const db = createEngineDB(sql, schema); - const created = await db.createUser({ name: "get-by-id-test" }); - const fetched = await db.getUser(created.id); - - expect(fetched).not.toBeNull(); - expect(fetched!.id).toBe(created.id); - expect(fetched!.name).toBe("get-by-id-test"); - }); - - test("getUser returns null for non-existent ID", async () => { - const db = createEngineDB(sql, schema); - const fetched = await db.getUser("00000000-0000-0000-0000-000000000000"); - - expect(fetched).toBeNull(); - }); - - test("getUserByName returns principal by name", async () => { - const db = createEngineDB(sql, schema); - await db.createUser({ name: "get-by-name-test" }); - const fetched = await db.getUserByName("get-by-name-test"); - - expect(fetched).not.toBeNull(); - expect(fetched!.name).toBe("get-by-name-test"); - }); - - test("getUserByName matches case-insensitively (citext)", async () => { - const db = createEngineDB(sql, schema); - const uniqueName = `ExactMatch_${Date.now()}`; - await db.createUser({ name: uniqueName }); - const fetched = await db.getUserByName(uniqueName); - - expect(fetched).not.toBeNull(); - expect(fetched!.name).toBe(uniqueName); - - // citext: different case should still match - const alsoFound = await db.getUserByName(uniqueName.toLowerCase()); - expect(alsoFound).not.toBeNull(); - expect(alsoFound!.id).toBe(fetched!.id); - }); - - test("listUsers returns all principals", async () => { - const db = createEngineDB(sql, schema); - const principals = await db.listUsers(); - - expect(principals.length).toBeGreaterThan(0); - expect(principals[0]!.id).toBeDefined(); - }); - - test("renameUser updates name", async () => { - const db = createEngineDB(sql, schema); - const created = await db.createUser({ name: "rename-test" }); - const result = await db.renameUser(created.id, "renamed-test"); - - expect(result).toBe(true); - - const fetched = await db.getUser(created.id); - expect(fetched!.name).toBe("renamed-test"); - }); - - test("deleteUser removes principal", async () => { - const db = createEngineDB(sql, schema); - const created = await db.createUser({ name: "delete-test" }); - const result = await db.deleteUser(created.id); - - expect(result).toBe(true); - - const fetched = await db.getUser(created.id); - expect(fetched).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// Grant Tests -// --------------------------------------------------------------------------- -describe("grant ops", () => { - let testPrincipalId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - const user = await db.createUser({ name: "grant-test-user" }); - testPrincipalId = user.id; - }); - - test("grantTreeAccess creates a grant", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "test.path", - actions: ["read", "create"], - }); - - const grant = await db.getTreeGrant(testPrincipalId, "test.path"); - expect(grant).not.toBeNull(); - expect(grant!.userId).toBe(testPrincipalId); - expect(grant!.treePath).toBe("test.path"); - expect(grant!.actions).toContain("read"); - expect(grant!.actions).toContain("create"); - }); - - test("grantTreeAccess upserts on conflict", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "upsert.path", - actions: ["read"], - }); - - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "upsert.path", - actions: ["read", "create", "update"], - }); - - const grant = await db.getTreeGrant(testPrincipalId, "upsert.path"); - expect(grant!.actions).toHaveLength(3); - }); - - test("revokeTreeAccess removes grant", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "revoke.path", - actions: ["read"], - }); - - const result = await db.revokeTreeAccess(testPrincipalId, "revoke.path"); - expect(result).toBe(true); - - const grant = await db.getTreeGrant(testPrincipalId, "revoke.path"); - expect(grant).toBeNull(); - }); - - test("listTreeGrants returns grants for principal", async () => { - const db = createEngineDB(sql, schema); - await db.grantTreeAccess({ - userId: testPrincipalId, - treePath: "list.path", - actions: ["read"], - }); - - const grants = await db.listTreeGrants(testPrincipalId); - expect(grants.length).toBeGreaterThan(0); - }); - - test("checkTreeAccess uses has_tree_access function", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("access-check-admin"); - - // Superuser should have access to everything - const hasAccess = await db.checkTreeAccess( - superuser.id, - "any.path", - "read", - ); - expect(hasAccess).toBe(true); - }); -}); - -// --------------------------------------------------------------------------- -// Owner Tests -// --------------------------------------------------------------------------- -describe("owner ops", () => { - let testPrincipalId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - const user = await db.createUser({ name: "owner-test-user" }); - testPrincipalId = user.id; - }); - - test("setTreeOwner creates ownership", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "owned.path"); - - const owner = await db.getTreeOwner("owned.path"); - expect(owner).not.toBeNull(); - expect(owner!.userId).toBe(testPrincipalId); - expect(owner!.treePath).toBe("owned.path"); - }); - - test("setTreeOwner upserts on conflict", async () => { - const db = createEngineDB(sql, schema); - const otherPrincipal = await db.createUser({ name: "other-owner" }); - - await db.setTreeOwner(testPrincipalId, "upsert.owned"); - await db.setTreeOwner(otherPrincipal.id, "upsert.owned"); - - const owner = await db.getTreeOwner("upsert.owned"); - expect(owner!.userId).toBe(otherPrincipal.id); - }); - - test("removeTreeOwner removes ownership", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "remove.owned"); - const result = await db.removeTreeOwner("remove.owned"); - - expect(result).toBe(true); - - const owner = await db.getTreeOwner("remove.owned"); - expect(owner).toBeNull(); - }); - - test("listTreeOwners returns owners for principal", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "list.owned"); - - const owners = await db.listTreeOwners(testPrincipalId); - expect(owners.length).toBeGreaterThan(0); - }); - - test("isOwnerOf checks ownership", async () => { - const db = createEngineDB(sql, schema); - await db.setTreeOwner(testPrincipalId, "isowner.path"); - - const isOwner = await db.isOwnerOf(testPrincipalId, "isowner.path.child"); - expect(isOwner).toBe(true); - - const isNotOwner = await db.isOwnerOf(testPrincipalId, "other.path"); - expect(isNotOwner).toBe(false); - }); -}); - -// --------------------------------------------------------------------------- -// Role Tests -// --------------------------------------------------------------------------- -describe("role ops", () => { - let roleId: string; - let memberId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - // Create a role (a principal used as a role for grouping) - const role = await db.createUser({ - name: "membership-role", - }); - roleId = role.id; - - const member = await db.createUser({ name: "role-member" }); - memberId = member.id; - }); - - test("addRoleMember adds member to role", async () => { - const db = createEngineDB(sql, schema); - await db.addRoleMember(roleId, memberId); - - const members = await db.listRoleMembers(roleId); - expect(members.length).toBeGreaterThan(0); - expect(members.some((m) => m.memberId === memberId)).toBe(true); - }); - - test("addRoleMember detects cycles", async () => { - const db = createEngineDB(sql, schema); - const role1 = await db.createUser({ - name: "cycle-role-1", - }); - const role2 = await db.createUser({ - name: "cycle-role-2", - }); - - await db.addRoleMember(role1.id, role2.id); - - // Try to create cycle: role2 -> role1 (but role1 -> role2 exists) - await expect(db.addRoleMember(role2.id, role1.id)).rejects.toThrow( - "would create a cycle", - ); - }); - - test("removeRoleMember removes member from role", async () => { - const db = createEngineDB(sql, schema); - const role = await db.createUser({ - name: "remove-role", - }); - const member = await db.createUser({ name: "remove-member" }); - - await db.addRoleMember(role.id, member.id); - const result = await db.removeRoleMember(role.id, member.id); - - expect(result).toBe(true); - - const members = await db.listRoleMembers(role.id); - expect(members.some((m) => m.memberId === member.id)).toBe(false); - }); - - test("listRolesForUser returns roles", async () => { - const db = createEngineDB(sql, schema); - const roles = await db.listRolesForUser(memberId); - - expect(roles.some((r) => r.id === roleId)).toBe(true); - }); - - test("hasAdminOption checks admin option", async () => { - const db = createEngineDB(sql, schema); - const role = await db.createUser({ - name: "admin-role", - }); - const admin = await db.createUser({ name: "admin-member" }); - - await db.addRoleMember(role.id, admin.id, true); - - const hasAdmin = await db.hasAdminOption(admin.id, role.id); - expect(hasAdmin).toBe(true); - }); -}); - -// --------------------------------------------------------------------------- -// Memory Tests -// --------------------------------------------------------------------------- -describe("memory ops", () => { - let testPrincipalId: string; - - beforeAll(async () => { - const db = createEngineDB(sql, schema); - // Create a superuser for memory tests (bypasses RLS) - const superuser = await db.createSuperuser("memory-test-admin"); - testPrincipalId = superuser.id; - }); - - test("createMemory creates a memory", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const memory = await db.createMemory({ - content: "Test memory content", - meta: { key: "value" }, - tree: "test.memories", - }); - - expect(memory.id).toBeDefined(); - expect(memory.content).toBe("Test memory content"); - expect(memory.meta).toEqual({ key: "value" }); - expect(memory.tree).toBe("test.memories"); - expect(memory.hasEmbedding).toBe(false); - expect(memory.createdAt).toBeInstanceOf(Date); - }); - - test("createMemory with temporal point-in-time", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const now = new Date(); - const memory = await db.createMemory({ - content: "Point in time memory", - temporal: { start: now }, - }); - - expect(memory.temporal).not.toBeNull(); - expect(memory.temporal!.start.getTime()).toBe( - memory.temporal!.end.getTime(), - ); - }); - - test("createMemory with temporal range", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const start = new Date("2024-01-01"); - const end = new Date("2024-01-02"); - const memory = await db.createMemory({ - content: "Range memory", - temporal: { start, end }, - }); - - expect(memory.temporal).not.toBeNull(); - expect(memory.temporal!.start.getTime()).toBe(start.getTime()); - }); - - test("getMemory returns memory by ID", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ content: "Get test" }); - const fetched = await db.getMemory(created.id); - - expect(fetched).not.toBeNull(); - expect(fetched!.id).toBe(created.id); - expect(fetched!.content).toBe("Get test"); - }); - - test("updateMemory updates content", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ content: "Original" }); - const updated = await db.updateMemory(created.id, { content: "Updated" }); - - expect(updated).not.toBeNull(); - expect(updated!.content).toBe("Updated"); - }); - - test("updateMemory updates meta", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ - content: "Meta test", - meta: { old: true }, - }); - const updated = await db.updateMemory(created.id, { meta: { new: true } }); - - expect(updated!.meta).toEqual({ new: true }); - }); - - test("deleteMemory removes memory", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const created = await db.createMemory({ content: "Delete test" }); - const result = await db.deleteMemory(created.id); - - expect(result).toBe(true); - - const fetched = await db.getMemory(created.id); - expect(fetched).toBeNull(); - }); - - test("deleteTree removes memories under path", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ content: "Tree 1", tree: "delete.tree.a" }); - await db.createMemory({ content: "Tree 2", tree: "delete.tree.b" }); - await db.createMemory({ content: "Other", tree: "other.tree" }); - - const result = await db.deleteTree("delete.tree"); - - expect(result.count).toBe(2); - }); - - test("moveTree moves memories to new path", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const m1 = await db.createMemory({ - content: "Move 1", - tree: "move.source", - }); - const m2 = await db.createMemory({ - content: "Move 2", - tree: "move.source.child", - }); - - const result = await db.moveTree("move.source", "move.destination"); - - expect(result.count).toBe(2); - - const fetched1 = await db.getMemory(m1.id); - expect(fetched1!.tree).toBe("move.destination"); - - const fetched2 = await db.getMemory(m2.id); - expect(fetched2!.tree).toBe("move.destination.child"); - }); - - test("moveTree dry-run counts without moving", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const m1 = await db.createMemory({ - content: "DryMove 1", - tree: "drymove.source", - }); - const m2 = await db.createMemory({ - content: "DryMove 2", - tree: "drymove.source.child", - }); - - // Dry-run preview uses countTree (same as RPC handler) - const preview = await db.countTree("drymove.source"); - expect(preview.count).toBe(2); - - // Verify memories were NOT moved - const fetched1 = await db.getMemory(m1.id); - expect(fetched1!.tree).toBe("drymove.source"); - - const fetched2 = await db.getMemory(m2.id); - expect(fetched2!.tree).toBe("drymove.source.child"); - }); - - test("countTree returns accurate count above 1000 (TNT-59 regression)", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Insert 1500 memories under a tree path. The bug capped the dry-run - // count at 1000 because the handler used searchMemories with limit:1000. - const total = 1500; - const batchSize = 500; - for (let start = 0; start < total; start += batchSize) { - const batch = Array.from({ length: batchSize }, (_, i) => ({ - content: `Bulk ${start + i}`, - tree: "bulk.count.regression", - })); - await db.batchCreateMemories(batch); - } - - // Sanity: searchMemories with limit:1000 (the old, buggy preview) caps at 1000. - const cappedPreview = await db.searchMemories({ - tree: "bulk.count.regression", - limit: 1000, - }); - expect(cappedPreview.total).toBe(1000); - - // countTree returns the true count, unbounded. - const count = await db.countTree("bulk.count.regression"); - expect(count.count).toBe(total); - - // Cleanup so subsequent tree-related tests aren't affected. - const deleted = await db.deleteTree("bulk.count.regression"); - expect(deleted.count).toBe(total); - }); - - test("countTree includes descendants and is empty for unknown paths", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ content: "A", tree: "count.tree" }); - await db.createMemory({ content: "B", tree: "count.tree.child" }); - await db.createMemory({ content: "C", tree: "count.tree.child.deep" }); - await db.createMemory({ content: "D", tree: "count.other" }); - - const inside = await db.countTree("count.tree"); - expect(inside.count).toBe(3); - - const empty = await db.countTree("count.does.not.exist"); - expect(empty.count).toBe(0); - }); - - test("batchCreateMemories creates multiple memories", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const ids = await db.batchCreateMemories([ - { content: "Batch 1", tree: "batch" }, - { content: "Batch 2", tree: "batch" }, - { content: "Batch 3", tree: "batch" }, - ]); - - expect(ids).toHaveLength(3); - - for (const id of ids) { - const memory = await db.getMemory(id); - expect(memory).not.toBeNull(); - } - }); - - test("batchCreateMemories skips duplicate ids without rolling back", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Two distinct UUIDv7s plus a duplicate of the first. The duplicate - // would otherwise hit the memory_pkey unique constraint and abort the - // transaction, taking the unique siblings down with it. - const idA = "019ddff0-0000-7000-8000-000000000a01"; - const idB = "019ddff0-0000-7000-8000-000000000b01"; - - const ids = await db.batchCreateMemories([ - { id: idA, content: "Dup batch A", tree: "dup.batch" }, - { id: idA, content: "Dup batch A redux", tree: "dup.batch" }, - { id: idB, content: "Dup batch B", tree: "dup.batch" }, - ]); - - // Only the unique inserts are returned — the duplicate is silently dropped. - expect(ids).toEqual([idA, idB]); - - // Both unique rows landed; the first content wins (dup is skipped, not updated). - const a = await db.getMemory(idA); - expect(a?.content).toBe("Dup batch A"); - const b = await db.getMemory(idB); - expect(b?.content).toBe("Dup batch B"); - }); - - test("batchCreateMemories tolerates ids that already exist in the table", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const id = "019ddff0-0000-7000-8000-000000000c01"; - await db.batchCreateMemories([ - { id, content: "Existing row", tree: "dup.preexisting" }, - ]); - - // Re-submitting a batch that includes the existing id should succeed, - // returning only the genuinely new ids and leaving the existing row's - // content untouched. - const newId = "019ddff0-0000-7000-8000-000000000c02"; - const ids = await db.batchCreateMemories([ - { id, content: "Re-attempted insert", tree: "dup.preexisting" }, - { - id: newId, - content: "Sibling that should land", - tree: "dup.preexisting", - }, - ]); - - expect(ids).toEqual([newId]); - const original = await db.getMemory(id); - expect(original?.content).toBe("Existing row"); - }); - - test("getTree returns tree structure", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ content: "Tree test 1", tree: "gettree.a.b" }); - await db.createMemory({ content: "Tree test 2", tree: "gettree.a.c" }); - - const tree = await db.getTree({ tree: "gettree" }); - - expect(tree.length).toBeGreaterThan(0); - expect(tree.some((n) => n.path === "gettree.a")).toBe(true); - }); - - test("searchMemories with filter-only returns results", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Create test memories - await db.createMemory({ - content: "Filter search test 1", - tree: "search.filter", - meta: { type: "test" }, - }); - await db.createMemory({ - content: "Filter search test 2", - tree: "search.filter", - meta: { type: "test" }, - }); - - const result = await db.searchMemories({ - tree: "search.filter", - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(2); - expect(result.results[0]!.score).toBe(1.0); // Filter-only uses score 1.0 - }); - - test("searchMemories with meta filter", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "Meta filter test", - tree: "search.meta", - meta: { category: "important", priority: 1 }, - }); - await db.createMemory({ - content: "Meta filter other", - tree: "search.meta", - meta: { category: "other" }, - }); - - const result = await db.searchMemories({ - meta: { category: "important" }, - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(1); - expect(result.results.some((r) => r.content === "Meta filter test")).toBe( - true, - ); - }); - - test("searchMemories with fulltext (BM25)", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "PostgreSQL is a powerful relational database", - tree: "search.bm25", - }); - await db.createMemory({ - content: "Redis is an in-memory key-value store", - tree: "search.bm25", - }); - - const result = await db.searchMemories({ - fulltext: "PostgreSQL database", - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(1); - expect(result.results[0]!.content).toContain("PostgreSQL"); - expect(result.results[0]!.score).toBeGreaterThan(0); - }); - - test("searchMemories with tree pattern (lquery)", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "Lquery test a.b", - tree: "lquery.a.b", - }); - await db.createMemory({ - content: "Lquery test a.c", - tree: "lquery.a.c", - }); - await db.createMemory({ - content: "Lquery test other", - tree: "other.path", - }); - - const result = await db.searchMemories({ - tree: "lquery.*", - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(2); - expect(result.results.every((r) => r.tree.startsWith("lquery"))).toBe(true); - }); - - test("searchMemories with temporal contains filter", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - const jan1 = new Date("2024-01-01"); - const jan15 = new Date("2024-01-15"); - const feb1 = new Date("2024-02-01"); - - await db.createMemory({ - content: "January event", - tree: "search.temporal", - temporal: { start: jan1, end: feb1 }, - }); - await db.createMemory({ - content: "Point in time event", - tree: "search.temporal", - temporal: { start: jan15 }, - }); - - // Search for events containing Jan 10 - const result = await db.searchMemories({ - temporal: { contains: new Date("2024-01-10") }, - limit: 10, - }); - - expect(result.results.length).toBeGreaterThanOrEqual(1); - expect(result.results.some((r) => r.content === "January event")).toBe( - true, - ); - }); - - test("searchMemories orderBy asc/desc", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - // Create memories with slight delay to ensure different timestamps - const m1 = await db.createMemory({ - content: "Order test first", - tree: "search.order", - }); - const m2 = await db.createMemory({ - content: "Order test second", - tree: "search.order", - }); - - // Descending (default) - newest first - const descResult = await db.searchMemories({ - tree: "search.order", - orderBy: "desc", - limit: 10, - }); - expect(descResult.results[0]!.id).toBe(m2.id); - - // Ascending - oldest first - const ascResult = await db.searchMemories({ - tree: "search.order", - orderBy: "asc", - limit: 10, - }); - expect(ascResult.results[0]!.id).toBe(m1.id); - }); - - test("searchMemories with grep filter", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "Error code ERR-42 occurred in production", - tree: "search.grep", - }); - await db.createMemory({ - content: "Warning code WARN-7 in staging", - tree: "search.grep", - }); - await db.createMemory({ - content: "All systems operational", - tree: "search.grep", - }); - - // Regex matching "ERR-\d+" should only return the first memory - const result = await db.searchMemories({ - grep: "ERR-\\d+", - tree: "search.grep", - limit: 10, - }); - - expect(result.results.length).toBe(1); - expect(result.results[0]!.content).toContain("ERR-42"); - }); - - test("searchMemories with grep + fulltext", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "PostgreSQL version 18.1 released with new features", - tree: "search.grepfull", - }); - await db.createMemory({ - content: "PostgreSQL conference announced for next year", - tree: "search.grepfull", - }); - - // BM25 matches both on "PostgreSQL", but grep narrows to version pattern - const result = await db.searchMemories({ - fulltext: "PostgreSQL", - grep: "version \\d+\\.\\d+", - limit: 10, - }); - - expect(result.results.length).toBe(1); - expect(result.results[0]!.content).toContain("version 18.1"); - }); - - test("searchMemories grep is case-insensitive", async () => { - const db = createEngineDB(sql, schema); - db.setUser(testPrincipalId); - - await db.createMemory({ - content: "TypeScript is great", - tree: "search.grepcase", - }); - await db.createMemory({ - content: "typescript lowercase", - tree: "search.grepcase", - }); - - // Case-insensitive: matches both "TypeScript" and "typescript" - const result = await db.searchMemories({ - grep: "TypeScript", - tree: "search.grepcase", - limit: 10, - }); - - expect(result.results.length).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// Transaction Tests -// --------------------------------------------------------------------------- -describe("withTransaction", () => { - test("executes multiple ops atomically", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("tx-test-admin"); - db.setUser(superuser.id); - - const result = await db.withTransaction("write", async (txDb) => { - const m1 = await txDb.createMemory({ - content: "TX Memory 1", - tree: "tx", - }); - const m2 = await txDb.createMemory({ - content: "TX Memory 2", - tree: "tx", - }); - return [m1.id, m2.id]; - }); - - expect(result).toHaveLength(2); - - // Verify both were created - for (const id of result) { - const memory = await db.getMemory(id); - expect(memory).not.toBeNull(); - } - }); - - test("rolls back on error", async () => { - const db = createEngineDB(sql, schema); - const superuser = await db.createSuperuser("rollback-test-admin"); - db.setUser(superuser.id); - - let createdId: string | null = null; - - try { - await db.withTransaction("write", async (txDb) => { - const m = await txDb.createMemory({ - content: "Rollback test", - tree: "rollback", - }); - createdId = m.id; - throw new Error("Intentional error"); - }); - } catch { - // Expected - } - - // Memory should not exist (rolled back) - if (createdId) { - const memory = await db.getMemory(createdId); - expect(memory).toBeNull(); - } - }); -}); diff --git a/packages/engine/db.ts b/packages/engine/db.ts deleted file mode 100644 index 816fe8b..0000000 --- a/packages/engine/db.ts +++ /dev/null @@ -1,183 +0,0 @@ -import type { SQL } from "bun"; -import { deriveContext, setLocalEngineTimeouts } from "./ops/_tx"; -import { type ApiKeyOps, apiKeyOps } from "./ops/api-key"; -import { type GrantOps, grantOps } from "./ops/grant"; -import { type MemoryOps, memoryOps } from "./ops/memory"; -import { type OwnerOps, ownerOps } from "./ops/owner"; -import { type RoleOps, roleOps } from "./ops/role"; -import { type UserOps, userOps } from "./ops/user"; -import type { OpsContext } from "./types"; - -export interface CreateEngineDBOptions { - /** Shard number for pgDog routing (future use) */ - shard?: number; -} - -/** - * All ops combined - */ -type AllOps = UserOps & ApiKeyOps & GrantOps & OwnerOps & RoleOps & MemoryOps; - -/** - * EngineDB interface - explicit type to avoid circular reference issues - */ -export interface EngineDB extends AllOps { - setUser(id: string): void; - getUserId(): string | null; - getSchema(): string; - getEngineSlug(): string; - withTransaction( - mode: "read" | "write", - fn: (db: EngineDB) => Promise, - ): Promise; -} - -/** - * Compose all ops into a single object - */ -function composeOps(ctx: OpsContext, engineSlug: string): AllOps { - return { - ...userOps(ctx), - ...apiKeyOps(ctx, engineSlug), - ...grantOps(ctx), - ...ownerOps(ctx), - ...roleOps(ctx), - ...memoryOps(ctx), - }; -} - -/** - * Extract engine slug from schema name (e.g., "me_abc123xyz789" -> "abc123xyz789") - */ -function extractSlugFromSchema(schema: string): string { - if (schema.startsWith("me_")) { - return schema.slice(3); - } - throw new Error(`Invalid schema name: ${schema} (must start with "me_")`); -} - -/** - * Create an EngineDB instance for a specific engine schema. - * - * EngineDB is the database abstraction layer for a single memory engine. - * It encapsulates all database operations and handles transaction management, - * role-based access control, and RLS context setup. - * - * @param sql - Database connection pool - * @param schema - Engine schema name (e.g., "me_abc123xyz789") - * @param options - Optional configuration (shard number for future pgDog routing) - */ -export function createEngineDB( - sql: SQL, - schema: string, - options?: CreateEngineDBOptions, -): EngineDB { - let userId: string | null = null; - const engineSlug = extractSlugFromSchema(schema); - - const ctx: OpsContext = { - sql, - schema, - shard: options?.shard, - inTransaction: false, - getUserId: () => userId, - }; - - const ops = composeOps(ctx, engineSlug); - - const db: EngineDB = { - ...ops, - - /** - * Set the current user ID for RLS context. - * This should be called after authentication, before making database calls. - */ - setUser(id: string): void { - userId = id; - }, - - /** - * Get the current user ID - */ - getUserId(): string | null { - return userId; - }, - - /** - * Get the schema name for this engine - */ - getSchema(): string { - return schema; - }, - - /** - * Get the engine slug (for API key generation) - */ - getEngineSlug(): string { - return engineSlug; - }, - - /** - * Execute multiple operations within a single transaction. - * - * Use this for batch operations that need to be atomic. - * Each operation inside the transaction will use the appropriate role - * (me_ro for reads, me_rw for writes). - * - * @param mode - "read" for read-only transaction, "write" for read-write - * @param fn - Function receiving a transactional EngineDB instance - */ - async withTransaction( - mode: "read" | "write", - fn: (db: EngineDB) => Promise, - ): Promise { - const role = mode === "read" ? "me_ro" : "me_rw"; - - return sql.begin(async (tx) => { - // Set up transaction context - if (ctx.shard !== undefined) { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${ctx.shard}`); - } - await setLocalEngineTimeouts(tx); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe(`SET LOCAL ROLE ${role}`); - if (userId) { - await tx`SELECT set_config('me.user_id', ${userId}, true)`; - } - - // Create a derived context for the transaction - const txCtx = deriveContext(ctx, tx); - const txOps = composeOps(txCtx, engineSlug); - - // Create a transactional EngineDB instance - const txDb: EngineDB = { - ...txOps, - setUser(id: string): void { - userId = id; - }, - getUserId(): string | null { - return userId; - }, - getSchema(): string { - return schema; - }, - getEngineSlug(): string { - return engineSlug; - }, - // Nested withTransaction just runs the function directly - // (already in a transaction) - async withTransaction( - _mode: "read" | "write", - nestedFn: (db: EngineDB) => Promise, - ): Promise { - return nestedFn(txDb); - }, - }; - - return fn(txDb); - }); - }, - }; - - return db; -} diff --git a/packages/engine/index.ts b/packages/engine/index.ts index e16b89c..a693619 100644 --- a/packages/engine/index.ts +++ b/packages/engine/index.ts @@ -1,40 +1,8 @@ -// Main exports -export { - type CreateEngineDBOptions, - createEngineDB, - type EngineDB, -} from "./db"; -// Re-export migrate module -export * from "./migrate"; -// Type exports -export { - type ApiKey, - type CreateApiKeyParams, - type CreateApiKeyResult, - type CreateMemoryParams, - type CreateUserParams, - type GetTreeParams, - type GrantTreeAccessParams, - type Memory, - NotImplementedError, - type OpsContext, - type RoleInfo, - type RoleMember, - type SearchParams, - type SearchResult, - type SearchResultItem, - type SearchWeights, - type TemporalFilter, - type TreeGrant, - type TreeNode, - type TreeOwner, - type UpdateMemoryParams, - type User, - type ValidateApiKeyResult, -} from "./types"; -// Utility exports -export { - extractEngineSlug, - formatApiKey, - parseApiKey, -} from "./util"; +// The engine package is the runtime layer over the new-model schemas: +// - core: control plane (core schema) — spaces, principals, membership, +// groups, tree-access grants, api keys. +// - space: data plane (per-space me_ schema) — memory CRUD, tree, search. +// Namespaced so callers pick a plane explicitly: `core.coreStore`, `space.spaceStore`. +// Subpath imports (`@memory.build/engine/core`, `/space`) are equivalent. +export * as core from "./core"; +export * as space from "./space"; diff --git a/packages/engine/migrate/bootstrap.ts b/packages/engine/migrate/bootstrap.ts deleted file mode 100644 index e3b9805..0000000 --- a/packages/engine/migrate/bootstrap.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { SQL, semver } from "bun"; -import { setLocalEngineTimeouts } from "../ops/_tx"; - -export async function bootstrap(sql: SQL): Promise { - await sql.begin(async (tx) => { - await setLocalEngineTimeouts(tx); - await ensurePrerequisites(tx); - await ensureRoles(tx); - }); -} - -async function ensurePrerequisites(sql: SQL): Promise { - const [{ server_version_num }] = await sql` - select current_setting('server_version_num')::int as server_version_num - `; - if (server_version_num < 180000) { - throw new Error( - `PostgreSQL version 18 or higher is required (found ${server_version_num})`, - ); - } - - await ensureExtension(sql, "citext", "1.6"); - await ensureExtension(sql, "ltree", "1.3"); - await ensureExtension(sql, "vector", "0.8.2"); - await ensureExtension(sql, "pg_textsearch", "1.1.0"); -} - -async function ensureExtension( - sql: SQL, - name: string, - minVersion: string, -): Promise { - const [installed] = await sql` - select extversion from pg_extension where extname = ${name} - `; - - if (installed) { - if (semver.order(installed.extversion, minVersion) >= 0) { - return; - } - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required (found ${installed.extversion} installed)`, - ); - } - - const [available] = await sql` - select default_version - from pg_available_extensions - where name = ${name} - `; - - if (!available || semver.order(available.default_version, minVersion) < 0) { - const found = available - ? `found ${available.default_version} available` - : "not available"; - throw new Error( - `Extension "${name}" version ${minVersion} or higher is required (${found})`, - ); - } - - try { - await sql`create extension if not exists ${sql(name)}`; - } catch (error: unknown) { - // Ignore duplicate extension errors (race condition in concurrent calls) - if ( - error instanceof SQL.PostgresError && - error.errno === "23505" && - error.constraint === "pg_extension_name_index" - ) { - return; - } - throw error; - } -} - -async function ensureRoles(sql: SQL): Promise { - await sql.unsafe(` - do $block$ - declare - _roles text[] = array['me_ro', 'me_rw', 'me_embed']; - _role text; - _sql text; - begin - for _role in select * from unnest(_roles) loop - perform - from pg_roles r - where r.rolname = _role; - if found then - continue; - end if; - _sql = format($sql$create role %I nologin$sql$, _role); - execute _sql; - _sql = format($sql$grant %I to %I$sql$, _role, current_user); - execute _sql; - end loop; - end; - $block$; - `); -} diff --git a/packages/engine/migrate/discover.test.ts b/packages/engine/migrate/discover.test.ts deleted file mode 100644 index 8f9233c..0000000 --- a/packages/engine/migrate/discover.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { - isValidEngineSchema, - isValidSlug, - schemaToSlug, - slugToSchema, -} from "./discover"; - -describe("isValidEngineSchema", () => { - test("valid 12-char lowercase alphanumeric", () => { - expect(isValidEngineSchema("me_abcdef123456")).toBe(true); - }); - - test("valid all digits", () => { - expect(isValidEngineSchema("me_000000000000")).toBe(true); - }); - - test("valid all letters", () => { - expect(isValidEngineSchema("me_abcdefghijkl")).toBe(true); - }); - - test("rejects too short", () => { - expect(isValidEngineSchema("me_abc")).toBe(false); - }); - - test("rejects too long", () => { - expect(isValidEngineSchema("me_abcdef1234567")).toBe(false); - }); - - test("rejects uppercase", () => { - expect(isValidEngineSchema("me_ABCDEF123456")).toBe(false); - }); - - test("rejects wrong prefix", () => { - expect(isValidEngineSchema("xx_abcdef123456")).toBe(false); - }); - - test("rejects no prefix", () => { - expect(isValidEngineSchema("abcdef123456")).toBe(false); - }); - - test("rejects empty string", () => { - expect(isValidEngineSchema("")).toBe(false); - }); - - test("rejects special characters", () => { - expect(isValidEngineSchema("me_abcdef12345!")).toBe(false); - }); - - test("rejects public schema", () => { - expect(isValidEngineSchema("public")).toBe(false); - }); - - test("rejects embedding schema", () => { - expect(isValidEngineSchema("embedding")).toBe(false); - }); -}); - -describe("isValidSlug", () => { - test("valid 12-char lowercase alphanumeric", () => { - expect(isValidSlug("abcdef123456")).toBe(true); - }); - - test("valid all digits", () => { - expect(isValidSlug("000000000000")).toBe(true); - }); - - test("rejects too short", () => { - expect(isValidSlug("abc")).toBe(false); - }); - - test("rejects too long", () => { - expect(isValidSlug("abcdef1234567")).toBe(false); - }); - - test("rejects uppercase", () => { - expect(isValidSlug("ABCDEF123456")).toBe(false); - }); - - test("rejects special characters", () => { - expect(isValidSlug("abcdef12345!")).toBe(false); - }); - - test("rejects empty string", () => { - expect(isValidSlug("")).toBe(false); - }); - - test("rejects hyphens", () => { - expect(isValidSlug("abc-def-12345")).toBe(false); - }); -}); - -describe("slugToSchema / schemaToSlug", () => { - test("round-trip slug → schema → slug", () => { - const slug = "abcdef123456"; - expect(schemaToSlug(slugToSchema(slug))).toBe(slug); - }); - - test("slugToSchema adds me_ prefix", () => { - expect(slugToSchema("abcdef123456")).toBe("me_abcdef123456"); - }); - - test("schemaToSlug removes me_ prefix", () => { - expect(schemaToSlug("me_abcdef123456")).toBe("abcdef123456"); - }); -}); diff --git a/packages/engine/migrate/discover.ts b/packages/engine/migrate/discover.ts deleted file mode 100644 index 27f0adf..0000000 --- a/packages/engine/migrate/discover.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { SQL } from "bun"; - -const ENGINE_SCHEMA_RE = /^me_[a-z0-9]{12}$/; -const SLUG_RE = /^[a-z0-9]{12}$/; - -export async function discoverEngineSchemas(sql: SQL): Promise { - const rows = await sql` - select nspname - from pg_namespace - where nspname ~ '^me_[a-z0-9]{12}$' - order by nspname - `; - return rows.map((r: { nspname: string }) => r.nspname); -} - -export function isValidEngineSchema(name: string): boolean { - return ENGINE_SCHEMA_RE.test(name); -} - -export function isValidSlug(slug: string): boolean { - return SLUG_RE.test(slug); -} - -export function slugToSchema(slug: string): string { - return `me_${slug}`; -} - -export function schemaToSlug(schema: string): string { - return schema.slice(3); -} - -export async function assertEngineSchema( - sql: SQL, - schema: string, -): Promise { - if (!isValidEngineSchema(schema)) { - throw new Error( - `Invalid engine schema: "${schema}" — must match me_[a-z0-9]{12}`, - ); - } - - const [row] = await sql` - select 1 from pg_namespace where nspname = ${schema} - `; - if (!row) { - throw new Error(`Engine schema "${schema}" does not exist`); - } -} diff --git a/packages/engine/migrate/index.ts b/packages/engine/migrate/index.ts deleted file mode 100644 index 758524e..0000000 --- a/packages/engine/migrate/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -export { bootstrap } from "./bootstrap"; -export { - assertEngineSchema, - discoverEngineSchemas, - isValidEngineSchema, - isValidSlug, - schemaToSlug, - slugToSchema, -} from "./discover"; -export type { ProvisionResult } from "./provision"; -export { provisionEngine } from "./provision"; -export type { MigrateResult } from "./runner"; -export { - dryRun, - getMigrations, - getVersion, - migrateAll, - migrateEngine, -} from "./runner"; -export type { EngineConfig, ResolvedConfig } from "./template"; -export { defaultConfig, resolveConfig, template } from "./template"; diff --git a/packages/engine/migrate/migrate.integration.test.ts b/packages/engine/migrate/migrate.integration.test.ts deleted file mode 100644 index feae8b0..0000000 --- a/packages/engine/migrate/migrate.integration.test.ts +++ /dev/null @@ -1,823 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { SQL } from "bun"; -import { bootstrap } from "./bootstrap"; -import { discoverEngineSchemas } from "./discover"; -import { provisionEngine } from "./provision"; -import { dryRun, getVersion, migrateAll, migrateEngine } from "./runner"; -import { - countMigrations, - getFunctions, - getIndexes, - getRoles, - getTableColumns, - schemaExists, - TestDatabase, - tableExists, -} from "./test-utils"; - -const testDb = new TestDatabase(); -let connectionString: string; -let sql: SQL; - -beforeAll(async () => { - connectionString = await testDb.create(); - sql = new SQL(connectionString); - await bootstrap(sql); -}); - -afterAll(async () => { - await sql.close(); - await testDb.drop(); -}); - -// --------------------------------------------------------------------------- -// Bootstrap Tests -// --------------------------------------------------------------------------- -describe("bootstrap", () => { - test("creates extensions", async () => { - const rows = await sql` - select extname from pg_extension - where extname in ('citext', 'ltree', 'vector', 'pg_textsearch') - order by extname - `; - const names = rows.map((r: { extname: string }) => r.extname); - expect(names).toEqual(["citext", "ltree", "pg_textsearch", "vector"]); - }); - - test("creates roles", async () => { - const roles = await getRoles(sql, "me_ro", "me_rw", "me_embed"); - expect(roles).toHaveLength(3); - for (const role of roles) { - expect(role.rolcanlogin).toBe(false); - } - }); - - test("does not create embedding schema", async () => { - expect(await schemaExists(sql, "embedding")).toBe(false); - }); - - test("is idempotent", async () => { - // Run bootstrap again — should not error - await bootstrap(sql); - - const rows = await sql` - select extname from pg_extension - where extname in ('citext', 'ltree', 'vector', 'pg_textsearch') - `; - expect(rows).toHaveLength(4); - }); -}); - -// --------------------------------------------------------------------------- -// Single-Engine Migration Tests -// --------------------------------------------------------------------------- -describe("single-engine migration", () => { - const slug = "testengine01"; - const schema = `me_${slug}`; - - beforeAll(async () => { - await provisionEngine(sql, slug, undefined, "0.1.0"); - }); - - test("creates all tables", async () => { - for (const table of [ - "memory", - "user", - "api_key", - "tree_grant", - "role_membership", - "tree_owner", - "migration", - "embedding_queue", - ]) { - expect(await tableExists(sql, schema, table)).toBe(true); - } - }); - - test("creates memory indexes", async () => { - const indexes = await getIndexes(sql, schema, "memory"); - expect(indexes).toContain("memory_meta_gin_idx"); - expect(indexes).toContain("memory_temporal_gist_idx"); - expect(indexes).toContain("memory_content_bm25_idx"); - expect(indexes).toContain("memory_embedding_hnsw_idx"); - expect(indexes).toContain("memory_tree_gist_idx"); - expect(indexes).toContain("memory_null_embedding_idx"); - }); - - test("is idempotent", async () => { - const result = await migrateEngine(sql, schema, undefined, "0.1.0"); - expect(result.status).toBe("ok"); - expect(result.applied).toHaveLength(0); - expect(await countMigrations(sql, schema)).toBe(7); - }); - - test("records migration metadata", async () => { - const rows = await sql.unsafe(` - select name, applied_at_version, applied_at - from ${schema}.migration - order by name - `); - expect(rows).toHaveLength(7); - for (const row of rows) { - expect(row.applied_at_version).toBe("0.1.0"); - expect(row.applied_at).toBeTruthy(); - } - }); - - test("template substitution with custom config", async () => { - // Verify the memory table was created (uses embedding_dimensions template var) - const cols = await getTableColumns(sql, schema, "memory"); - const embCol = cols.find((c) => c.column_name === "embedding"); - expect(embCol).toBeTruthy(); - }); - - test("memory trigger nulls embedding on content change", async () => { - // Insert a memory with a fake embedding - const dims = 1536; - const embedding = `[${Array(dims).fill(0.1).join(",")}]`; - await sql.unsafe(` - insert into ${schema}.memory (content, embedding) - values ('original content', '${embedding}') - `); - - const [before] = await sql.unsafe(` - select id, embedding from ${schema}.memory - where content = 'original content' - `); - expect(before.embedding).not.toBeNull(); - - // Update content without explicitly setting embedding - await sql.unsafe(` - update ${schema}.memory - set content = 'updated content' - where id = '${before.id}' - `); - - const [after] = await sql.unsafe(` - select embedding from ${schema}.memory - where id = '${before.id}' - `); - expect(after.embedding).toBeNull(); - }); - - test("memory trigger increments embedding_version", async () => { - await sql.unsafe(` - insert into ${schema}.memory (content) - values ('version test content') - `); - - const [before] = await sql.unsafe(` - select id, embedding_version from ${schema}.memory - where content = 'version test content' - `); - expect(before.embedding_version).toBe(1); - - await sql.unsafe(` - update ${schema}.memory - set content = 'version test updated' - where id = '${before.id}' - `); - - const [after] = await sql.unsafe(` - select embedding_version from ${schema}.memory - where id = '${before.id}' - `); - expect(after.embedding_version).toBe(2); - }); - - test("memory trigger preserves embedding when explicitly set", async () => { - const dims = 1536; - const embedding = `[${Array(dims).fill(0.2).join(",")}]`; - await sql.unsafe(` - insert into ${schema}.memory (content, embedding) - values ('preserve test', '${embedding}') - `); - - const [{ id }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'preserve test' - `); - - const newEmbedding = `[${Array(dims).fill(0.3).join(",")}]`; - await sql.unsafe(` - update ${schema}.memory - set content = 'preserve test updated', embedding = '${newEmbedding}' - where id = '${id}' - `); - - const [after] = await sql.unsafe(` - select embedding from ${schema}.memory where id = '${id}' - `); - expect(after.embedding).not.toBeNull(); - }); - - test("auth tables have correct structure", async () => { - // User table: thing that accesses memories (or a role if can_login = false) - const userCols = await getTableColumns(sql, schema, "user"); - const colNames = userCols.map((c) => c.column_name); - expect(colNames).toContain("id"); - expect(colNames).toContain("name"); - expect(colNames).toContain("identity_id"); - expect(colNames).toContain("can_login"); - expect(colNames).toContain("superuser"); - expect(colNames).toContain("createrole"); - expect(colNames).toContain("created_at"); - expect(colNames).toContain("updated_at"); - - // api_key table exists in engine (user-scoped, engine-scoped) - expect(await tableExists(sql, schema, "api_key")).toBe(true); - const apiKeyCols = await getTableColumns(sql, schema, "api_key"); - const apiKeyColNames = apiKeyCols.map((c) => c.column_name); - expect(apiKeyColNames).toContain("user_id"); - expect(apiKeyColNames).toContain("lookup_id"); - expect(apiKeyColNames).toContain("key_hash"); - }); - - test("RLS policies enabled on memory", async () => { - const [row] = await sql` - select relrowsecurity - from pg_class c - join pg_namespace n on n.oid = c.relnamespace - where n.nspname = ${schema} and c.relname = 'memory' - `; - expect(row.relrowsecurity).toBe(true); - }); - - test("functions have explicit search_path", async () => { - const funcs = await getFunctions(sql, schema); - for (const func of funcs) { - expect(func.proconfig).toBeTruthy(); - const hasSearchPath = func.proconfig!.some((c: string) => - c.startsWith("search_path="), - ); - expect(hasSearchPath).toBe(true); - } - }); - - test("meta jsonb must be object", async () => { - expect(async () => { - await sql.unsafe(` - insert into ${schema}.memory (content, meta) values ('test', '[]') - `); - }).toThrow(); - }); - - test("temporal constraints enforced", async () => { - // Point-in-time: valid - await sql.unsafe(` - insert into ${schema}.memory (content, temporal) - values ('point', '[2024-01-01, 2024-01-01]') - `); - - // Range: valid - await sql.unsafe(` - insert into ${schema}.memory (content, temporal) - values ('range', '[2024-01-01, 2024-06-01)') - `); - - // Invalid: exclusive lower bound - expect(async () => { - await sql.unsafe(` - insert into ${schema}.memory (content, temporal) - values ('bad', '(2024-01-01, 2024-06-01)') - `); - }).toThrow(); - }); - - test("embedding_version defaults to 1", async () => { - await sql.unsafe(` - insert into ${schema}.memory (content) values ('ev default test') - `); - const [row] = await sql.unsafe(` - select embedding_version from ${schema}.memory - where content = 'ev default test' - `); - expect(row.embedding_version).toBe(1); - }); - - test("enqueue trigger fires on insert", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - await sql.unsafe(` - insert into ${schema}.memory (content) values ('trigger insert test') - `); - - const rows = await sql.unsafe(` - select memory_id, embedding_version - from ${schema}.embedding_queue - `); - expect(rows.length).toBeGreaterThanOrEqual(1); - }); - - test("enqueue trigger fires on content update", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - await sql.unsafe(` - insert into ${schema}.memory (content) values ('trigger update before') - `); - - // Clear queue entries from the insert - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - const [{ id }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'trigger update before' - `); - - await sql.unsafe(` - update ${schema}.memory set content = 'trigger update after' where id = '${id}' - `); - - const rows = await sql.unsafe(` - select embedding_version - from ${schema}.embedding_queue - `); - expect(rows.length).toBeGreaterThanOrEqual(1); - }); - - test("memory deletion cascades to embedding_queue", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - await sql.unsafe(` - insert into ${schema}.memory (content) values ('cascade test') - `); - - const [{ id }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'cascade test' - `); - - // Verify queue entry exists - const before = await sql.unsafe(` - select count(*)::int as cnt from ${schema}.embedding_queue where memory_id = '${id}' - `); - expect(before[0].cnt).toBeGreaterThanOrEqual(1); - - // Delete memory — queue entry should cascade - await sql.unsafe(`delete from ${schema}.memory where id = '${id}'`); - - const after = await sql.unsafe(` - select count(*)::int as cnt from ${schema}.embedding_queue where memory_id = '${id}' - `); - expect(after[0].cnt).toBe(0); - }); - - test("me_embed role has correct per-schema grants", async () => { - // Check schema usage - const [{ has_usage }] = await sql` - select has_schema_privilege('me_embed', ${schema}, 'USAGE') as has_usage - `; - expect(has_usage).toBe(true); - - // Check memory table privileges - const [{ has_select }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.memory`}, 'SELECT') as has_select - `; - expect(has_select).toBe(true); - - const [{ has_update }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.memory`}, 'UPDATE') as has_update - `; - expect(has_update).toBe(true); - - // Check embedding_queue table privileges - const [{ eq_select }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.embedding_queue`}, 'SELECT') as eq_select - `; - expect(eq_select).toBe(true); - - const [{ eq_update }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.embedding_queue`}, 'UPDATE') as eq_update - `; - expect(eq_update).toBe(true); - - const [{ eq_delete }] = await sql` - select has_table_privilege('me_embed', ${`${schema}.embedding_queue`}, 'DELETE') as eq_delete - `; - expect(eq_delete).toBe(true); - - // Check claim function privilege - const [{ has_execute }] = await sql` - select has_function_privilege('me_embed', ${`${schema}.claim_embedding_batch(int, interval)`}, 'EXECUTE') as has_execute - `; - expect(has_execute).toBe(true); - }); - - test("me_rw cannot access embedding_queue", async () => { - const [{ has_select }] = await sql` - select has_table_privilege('me_rw', ${`${schema}.embedding_queue`}, 'SELECT') as has_select - `; - expect(has_select).toBe(false); - }); - - test("me_ro cannot access embedding_queue", async () => { - const [{ has_select }] = await sql` - select has_table_privilege('me_ro', ${`${schema}.embedding_queue`}, 'SELECT') as has_select - `; - expect(has_select).toBe(false); - }); - - test("embedding_queue has FK with ON DELETE CASCADE", async () => { - const fks = await sql` - select - tc.constraint_name, - rc.delete_rule - from information_schema.table_constraints tc - join information_schema.referential_constraints rc - on tc.constraint_name = rc.constraint_name - and tc.constraint_schema = rc.constraint_schema - where tc.table_schema = ${schema} - and tc.table_name = 'embedding_queue' - and tc.constraint_type = 'FOREIGN KEY' - `; - expect(fks).toHaveLength(1); - expect(fks[0].delete_rule).toBe("CASCADE"); - }); - - test("per-engine enqueue_embedding and claim_embedding_batch functions exist", async () => { - const funcs = await getFunctions(sql, schema); - const names = funcs.map((f) => f.proname); - expect(names).toContain("enqueue_embedding"); - expect(names).toContain("claim_embedding_batch"); - expect(names).toContain("prune_embedding_queue"); - }); - - test("me_embed can execute prune_embedding_queue", async () => { - const [{ has_execute }] = await sql` - select has_function_privilege( - 'me_embed', - ${`${schema}.prune_embedding_queue(interval)`}, - 'EXECUTE' - ) as has_execute - `; - expect(has_execute).toBe(true); - }); - - test("prune_embedding_queue removes terminal rows older than retention", async () => { - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - // Insert a memory we can reference (avoid FK fan-out) - await sql.unsafe(` - insert into ${schema}.memory (content) values ('prune test memory') - `); - const [{ id: memId }] = await sql.unsafe(` - select id from ${schema}.memory where content = 'prune test memory' - `); - - // Clear the auto-enqueued row from the trigger - await sql.unsafe(`delete from ${schema}.embedding_queue`); - - // Old completed (should be pruned) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 1, 'completed', now() - interval '10 days') - `); - // Old failed (should be pruned) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 2, 'failed', now() - interval '10 days') - `); - // Old cancelled (should be pruned) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 3, 'cancelled', now() - interval '10 days') - `); - // Recent completed (should be kept) - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 4, 'completed', now() - interval '1 day') - `); - // Old but outcome IS NULL — active queue item, must NOT be pruned - await sql.unsafe(` - insert into ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - values - ('${memId}', 5, null, now() - interval '30 days') - `); - - const [{ pruned }] = await sql.unsafe( - `select ${schema}.prune_embedding_queue(interval '7 days') as pruned`, - ); - expect(Number(pruned)).toBe(3); - - const remaining = await sql.unsafe( - `select embedding_version, outcome from ${schema}.embedding_queue - where memory_id = '${memId}' - order by embedding_version`, - ); - expect(remaining).toHaveLength(2); - expect(remaining[0].embedding_version).toBe(4); - expect(remaining[0].outcome).toBe("completed"); - expect(remaining[1].embedding_version).toBe(5); - expect(remaining[1].outcome).toBeNull(); - }); -}); - -// --------------------------------------------------------------------------- -// Multi-Engine Tests -// --------------------------------------------------------------------------- -describe("multi-engine migration", () => { - const slugs = ["aaaa00000001", "aaaa00000002", "aaaa00000003"]; - const schemas = slugs.map((s) => `me_${s}`); - - beforeAll(async () => { - for (const slug of slugs) { - await provisionEngine(sql, slug, undefined, "0.1.0"); - } - }); - - test("migrateAll migrates multiple schemas", async () => { - // All already migrated, should be no-op - const results = await migrateAll(sql, schemas, undefined, "0.1.0"); - expect(results.size).toBe(3); - for (const [, result] of results) { - expect(result.status).toBe("ok"); - expect(result.applied).toHaveLength(0); - } - for (const schema of schemas) { - expect(await tableExists(sql, schema, "memory")).toBe(true); - } - }); - - test("migrateAll isolates failures", async () => { - const badSchema = "me_badschema000"; - // Don't create this schema — migration should fail - const allSchemas = [...schemas, badSchema]; - - const results = await migrateAll(sql, allSchemas, undefined, "0.1.0"); - - // Good schemas should succeed (already migrated, so 0 applied) - for (const schema of schemas) { - expect(results.get(schema)!.status).toBe("ok"); - } - - // Bad schema should error - expect(results.get(badSchema)!.status).toBe("error"); - expect(results.get(badSchema)!.error).toBeTruthy(); - }); - - test("concurrency control processes with concurrency=1", async () => { - const results = await migrateAll(sql, schemas, undefined, "0.1.0", { - concurrency: 1, - }); - expect(results.size).toBe(3); - for (const [, result] of results) { - expect(result.status).toBe("ok"); - } - }); -}); - -// --------------------------------------------------------------------------- -// Discovery Tests -// --------------------------------------------------------------------------- -describe("discovery", () => { - test("finds engine schemas", async () => { - // me_testengine01 was created earlier, plus multi schemas - const discovered = await discoverEngineSchemas(sql); - expect(discovered).toContain("me_testengine01"); - expect(discovered).toContain("me_aaaa00000001"); - }); - - test("ignores non-engine schemas", async () => { - const discovered = await discoverEngineSchemas(sql); - expect(discovered).not.toContain("public"); - // embedding schema no longer exists - expect(discovered).not.toContain("pg_catalog"); - }); - - test("returns sorted results", async () => { - const discovered = await discoverEngineSchemas(sql); - const sorted = [...discovered].sort(); - expect(discovered).toEqual(sorted); - }); -}); - -// --------------------------------------------------------------------------- -// Advisory Lock Tests -// --------------------------------------------------------------------------- -describe("advisory locks", () => { - test("concurrent migrateEngine on same schema — only one applies", async () => { - const slug = "locktest0001"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - - // Now run concurrent migrations (all should succeed with 0 applied due to idempotency) - const results = await Promise.all([ - migrateEngine(sql, schema, undefined, "0.1.0"), - migrateEngine(sql, schema, undefined, "0.1.0"), - migrateEngine(sql, schema, undefined, "0.1.0"), - ]); - - // All should complete (ok or skipped) - for (const result of results) { - expect(["ok", "skipped"]).toContain(result.status); - } - - // Exactly 7 migrations should exist - expect(await countMigrations(sql, schema)).toBe(7); - }); - - test("concurrent migrateEngine on different schemas — both proceed", async () => { - const slugA = "locktest0002"; - const slugB = "locktest0003"; - const schemaA = `me_${slugA}`; - const schemaB = `me_${slugB}`; - - // Provision both first - await Promise.all([ - provisionEngine(sql, slugA, undefined, "0.1.0"), - provisionEngine(sql, slugB, undefined, "0.1.0"), - ]); - - // Now run migrations (should be no-ops since provisioning ran them) - const [resultA, resultB] = await Promise.all([ - migrateEngine(sql, schemaA, undefined, "0.1.0"), - migrateEngine(sql, schemaB, undefined, "0.1.0"), - ]); - - expect(resultA.status).toBe("ok"); - expect(resultB.status).toBe("ok"); - }); -}); - -// --------------------------------------------------------------------------- -// Provisioning Tests -// --------------------------------------------------------------------------- -describe("provisioning", () => { - test("creates schema and runs all migrations", async () => { - const result = await provisionEngine( - sql, - "prov00000001", - undefined, - "0.1.0", - ); - expect(result.schema).toBe("me_prov00000001"); - expect(result.migrateResult.status).toBe("ok"); - expect(result.migrateResult.applied).toHaveLength(7); - expect(await schemaExists(sql, "me_prov00000001")).toBe(true); - expect(await tableExists(sql, "me_prov00000001", "memory")).toBe(true); - expect(await tableExists(sql, "me_prov00000001", "user")).toBe(true); - }); - - test("validates slug format", () => { - expect(provisionEngine(sql, "BAD", undefined, "0.1.0")).rejects.toThrow( - "Invalid engine slug", - ); - - expect( - provisionEngine(sql, "too-short", undefined, "0.1.0"), - ).rejects.toThrow("Invalid engine slug"); - }); - - test("fails if schema already exists", async () => { - await provisionEngine(sql, "prov00000002", undefined, "0.1.0"); - - await expect( - provisionEngine(sql, "prov00000002", undefined, "0.1.0"), - ).rejects.toThrow(); - }); - - test("creates version table", async () => { - const slug = "prov00000003"; - await provisionEngine(sql, slug, undefined, "0.1.0"); - expect(await tableExists(sql, `me_${slug}`, "version")).toBe(true); - expect(await getVersion(sql, `me_${slug}`)).toBe("0.1.0"); - }); -}); - -// --------------------------------------------------------------------------- -// Dry Run Tests -// --------------------------------------------------------------------------- -describe("dry run", () => { - test("shows all pending for new schema", async () => { - // Create a schema manually without running migrations (simulating a fresh schema) - const schema = "me_dryrun000001"; - await sql.unsafe(`create schema if not exists ${schema}`); - - const result = await dryRun(sql, schema); - expect(result.pending).toHaveLength(7); - expect(result.applied).toHaveLength(0); - }); - - test("shows none pending after full migration", async () => { - const slug = "dryrun000002"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - - const result = await dryRun(sql, schema); - expect(result.pending).toHaveLength(0); - expect(result.applied).toHaveLength(7); - }); -}); - -// --------------------------------------------------------------------------- -// Version Tracking Tests -// --------------------------------------------------------------------------- -describe("version tracking", () => { - test("applied_at_version records correctly", async () => { - const slug = "version00001"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "1.2.3"); - - const rows = await sql.unsafe(` - select applied_at_version from ${schema}.migration - `); - for (const row of rows) { - expect(row.applied_at_version).toBe("1.2.3"); - } - }); - - test("re-migrate with same migrations is no-op", async () => { - const slug = "version00002"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - - const result = await migrateEngine(sql, schema, undefined, "0.1.0"); - expect(result.applied).toHaveLength(0); - }); - - test("rejects downgrade", async () => { - const slug = "version00003"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.2.0"); - - await expect( - migrateEngine(sql, schema, undefined, "0.1.0"), - ).rejects.toThrow("older than database version"); - }); - - test("updates version on upgrade", async () => { - const slug = "version00004"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "0.1.0"); - expect(await getVersion(sql, schema)).toBe("0.1.0"); - - await migrateEngine(sql, schema, undefined, "0.2.0"); - expect(await getVersion(sql, schema)).toBe("0.2.0"); - }); - - test("getVersion returns current version", async () => { - const slug = "version00005"; - const schema = `me_${slug}`; - await provisionEngine(sql, slug, undefined, "1.2.3"); - expect(await getVersion(sql, schema)).toBe("1.2.3"); - }); -}); - -// --------------------------------------------------------------------------- -// Cross-Engine Isolation Tests -// --------------------------------------------------------------------------- -describe("cross-engine isolation", () => { - const slugA = "isolate00001"; - const slugB = "isolate00002"; - const schemaA = `me_${slugA}`; - const schemaB = `me_${slugB}`; - - beforeAll(async () => { - await provisionEngine(sql, slugA, undefined, "0.1.0"); - await provisionEngine(sql, slugB, undefined, "0.1.0"); - }); - - test("data isolated between schemas", async () => { - await sql.unsafe(` - insert into ${schemaA}.memory (content) values ('only in A') - `); - - const rowsA = await sql.unsafe(` - select content from ${schemaA}.memory where content = 'only in A' - `); - expect(rowsA).toHaveLength(1); - - const rowsB = await sql.unsafe(` - select content from ${schemaB}.memory where content = 'only in A' - `); - expect(rowsB).toHaveLength(0); - }); - - test("embedding queue entries are per-engine", async () => { - await sql.unsafe(`delete from ${schemaA}.embedding_queue`); - await sql.unsafe(`delete from ${schemaB}.embedding_queue`); - - await sql.unsafe(` - insert into ${schemaA}.memory (content) values ('queue test A') - `); - await sql.unsafe(` - insert into ${schemaB}.memory (content) values ('queue test B') - `); - - const rowsA = await sql.unsafe(` - select memory_id from ${schemaA}.embedding_queue - `); - expect(rowsA).toHaveLength(1); - - const rowsB = await sql.unsafe(` - select memory_id from ${schemaB}.embedding_queue - `); - expect(rowsB).toHaveLength(1); - }); -}); diff --git a/packages/engine/migrate/migrations/001_updated_at.sql b/packages/engine/migrate/migrations/001_updated_at.sql deleted file mode 100644 index 4e11805..0000000 --- a/packages/engine/migrate/migrations/001_updated_at.sql +++ /dev/null @@ -1,10 +0,0 @@ --- generic trigger function to update updated_at timestamp -create function {{schema}}.update_updated_at() -returns trigger -as $func$ -begin - new.updated_at = pg_catalog.now(); - return new; -end; -$func$ language plpgsql volatile security definer -set search_path to {{schema}}, pg_temp; diff --git a/packages/engine/migrate/migrations/003_memory_trigger.sql b/packages/engine/migrate/migrations/003_memory_trigger.sql deleted file mode 100644 index 48c5a81..0000000 --- a/packages/engine/migrate/migrations/003_memory_trigger.sql +++ /dev/null @@ -1,34 +0,0 @@ --- before-update trigger for memory table -create function {{schema}}.memory_before_update() -returns trigger -as $func$ -begin - -- always update the timestamp - new.updated_at = pg_catalog.now(); - - -- content changed -> new embedding needs to be generated - if old.content is distinct from new.content - and old.embedding is not distinct from new.embedding - then - new.embedding = null; - new.embedding_attempts = 0; - new.embedding_version = old.embedding_version operator(pg_catalog.+) 1; - new.embedding_last_error = null; - end if; - - -- likely the embedding engine setting the embedding - if new.embedding is not null then - new.embedding_attempts = 0; - new.embedding_last_error = null; - end if; - - return new; -end; -$func$ language plpgsql volatile security definer -set search_path to {{schema}}, public, pg_temp; -- public required for pgvector's `is not distinct from` - -create trigger memory_before_update_trg -before update on {{schema}}.memory -for each row -execute function {{schema}}.memory_before_update(); - diff --git a/packages/engine/migrate/migrations/004_auth_tables.sql b/packages/engine/migrate/migrations/004_auth_tables.sql deleted file mode 100644 index e32fa6b..0000000 --- a/packages/engine/migrate/migrations/004_auth_tables.sql +++ /dev/null @@ -1,227 +0,0 @@ --- ===== Users ===== --- User: thing that accesses memories, or a role (can_login = false) --- identity_id is a soft FK to accounts.identity (nullable for service users) --- Note: "user" is a reserved word, must be quoted -create table {{schema}}."user" -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, name citext not null unique -, identity_id uuid check (identity_id is null or uuid_extract_version(identity_id) = 7) -- soft FK to accounts.identity -, can_login boolean not null default true -- false = role (grant container) -, superuser boolean not null default false -, createrole boolean not null default false -- can create other users/roles -, created_at timestamptz not null default now() -, updated_at timestamptz -); - -create index idx_user_identity_id on {{schema}}."user" (identity_id) where identity_id is not null; - -create trigger user_updated_at - before update on {{schema}}."user" - for each row - execute function {{schema}}.update_updated_at(); - --- ===== API Keys ===== --- Engine-scoped, user-scoped authentication -create table {{schema}}.api_key -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, user_id uuid not null references {{schema}}."user" on delete cascade -, lookup_id text unique not null check (lookup_id ~ '^[A-Za-z0-9_-]{16}$') -, key_hash text not null -, name text not null -, expires_at timestamptz -, created_at timestamptz not null default now() -, revoked_at timestamptz -); - -create index idx_api_key_user on {{schema}}.api_key (user_id); -create index idx_api_key_lookup on {{schema}}.api_key (lookup_id) where revoked_at is null; - --- ===== Tree Grants ===== -create table {{schema}}.tree_grant -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, user_id uuid not null references {{schema}}."user"(id) on delete cascade -, tree_path ltree not null -, actions text[] not null -, granted_by uuid references {{schema}}."user"(id) -, created_at timestamptz not null default now() -, with_grant_option boolean not null default false -, constraint valid_actions check ( - actions <@ '{read,create,update,delete}'::text[] - ) -); - -create unique index idx_tree_grant_unique - on {{schema}}.tree_grant (user_id, tree_path); - -create index idx_tree_grant_user - on {{schema}}.tree_grant using btree (user_id); - -create index idx_tree_grant_path - on {{schema}}.tree_grant using gist (tree_path); - --- ===== Role Membership ===== -create table {{schema}}.role_membership -( role_id uuid not null references {{schema}}."user"(id) on delete cascade -, member_id uuid not null references {{schema}}."user"(id) on delete cascade -, with_admin_option boolean not null default false -, created_at timestamptz not null default now() -, primary key (role_id, member_id) -, constraint no_self_membership check (role_id <> member_id) -); - -create index idx_role_membership_member on {{schema}}.role_membership(member_id); - --- ===== Cycle Detection ===== -create function {{schema}}.would_create_cycle -( _role_id uuid -, _member_id uuid -) -returns boolean -as $func$ - with recursive ancestors(id) as ( - select rm.role_id - from {{schema}}.role_membership rm - where rm.member_id = _role_id - union - select rm.role_id - from {{schema}}.role_membership rm - inner join ancestors a on a.id = rm.member_id - ) - select _member_id = _role_id - or exists - ( - select 1 - from ancestors - where id = _member_id - ) -$func$ language sql stable security invoker -set search_path to pg_catalog, {{schema}}, pg_temp -; - --- ===== Tree Ownership ===== -create table {{schema}}.tree_owner -( tree_path ltree primary key -, user_id uuid not null references {{schema}}."user"(id) on delete cascade -, created_by uuid references {{schema}}."user"(id) -, created_at timestamptz not null default now() -); - -create index idx_tree_owner_user on {{schema}}.tree_owner (user_id); -create index idx_tree_owner_gist on {{schema}}.tree_owner using gist (tree_path); - --- ===== Access Checking (role-aware) ===== --- Returns set of tree paths the user can access for the given action. --- Superusers get ''::ltree (empty root) which matches all paths via <@. - -create function {{schema}}.tree_access -( _user_id uuid -, _action text -) -returns setof ltree -as $func$ - with recursive effective_roles(user_id) as - ( - select _user_id - union - select rm.role_id - from {{schema}}.role_membership rm - inner join effective_roles er on (er.user_id = rm.member_id) - ) - select distinct tree_path - from - ( - -- superuser: empty ltree matches everything via <@ - select ''::ltree as tree_path - from {{schema}}."user" u - inner join effective_roles er on (u.id = er.user_id) - where u.superuser - union - -- ownership grants full access - select o.tree_path - from {{schema}}.tree_owner o - inner join effective_roles er on (er.user_id = o.user_id) - union - -- explicit grants for the requested action - select g.tree_path - from {{schema}}.tree_grant g - inner join effective_roles er on (er.user_id = g.user_id) - where _action = any(g.actions) - ) -$func$ -language sql stable security definer -set search_path to pg_catalog, {{schema}}, public, pg_temp -; - -revoke all on function {{schema}}.tree_access(uuid, text) from public; -grant execute on function {{schema}}.tree_access(uuid, text) to me_ro, me_rw; - --- defense in depth: revoke PUBLIC access on auth tables -revoke all on {{schema}}."user" from public; -revoke all on {{schema}}.api_key from public; -revoke all on {{schema}}.tree_grant from public; -revoke all on {{schema}}.role_membership from public; -revoke all on {{schema}}.tree_owner from public; - --- ===== RLS on memory ===== -alter table {{schema}}.memory enable row level security; - -create policy memory_select on {{schema}}.memory - for select to me_ro, me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'read') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_insert on {{schema}}.memory - for insert to me_rw - with check - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'create') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_update on {{schema}}.memory - for update to me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) - where tree <@ ta.tree_path - ) - ) - with check - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_delete on {{schema}}.memory - for delete to me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'delete') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - --- ===== Memory FK ===== -alter table {{schema}}.memory add constraint memory_created_by_fk - foreign key (created_by) references {{schema}}."user"(id) on delete set null; diff --git a/packages/engine/migrate/migrations/005_embedding_queue.sql b/packages/engine/migrate/migrations/005_embedding_queue.sql deleted file mode 100644 index c325316..0000000 --- a/packages/engine/migrate/migrations/005_embedding_queue.sql +++ /dev/null @@ -1,150 +0,0 @@ --- per-engine embedding queue table -create table {{schema}}.embedding_queue -( id bigint generated always as identity primary key -, memory_id uuid not null references {{schema}}.memory(id) on delete cascade -, embedding_version int not null -, vt timestamptz not null default now() -, outcome text check (outcome is null or outcome in ('completed', 'failed', 'cancelled')) -, attempts int not null default 0 -, max_attempts int not null default 3 -, last_error text -, created_at timestamptz not null default now() -); - -create index embedding_queue_claim_idx - on {{schema}}.embedding_queue (vt) - where outcome is null; -create index embedding_queue_memory_idx - on {{schema}}.embedding_queue (memory_id, embedding_version desc) - where outcome is null; -create index embedding_queue_archive_idx - on {{schema}}.embedding_queue (created_at) - where outcome is not null; - --- enqueue function (SECURITY DEFINER — me_rw cannot access queue directly) -create or replace function {{schema}}.enqueue_embedding() -returns trigger -language plpgsql volatile security definer -set search_path to pg_catalog, {{schema}}, pg_temp -as $func$ -begin - insert into {{schema}}.embedding_queue (memory_id, embedding_version) - values (new.id, new.embedding_version); - return new; -end; -$func$; - --- enqueue triggers -create trigger memory_enqueue_embedding_insert - after insert on {{schema}}.memory - for each row - when (new.embedding is null) - execute function {{schema}}.enqueue_embedding(); - -create trigger memory_enqueue_embedding_update - after update on {{schema}}.memory - for each row - when (old.content is distinct from new.content - and new.embedding is null - and new.embedding_attempts < 3) - execute function {{schema}}.enqueue_embedding(); - --- claim function for embedding worker -create or replace function {{schema}}.claim_embedding_batch( - batch_size int default 10, - lock_duration interval default '5 minutes' -) -returns table (queue_id bigint, memory_id uuid, embedding_version int, content text) -language plpgsql volatile -set search_path to pg_catalog, {{schema}}, pg_temp -as $func$ -declare - rec record; - mem record; - claimed_count int := 0; -begin - -- bulk-cancel visible queue rows superseded by a newer row for the same memory - update {{schema}}.embedding_queue eq - set outcome = 'cancelled' - where eq.outcome is null - and eq.vt <= now() - and exists ( - select 1 - from {{schema}}.embedding_queue newer - where newer.memory_id = eq.memory_id - and newer.embedding_version > eq.embedding_version - and newer.outcome is null - ); - - -- sweep: finalize exhausted rows orphaned by worker crash - -- (attempts reached max but outcome was never written back) - update {{schema}}.embedding_queue - set outcome = 'failed' - , last_error = coalesce(last_error, 'exceeded max attempts (worker crash)') - where outcome is null - and vt <= now() - and attempts >= max_attempts; - - for rec in - select eq.id, eq.memory_id, eq.embedding_version - from {{schema}}.embedding_queue eq - where eq.outcome is null - and eq.vt <= now() - and eq.attempts < eq.max_attempts - order by eq.vt - for update skip locked - loop - -- check memory still exists + current version - select m.content, m.embedding_version - into mem - from {{schema}}.memory m - where m.id = rec.memory_id; - - if not found or mem.content is null then - -- memory deleted or empty → cancel queue row - update {{schema}}.embedding_queue - set outcome = 'cancelled' - where id = rec.id; - continue; - end if; - - if rec.embedding_version <> mem.embedding_version then - -- stale version → cancel - update {{schema}}.embedding_queue - set outcome = 'cancelled' - where id = rec.id; - continue; - end if; - - -- claim this row - update {{schema}}.embedding_queue - set vt = now() + lock_duration - , attempts = {{schema}}.embedding_queue.attempts + 1 - where id = rec.id; - - queue_id := rec.id; - memory_id := rec.memory_id; - embedding_version := rec.embedding_version; - content := mem.content; - return next; - - claimed_count := claimed_count + 1; - exit when claimed_count >= batch_size; - end loop; -end; -$func$; - --- me_embed RLS — system role, unrestricted access to all memories -create policy memory_embed_select on {{schema}}.memory - for select to me_embed - using (true); - -create policy memory_embed_update on {{schema}}.memory - for update to me_embed - using (true); - --- me_embed grants (memory + queue + claim function) -grant usage on schema {{schema}} to me_embed; -grant select, update on {{schema}}.memory to me_embed; -grant select, update, delete on {{schema}}.embedding_queue to me_embed; -grant execute on function {{schema}}.claim_embedding_batch(int, interval) to me_embed; diff --git a/packages/engine/migrate/migrations/006_prune_embedding_queue.sql b/packages/engine/migrate/migrations/006_prune_embedding_queue.sql deleted file mode 100644 index 1a17b08..0000000 --- a/packages/engine/migrate/migrations/006_prune_embedding_queue.sql +++ /dev/null @@ -1,27 +0,0 @@ --- prune terminal queue rows older than the retention window. --- runs opportunistically from the worker on engines that returned no --- claimable work, so the queue table doesn't grow unbounded. --- --- relies on embedding_queue_archive_idx (created_at) where outcome is not null --- from migration 005, so the no-op case is cheap. -create or replace function {{schema}}.prune_embedding_queue -( retention interval default '7 days' -) -returns bigint -language plpgsql volatile -set search_path to pg_catalog, {{schema}}, pg_temp -as $func$ -declare - pruned bigint; -begin - delete from {{schema}}.embedding_queue - where outcome is not null - and created_at < now() - retention; - get diagnostics pruned = row_count; - return pruned; -end; -$func$; - --- me_embed already has DELETE on embedding_queue (granted in 005); --- this just exposes the function entrypoint. -grant execute on function {{schema}}.prune_embedding_queue(interval) to me_embed; diff --git a/packages/engine/migrate/migrations/007_drop_api_key_last_used_at.sql b/packages/engine/migrate/migrations/007_drop_api_key_last_used_at.sql deleted file mode 100644 index 8094573..0000000 --- a/packages/engine/migrate/migrations/007_drop_api_key_last_used_at.sql +++ /dev/null @@ -1,2 +0,0 @@ -alter table {{schema}}.api_key - drop column if exists last_used_at; diff --git a/packages/engine/migrate/provision.ts b/packages/engine/migrate/provision.ts deleted file mode 100644 index d834b6a..0000000 --- a/packages/engine/migrate/provision.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { SQL } from "bun"; - -import { setLocalEngineTimeouts } from "../ops/_tx"; -import { isValidSlug, slugToSchema } from "./discover"; -import { type MigrateResult, migrateEngine } from "./runner"; -import type { EngineConfig } from "./template"; - -export interface ProvisionResult { - schema: string; - migrateResult: MigrateResult; -} - -export async function provisionEngine( - sql: SQL, - slug: string, - config: EngineConfig | undefined, - serverVersion: string, - shardId?: number, -): Promise { - if (!isValidSlug(slug)) { - throw new Error( - `Invalid engine slug: "${slug}" — must be 12 lowercase alphanumeric characters`, - ); - } - - const schema = slugToSchema(slug); - - // Transaction 1: Create schema infrastructure (all or nothing) - await sql.begin(async (tx) => { - if (shardId !== undefined) { - await tx.unsafe(`set local pgdog.shard to ${shardId}`); - } - await setLocalEngineTimeouts(tx); - - // Create schema (fails if exists - use migrateEngine for existing schemas) - await tx.unsafe(`create schema ${schema}`); - - // Version tracking table (single row) - await tx.unsafe(` - create table ${schema}.version - ( version text not null check (version ~ '^\\d+\\.\\d+\\.\\d+$') - , at timestamptz not null default now() - ) - `); - await tx.unsafe(`create unique index on ${schema}.version ((true))`); - await tx.unsafe(`insert into ${schema}.version (version) values ('0.0.0')`); - - // Grant usage to all roles - await tx.unsafe( - `grant usage on schema ${schema} to me_ro, me_rw, me_embed`, - ); - }); - - // Transaction 2: Run migrations - const migrateResult = await migrateEngine( - sql, - schema, - config, - serverVersion, - shardId, - ); - - return { schema, migrateResult }; -} diff --git a/packages/engine/migrate/runner.test.ts b/packages/engine/migrate/runner.test.ts deleted file mode 100644 index aaf355f..0000000 --- a/packages/engine/migrate/runner.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { getMigrations } from "./runner"; - -describe("getMigrations", () => { - test("returns 7 migrations", () => { - expect(getMigrations()).toHaveLength(7); - }); - - test("migrations are sorted by name", () => { - const names = getMigrations().map((m) => m.name); - const sorted = [...names].sort(); - expect(names).toEqual(sorted); - }); - - test("migration names match NNN_name pattern", () => { - for (const { name } of getMigrations()) { - expect(name).toMatch(/^\d{3}_\w+$/); - } - }); - - test("contains expected migration names", () => { - const names = getMigrations().map((m) => m.name); - expect(names).toContain("001_updated_at"); - expect(names).toContain("002_memory"); - expect(names).toContain("003_memory_trigger"); - expect(names).toContain("004_auth_tables"); - expect(names).toContain("005_embedding_queue"); - expect(names).toContain("006_prune_embedding_queue"); - expect(names).toContain("007_drop_api_key_last_used_at"); - }); -}); diff --git a/packages/engine/migrate/runner.ts b/packages/engine/migrate/runner.ts deleted file mode 100644 index c557ccb..0000000 --- a/packages/engine/migrate/runner.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { type SQL, semver } from "bun"; -import { setLocalEngineTimeouts } from "../ops/_tx"; -import { assertEngineSchema } from "./discover"; -import migration001 from "./migrations/001_updated_at.sql" with { - type: "text", -}; -import migration002 from "./migrations/002_memory.sql" with { type: "text" }; -import migration003 from "./migrations/003_memory_trigger.sql" with { - type: "text", -}; -import migration004 from "./migrations/004_auth_tables.sql" with { - type: "text", -}; -import migration005 from "./migrations/005_embedding_queue.sql" with { - type: "text", -}; -import migration006 from "./migrations/006_prune_embedding_queue.sql" with { - type: "text", -}; -import migration007 from "./migrations/007_drop_api_key_last_used_at.sql" with { - type: "text", -}; -import { type EngineConfig, resolveConfig, template } from "./template"; - -interface Migration { - name: string; - sql: string; -} - -const migrations: Migration[] = [ - { name: "001_updated_at", sql: migration001 }, - { name: "002_memory", sql: migration002 }, - { name: "003_memory_trigger", sql: migration003 }, - { name: "004_auth_tables", sql: migration004 }, - { name: "005_embedding_queue", sql: migration005 }, - { name: "006_prune_embedding_queue", sql: migration006 }, - { name: "007_drop_api_key_last_used_at", sql: migration007 }, -]; - -export interface MigrateResult { - schema: string; - status: "ok" | "skipped" | "error"; - applied: string[]; - error?: Error; -} - -const MAX_LOCK_RETRIES = 5; -const BASE_DELAY_MS = 100; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function assertSchemaOwnership(tx: SQL, schema: string): Promise { - const [result] = await tx` - select - n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner - from pg_catalog.pg_namespace n - where n.nspname = ${schema} - `; - - if (!result?.is_owner) { - throw new Error( - `Only the owner of the ${schema} schema can run database migrations`, - ); - } -} - -export async function migrateEngine( - sql: SQL, - schema: string, - config: EngineConfig | undefined, - serverVersion: string, - shardId?: number, -): Promise { - const resolved = resolveConfig(schema, config); - - return await sql.begin(async (tx) => { - if (shardId !== undefined) { - await tx.unsafe(`set local pgdog.shard to ${shardId}`); - } - await setLocalEngineTimeouts(tx); - - await assertEngineSchema(tx, schema); - - // 1. Acquire advisory lock with retry - const [{ lock_id }] = await tx` - select hashtext(${schema})::bigint as lock_id - `; - - let acquired = false; - for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { - const [result] = await tx` - select pg_try_advisory_xact_lock(${lock_id}) as acquired - `; - if (result.acquired) { - acquired = true; - break; - } - if (attempt < MAX_LOCK_RETRIES - 1) { - await sleep(BASE_DELAY_MS * 2 ** attempt); - } - } - - if (!acquired) { - return { schema, status: "skipped" as const, applied: [] }; - } - - // 2. Check ownership - await assertSchemaOwnership(tx, schema); - - // 3. Check version (reject downgrades) - const [{ version: dbVersion }] = await tx.unsafe( - `select version from ${schema}.version`, - ); - - const cmp = semver.order(serverVersion, dbVersion); - if (cmp < 0) { - throw new Error( - `Server version (${serverVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the server.", - ); - } - - // 4. Scaffold migration tracking table - await tx.unsafe(` - create table if not exists ${schema}.migration - ( name text not null primary key - , applied_at_version text not null - , applied_at timestamptz not null default pg_catalog.clock_timestamp() - ) - `); - - // 5. Run migrations - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - const applied: string[] = []; - - for (const migration of sorted) { - const [existing] = await tx.unsafe( - `select 1 from ${schema}.migration where name = $1`, - [migration.name], - ); - - if (existing) { - continue; - } - - const renderedSql = template(migration.sql, resolved); - await tx.unsafe(renderedSql); - await tx.unsafe( - `insert into ${schema}.migration (name, applied_at_version) values ($1, $2)`, - [migration.name, serverVersion], - ); - applied.push(migration.name); - } - - // 6. Update version if app version is newer - if (cmp > 0) { - await tx.unsafe(`update ${schema}.version set version = $1, at = now()`, [ - serverVersion, - ]); - } - - return { schema, status: "ok" as const, applied }; - }); -} - -export async function migrateAll( - sql: SQL, - schemas: string[], - config: EngineConfig | undefined, - serverVersion: string, - options?: { concurrency?: number; shardId?: number }, -): Promise> { - const concurrency = options?.concurrency ?? 10; - const shardId = options?.shardId; - const results = new Map(); - - // Simple semaphore for bounded parallelism - let active = 0; - let idx = 0; - - const runOne = async (schema: string): Promise => { - try { - const result = await migrateEngine( - sql, - schema, - config, - serverVersion, - shardId, - ); - results.set(schema, result); - } catch (error) { - results.set(schema, { - schema, - status: "error", - applied: [], - error: error instanceof Error ? error : new Error(String(error)), - }); - } - }; - - await new Promise((resolve) => { - if (schemas.length === 0) { - resolve(); - return; - } - - let completed = 0; - - const next = () => { - while (active < concurrency && idx < schemas.length) { - const schema = schemas[idx++]; - if (!schema) break; - active++; - runOne(schema).then(() => { - active--; - completed++; - if (completed === schemas.length) { - resolve(); - } else { - next(); - } - }); - } - }; - - next(); - }); - - return results; -} - -export async function dryRun( - sql: SQL, - schema: string, - _config?: EngineConfig, -): Promise<{ pending: string[]; applied: string[] }> { - await assertEngineSchema(sql, schema); - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); - - // Check if migration table exists - const [{ exists }] = await sql` - select exists ( - select 1 - from information_schema.tables - where table_schema = ${schema} - and table_name = 'migration' - ) as exists - `; - - if (!exists) { - return { - pending: sorted.map((m) => m.name), - applied: [], - }; - } - - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - const appliedSet = new Set(rows.map((r: { name: string }) => r.name)); - const applied = sorted - .filter((m) => appliedSet.has(m.name)) - .map((m) => m.name); - const pending = sorted - .filter((m) => !appliedSet.has(m.name)) - .map((m) => m.name); - - return { pending, applied }; -} - -export function getMigrations(): ReadonlyArray<{ name: string }> { - return [...migrations] - .sort((a, b) => a.name.localeCompare(b.name)) - .map(({ name }) => ({ name })); -} - -export async function getVersion(sql: SQL, schema: string): Promise { - await assertEngineSchema(sql, schema); - const [row] = await sql.unsafe(`select version from ${schema}.version`); - return row.version; -} diff --git a/packages/engine/migrate/template.test.ts b/packages/engine/migrate/template.test.ts deleted file mode 100644 index b657622..0000000 --- a/packages/engine/migrate/template.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { defaultConfig, resolveConfig, template } from "./template"; - -describe("template function", () => { - test("replaces single variable", () => { - const sql = "CREATE TABLE foo (id {{type}})"; - const result = template(sql, { type: "UUID" }); - expect(result).toBe("CREATE TABLE foo (id UUID)"); - }); - - test("replaces multiple variables", () => { - const sql = "CREATE INDEX ON table USING {{method}} WITH (m = {{m}})"; - const result = template(sql, { method: "hnsw", m: 16 }); - expect(result).toBe("CREATE INDEX ON table USING hnsw WITH (m = 16)"); - }); - - test("replaces same variable multiple times", () => { - const sql = "{{var}} and {{var}} and {{var}}"; - const result = template(sql, { var: "test" }); - expect(result).toBe("test and test and test"); - }); - - test("throws on missing variable", () => { - const sql = "CREATE TABLE foo (x {{missing}})"; - expect(() => template(sql, {})).toThrow( - "Missing template variable: missing", - ); - }); - - test("handles numeric values", () => { - const sql = "WITH (m = {{value}})"; - const result = template(sql, { value: 1536 }); - expect(result).toBe("WITH (m = 1536)"); - }); - - test("handles decimal values", () => { - const sql = "WITH (k1 = {{k1}}, b = {{b}})"; - const result = template(sql, { k1: 1.2, b: 0.75 }); - expect(result).toBe("WITH (k1 = 1.2, b = 0.75)"); - }); - - test("handles boolean values", () => { - const sql = "SET enabled = {{enabled}}"; - const result = template(sql, { enabled: true }); - expect(result).toBe("SET enabled = true"); - }); - - test("handles empty string values", () => { - const sql = "SET value = '{{value}}'"; - const result = template(sql, { value: "" }); - expect(result).toBe("SET value = ''"); - }); - - test("handles variables with underscores", () => { - const sql = "halfvec({{embedding_dimensions}})"; - const result = template(sql, { embedding_dimensions: 768 }); - expect(result).toBe("halfvec(768)"); - }); - - test("handles variables with numbers", () => { - const sql = "WITH (k1 = {{bm25_k1}})"; - const result = template(sql, { bm25_k1: 1.2 }); - expect(result).toBe("WITH (k1 = 1.2)"); - }); - - test("preserves text outside of variables", () => { - const sql = "CREATE INDEX idx ON table USING {{method}} (column)"; - const result = template(sql, { method: "btree" }); - expect(result).toBe("CREATE INDEX idx ON table USING btree (column)"); - }); - - test("handles variables at start of string", () => { - const sql = "{{type}} NOT NULL"; - const result = template(sql, { type: "UUID" }); - expect(result).toBe("UUID NOT NULL"); - }); - - test("handles variables at end of string", () => { - const sql = "CREATE TYPE {{type}}"; - const result = template(sql, { type: "custom" }); - expect(result).toBe("CREATE TYPE custom"); - }); - - test("handles no variables", () => { - const sql = "CREATE TABLE foo (id UUID)"; - const result = template(sql, {}); - expect(result).toBe("CREATE TABLE foo (id UUID)"); - }); - - test("handles real migration template", () => { - const sql = "halfvec({{embedding_dimensions}})"; - const result = template(sql, { embedding_dimensions: 1536 }); - expect(result).toBe("halfvec(1536)"); - }); - - test("handles BM25 index template", () => { - const sql = - "with (text_config = '{{bm25_text_config}}', k1 = {{bm25_k1}}, b = {{bm25_b}})"; - const result = template(sql, { - bm25_text_config: "english", - bm25_k1: 1.2, - bm25_b: 0.75, - }); - expect(result).toBe("with (text_config = 'english', k1 = 1.2, b = 0.75)"); - }); - - test("handles HNSW index template", () => { - const sql = - "with (m = {{hnsw_m}}, ef_construction = {{hnsw_ef_construction}})"; - const result = template(sql, { - hnsw_m: 16, - hnsw_ef_construction: 64, - }); - expect(result).toBe("with (m = 16, ef_construction = 64)"); - }); - - test("handles schema variable substitution", () => { - const sql = "CREATE TABLE {{schema}}.memory (id uuid)"; - const result = template(sql, { schema: "me_abc123def456" }); - expect(result).toBe("CREATE TABLE me_abc123def456.memory (id uuid)"); - }); -}); - -describe("config merging", () => { - test("merging empty config uses defaults", () => { - const resolved = resolveConfig("me_test123test"); - expect(resolved.embedding_dimensions).toBe(1536); - expect(resolved.bm25_text_config).toBe("english"); - expect(resolved.bm25_k1).toBe(1.2); - expect(resolved.bm25_b).toBe(0.75); - expect(resolved.hnsw_m).toBe(16); - expect(resolved.hnsw_ef_construction).toBe(64); - expect(resolved.schema).toBe("me_test123test"); - }); - - test("partial override only changes specified values", () => { - const resolved = resolveConfig("me_test123test", { - embedding_dimensions: 768, - }); - expect(resolved.embedding_dimensions).toBe(768); - expect(resolved.bm25_text_config).toBe("english"); - }); - - test("multiple overrides work correctly", () => { - const resolved = resolveConfig("me_test123test", { - embedding_dimensions: 384, - bm25_text_config: "simple", - hnsw_m: 32, - }); - expect(resolved.embedding_dimensions).toBe(384); - expect(resolved.bm25_text_config).toBe("simple"); - expect(resolved.hnsw_m).toBe(32); - expect(resolved.bm25_k1).toBe(1.2); - expect(resolved.bm25_b).toBe(0.75); - }); - - test("numeric config values preserved as numbers", () => { - const resolved = resolveConfig("me_test123test", { - bm25_k1: 2.5, - bm25_b: 0.9, - }); - expect(typeof resolved.bm25_k1).toBe("number"); - expect(typeof resolved.bm25_b).toBe("number"); - expect(resolved.bm25_k1).toBe(2.5); - expect(resolved.bm25_b).toBe(0.9); - }); - - test("schema is always set from argument", () => { - const resolved = resolveConfig("me_abc123def456", { - embedding_dimensions: 768, - }); - expect(resolved.schema).toBe("me_abc123def456"); - }); -}); - -describe("defaultConfig", () => { - test("has all required fields", () => { - expect(defaultConfig.embedding_dimensions).toBe(1536); - expect(defaultConfig.bm25_text_config).toBe("english"); - expect(defaultConfig.bm25_k1).toBe(1.2); - expect(defaultConfig.bm25_b).toBe(0.75); - expect(defaultConfig.hnsw_m).toBe(16); - expect(defaultConfig.hnsw_ef_construction).toBe(64); - }); -}); diff --git a/packages/engine/migrate/template.ts b/packages/engine/migrate/template.ts deleted file mode 100644 index f3ffa1d..0000000 --- a/packages/engine/migrate/template.ts +++ /dev/null @@ -1,38 +0,0 @@ -export function template(sql: string, vars: Record): string { - return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { - if (!(key in vars)) { - throw new Error(`Missing template variable: ${key}`); - } - return String(vars[key]); - }); -} - -// Global index/search configuration — same for all engines in a database. -// Schema is not included here because it's per-engine, not per-database. -export interface EngineConfig { - embedding_dimensions?: number; - bm25_text_config?: string; - bm25_k1?: number; - bm25_b?: number; - hnsw_m?: number; - hnsw_ef_construction?: number; -} - -// All defaults filled in + per-engine schema attached. Used internally by template(). -export type ResolvedConfig = Required & { schema: string }; - -export const defaultConfig: Required = { - embedding_dimensions: 1536, - bm25_text_config: "english", - bm25_k1: 1.2, - bm25_b: 0.75, - hnsw_m: 16, - hnsw_ef_construction: 64, -}; - -export function resolveConfig( - schema: string, - config?: EngineConfig, -): ResolvedConfig { - return { ...defaultConfig, ...config, schema }; -} diff --git a/packages/engine/migrate/test-utils.ts b/packages/engine/migrate/test-utils.ts deleted file mode 100644 index a24648f..0000000 --- a/packages/engine/migrate/test-utils.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { SQL } from "bun"; - -function assertSafeIdentifier(name: string): void { - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { - throw new Error(`Unsafe database identifier: ${name}`); - } -} - -export class TestDatabase { - private dbName: string | null = null; - private readonly adminUrl: string; - - constructor(adminUrl = "postgresql://postgres@localhost:5432/postgres") { - this.adminUrl = adminUrl; - } - - async create(): Promise { - this.dbName = `test_me_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - - assertSafeIdentifier(this.dbName); - const sql = new SQL(this.adminUrl); - try { - await sql.unsafe(`create database ${this.dbName}`); - } finally { - await sql.close(); - } - - const url = new URL(this.adminUrl); - url.pathname = `/${this.dbName}`; - return url.toString(); - } - - async drop(): Promise { - if (!this.dbName) { - return; - } - - assertSafeIdentifier(this.dbName); - const sql = new SQL(this.adminUrl); - try { - await sql` - select pg_terminate_backend(pg_stat_activity.pid) - from pg_stat_activity - where pg_stat_activity.datname = ${this.dbName} - and pid <> pg_backend_pid() - `; - - await sql.unsafe(`drop database if exists ${this.dbName}`); - } finally { - await sql.close(); - this.dbName = null; - } - } -} - -export async function getAppliedMigrations( - sql: SQL, - schema: string, -): Promise { - const rows = await sql.unsafe( - `select name from ${schema}.migration order by name`, - ); - return rows.map((r: { name: string }) => r.name); -} - -export async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 - from information_schema.tables - where table_schema = ${schema} - and table_name = ${table} - ) as exists - `; - return row.exists; -} - -export async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 - from information_schema.schemata - where schema_name = ${name} - ) as exists - `; - return row.exists; -} - -export async function countMigrations( - sql: SQL, - schema: string, -): Promise { - const [row] = await sql.unsafe( - `select count(*)::int as count from ${schema}.migration`, - ); - return row.count; -} - -export async function getTableColumns( - sql: SQL, - schema: string, - table: string, -): Promise< - Array<{ column_name: string; data_type: string; is_nullable: string }> -> { - return await sql` - select column_name, data_type, is_nullable - from information_schema.columns - where table_schema = ${schema} - and table_name = ${table} - order by ordinal_position - `; -} - -export async function getIndexes( - sql: SQL, - schema: string, - table: string, -): Promise { - const rows = await sql` - select indexname - from pg_indexes - where schemaname = ${schema} - and tablename = ${table} - order by indexname - `; - return rows.map((r: { indexname: string }) => r.indexname); -} - -export async function getRoles( - sql: SQL, - ...names: string[] -): Promise> { - const pgArray = `{${names.join(",")}}`; - return await sql.unsafe( - `select rolname, rolcanlogin - from pg_roles - where rolname = any($1::text[]) - order by rolname`, - [pgArray], - ); -} - -export async function getFunctions( - sql: SQL, - schema: string, -): Promise> { - return await sql` - select p.proname, p.proconfig - from pg_proc p - join pg_namespace n on n.oid = p.pronamespace - where n.nspname = ${schema} - order by p.proname - `; -} diff --git a/packages/engine/ops/_tx.ts b/packages/engine/ops/_tx.ts deleted file mode 100644 index a62e0a2..0000000 --- a/packages/engine/ops/_tx.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { span } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import type { OpsContext } from "../types"; - -/** - * Transaction modes: - * - "read": Sets ROLE me_ro (read-only, RLS enforced) - * - "write": Sets ROLE me_rw (read-write, RLS enforced) - * - "admin": No role change (runs as connection owner, bypasses RLS) - * - * Admin mode is used for auth operations (principal, api_key, grant, etc.) - * which are not protected by RLS — only the memory table has RLS policies. - */ -type TransactionMode = "read" | "write" | "admin"; - -const ROLE_MAP = { - read: "me_ro", - write: "me_rw", - admin: null, // No role change -} as const; - -export interface EngineTimeouts { - statementTimeout: string; - lockTimeout: string; - transactionTimeout: string; - idleInTransactionSessionTimeout: string; -} - -export const DEFAULT_ENGINE_TIMEOUTS: EngineTimeouts = { - statementTimeout: process.env.ENGINE_STATEMENT_TIMEOUT ?? "25s", - lockTimeout: process.env.ENGINE_LOCK_TIMEOUT ?? "5s", - transactionTimeout: process.env.ENGINE_TRANSACTION_TIMEOUT ?? "30s", - idleInTransactionSessionTimeout: - process.env.ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? "30s", -}; - -/** - * Bound engine queries so production failures surface before clients give up. - * Uses transaction-local GUCs so pooled connections do not retain settings. - */ -export async function setLocalEngineTimeouts( - sql: SQL, - timeouts: EngineTimeouts = DEFAULT_ENGINE_TIMEOUTS, -): Promise { - await sql.unsafe("SELECT set_config('statement_timeout', $1, true)", [ - timeouts.statementTimeout, - ]); - await sql.unsafe("SELECT set_config('lock_timeout', $1, true)", [ - timeouts.lockTimeout, - ]); - await sql.unsafe("SELECT set_config('transaction_timeout', $1, true)", [ - timeouts.transactionTimeout, - ]); - await sql.unsafe( - "SELECT set_config('idle_in_transaction_session_timeout', $1, true)", - [timeouts.idleInTransactionSessionTimeout], - ); -} - -/** - * Execute a function within a transaction context. - * - * If already inside a transaction (ctx.inTransaction is true), just sets - * the appropriate role and runs the function on the existing handle. - * - * If not inside a transaction, opens a new one with: - * - SET LOCAL pgdog.shard (if shard is set, for future sharding) - * - SET LOCAL statement_timeout / lock_timeout / transaction timeouts - * - SET LOCAL search_path (insurance — all SQL is schema-qualified) - * - SET LOCAL ROLE (me_ro for read, me_rw for write) - * - set_config('me.user_id', ...) (for RLS) - */ -export async function withTx( - ctx: OpsContext, - mode: TransactionMode, - operation: string, - fn: (sql: SQL) => Promise, -): Promise { - const role = ROLE_MAP[mode]; - - if (ctx.inTransaction) { - // Already in a transaction — set role (if not admin) and run directly - if (role) { - await ctx.sql.unsafe(`SET LOCAL ROLE ${role}`); - } - return fn(ctx.sql); - } - - // Open new transaction with telemetry span - return span(`db.${operation}`, { - attributes: { - "db.schema": ctx.schema, - "db.mode": mode, - "db.role": role ?? "owner", - "db.operation": operation, - }, - callback: () => - ctx.sql.begin(async (tx) => { - // Future: pgDog shard routing - if (ctx.shard !== undefined) { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${ctx.shard}`); - } - - await setLocalEngineTimeouts(tx); - - // Set search_path: engine schema first, then public (for extension types like ltree) - await tx.unsafe(`SET LOCAL search_path TO ${ctx.schema}, public`); - - // Set role for permission control (skip for admin mode) - if (role) { - await tx.unsafe(`SET LOCAL ROLE ${role}`); - } - - // Set user_id for RLS policies (only meaningful for read/write modes) - const userId = ctx.getUserId(); - if (userId && role) { - await tx`SELECT set_config('me.user_id', ${userId}, true)`; - } - - return fn(tx); - }), - }); -} - -/** - * Helper to create a derived OpsContext for use inside withTransaction() - */ -export function deriveContext(ctx: OpsContext, tx: SQL): OpsContext { - return { - ...ctx, - sql: tx, - inTransaction: true, - }; -} diff --git a/packages/engine/ops/api-key.ts b/packages/engine/ops/api-key.ts deleted file mode 100644 index 68843a7..0000000 --- a/packages/engine/ops/api-key.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { span } from "@pydantic/logfire-node"; -import type { - ApiKey, - CreateApiKeyParams, - CreateApiKeyResult, - OpsContext, - ValidateApiKeyResult, -} from "../types"; -import { - formatApiKey, - generateLookupId, - generateSecret, - hashSecret, - verifySecret, -} from "../util/api-key"; -import { withTx } from "./_tx"; - -// Row type from database -interface ApiKeyRow { - id: string; - user_id: string; - lookup_id: string; - key_hash: string; - name: string; - expires_at: Date | null; - created_at: Date; - revoked_at: Date | null; -} - -function rowToApiKey(row: ApiKeyRow): ApiKey { - return { - id: row.id, - userId: row.user_id, - lookupId: row.lookup_id, - name: row.name, - expiresAt: row.expires_at, - createdAt: row.created_at, - revokedAt: row.revoked_at, - }; -} - -export function apiKeyOps(ctx: OpsContext, engineSlug: string) { - const { schema } = ctx; - - return { - /** - * Create a new API key for a user - * Returns the full key string (only available at creation time) - */ - async createApiKey( - params: CreateApiKeyParams, - ): Promise { - const { userId, name, expiresAt = null } = params; - - const lookupId = generateLookupId(); - const secret = generateSecret(); - const keyHash = await hashSecret(secret); - const rawKey = formatApiKey(engineSlug, lookupId, secret); - - return withTx(ctx, "admin", "createApiKey", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.api_key - (user_id, lookup_id, key_hash, name, expires_at) - values - (${userId}, ${lookupId}, ${keyHash}, ${name}, ${expiresAt}) - returning id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create API key"); - } - return { - apiKey: rowToApiKey(row), - rawKey, - }; - }); - }, - - /** - * Validate an API key and return the user ID if valid - */ - async validateApiKey( - lookupId: string, - secret: string, - ): Promise { - return withTx(ctx, "admin", "validateApiKey", async (sql) => { - const row = await span("db.api_key.lookup", { - attributes: { - "db.schema": schema, - "engine.slug": engineSlug, - "api_key.lookup_id": lookupId, - }, - callback: async () => { - const [apiKey] = await sql` - select id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - from ${sql.unsafe(schema)}.api_key - where lookup_id = ${lookupId} - `; - return apiKey; - }, - }); - - if (!row) { - return { valid: false, error: "API key not found" }; - } - - if (row.revoked_at) { - return { valid: false, error: "API key has been revoked" }; - } - - if (row.expires_at && row.expires_at < new Date()) { - return { valid: false, error: "API key has expired" }; - } - - const secretValid = await span("auth.api_key.verify_secret", { - attributes: { - "db.schema": schema, - "engine.slug": engineSlug, - "api_key.id": row.id, - "api_key.lookup_id": lookupId, - }, - callback: () => verifySecret(secret, row.key_hash), - }); - if (!secretValid) { - return { valid: false, error: "Invalid API key secret" }; - } - - return { - valid: true, - userId: row.user_id, - apiKeyId: row.id, - }; - }); - }, - - /** - * Get an API key by ID (without the secret/hash) - */ - async getApiKey(id: string): Promise { - return withTx(ctx, "admin", "getApiKey", async (sql) => { - const [row] = await sql` - select id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - from ${sql.unsafe(schema)}.api_key - where id = ${id} - `; - return row ? rowToApiKey(row) : null; - }); - }, - - /** - * List all API keys for a user - */ - async listApiKeys(userId: string): Promise { - return withTx(ctx, "admin", "listApiKeys", async (sql) => { - const rows = await sql` - select id, user_id, lookup_id, key_hash, name, expires_at, created_at, revoked_at - from ${sql.unsafe(schema)}.api_key - where user_id = ${userId} - order by created_at desc - `; - return rows.map(rowToApiKey); - }); - }, - - /** - * Revoke an API key - */ - async revokeApiKey(id: string): Promise { - return withTx(ctx, "admin", "revokeApiKey", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.api_key - set revoked_at = now() - where id = ${id} - and revoked_at is null - `; - return result.count > 0; - }); - }, - - /** - * Delete an API key permanently - */ - async deleteApiKey(id: string): Promise { - return withTx(ctx, "admin", "deleteApiKey", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.api_key - where id = ${id} - `; - return result.count > 0; - }); - }, - }; -} - -export type ApiKeyOps = ReturnType; diff --git a/packages/engine/ops/grant.ts b/packages/engine/ops/grant.ts deleted file mode 100644 index a783afb..0000000 --- a/packages/engine/ops/grant.ts +++ /dev/null @@ -1,172 +0,0 @@ -import type { GrantTreeAccessParams, OpsContext, TreeGrant } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface TreeGrantRow { - id: string; - user_id: string; - user_name: string; - tree_path: string; - actions: string[]; - granted_by: string | null; - with_grant_option: boolean; - created_at: Date; -} - -function rowToTreeGrant(row: TreeGrantRow): TreeGrant { - return { - id: row.id, - userId: row.user_id, - userName: row.user_name, - treePath: row.tree_path, - actions: row.actions, - grantedBy: row.granted_by, - withGrantOption: row.with_grant_option, - createdAt: row.created_at, - }; -} - -export function grantOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Grant tree access to a user (upserts on user_id + tree_path) - */ - async grantTreeAccess(params: GrantTreeAccessParams): Promise { - const { - userId, - treePath, - actions, - grantedBy = null, - withGrantOption = false, - } = params; - - await withTx(ctx, "admin", "grantTreeAccess", async (sql) => { - // Format actions as PostgreSQL array literal - const actionsArray = `{${actions.join(",")}}`; - await sql` - insert into ${sql.unsafe(schema)}.tree_grant - (user_id, tree_path, actions, granted_by, with_grant_option) - values - (${userId}, ${treePath}::ltree, ${actionsArray}::text[], ${grantedBy}, ${withGrantOption}) - on conflict (user_id, tree_path) - do update set - actions = excluded.actions, - granted_by = excluded.granted_by, - with_grant_option = excluded.with_grant_option - `; - }); - }, - - /** - * Revoke tree access from a user - */ - async revokeTreeAccess(userId: string, treePath: string): Promise { - return withTx(ctx, "admin", "revokeTreeAccess", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.tree_grant - where user_id = ${userId} - and tree_path = ${treePath}::ltree - `; - return result.count > 0; - }); - }, - - /** - * List tree grants, optionally filtered by user - */ - async listTreeGrants(userId?: string): Promise { - return withTx(ctx, "admin", "listTreeGrants", async (sql) => { - if (userId) { - const rows = await sql` - select g.id, g.user_id, u.name as user_name, g.tree_path::text, g.actions, g.granted_by, g.with_grant_option, g.created_at - from ${sql.unsafe(schema)}.tree_grant g - join ${sql.unsafe(schema)}."user" u on u.id = g.user_id - where g.user_id = ${userId} - order by g.tree_path - `; - return rows.map(rowToTreeGrant); - } - - const rows = await sql` - select g.id, g.user_id, u.name as user_name, g.tree_path::text, g.actions, g.granted_by, g.with_grant_option, g.created_at - from ${sql.unsafe(schema)}.tree_grant g - join ${sql.unsafe(schema)}."user" u on u.id = g.user_id - order by u.name, g.tree_path - `; - return rows.map(rowToTreeGrant); - }); - }, - - /** - * Get a specific grant by user and tree path - */ - async getTreeGrant( - userId: string, - treePath: string, - ): Promise { - return withTx(ctx, "admin", "getTreeGrant", async (sql) => { - const rows = await sql` - select g.id, g.user_id, u.name as user_name, g.tree_path::text, g.actions, g.granted_by, g.with_grant_option, g.created_at - from ${sql.unsafe(schema)}.tree_grant g - join ${sql.unsafe(schema)}."user" u on u.id = g.user_id - where g.user_id = ${userId} - and g.tree_path = ${treePath}::ltree - `; - const row = rows[0]; - return row ? rowToTreeGrant(row) : null; - }); - }, - - /** - * Check if a user has access to a tree path for a given action - * Uses the database's tree_access function (includes role inheritance) - */ - async checkTreeAccess( - userId: string, - treePath: string, - action: string, - ): Promise { - return withTx(ctx, "admin", "checkTreeAccess", async (sql) => { - const rows = await sql<{ allowed: boolean }[]>` - select exists( - select 1 - from ${sql.unsafe(schema)}.tree_access( - ${userId}::uuid, - ${action} - ) ta(tree_path) - where ${treePath}::ltree <@ ta.tree_path - ) as allowed - `; - return rows[0]?.allowed ?? false; - }); - }, - - /** - * Check if a user has grant option for a tree path and actions - */ - async hasGrantOption( - userId: string, - treePath: string, - actions: string[], - ): Promise { - return withTx(ctx, "admin", "hasGrantOption", async (sql) => { - const actionsArray = `{${actions.join(",")}}`; - const rows = await sql<{ has_option: boolean }[]>` - select exists ( - select 1 - from ${sql.unsafe(schema)}.tree_grant - where user_id = ${userId} - and ${treePath}::ltree <@ tree_path - and with_grant_option = true - and actions @> ${actionsArray}::text[] - ) as has_option - `; - return rows[0]?.has_option ?? false; - }); - }, - }; -} - -export type GrantOps = ReturnType; diff --git a/packages/engine/ops/index.ts b/packages/engine/ops/index.ts deleted file mode 100644 index 5ab1f57..0000000 --- a/packages/engine/ops/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { deriveContext, withTx } from "./_tx"; -export { type ApiKeyOps, apiKeyOps } from "./api-key"; -export { type GrantOps, grantOps } from "./grant"; -export { type MemoryOps, memoryOps } from "./memory"; -export { type OwnerOps, ownerOps } from "./owner"; -export { type RoleOps, roleOps } from "./role"; -export { type UserOps, userOps } from "./user"; diff --git a/packages/engine/ops/memory.test.ts b/packages/engine/ops/memory.test.ts deleted file mode 100644 index 9a69101..0000000 --- a/packages/engine/ops/memory.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { detectTreeFilterType, rrfFusion } from "./memory"; - -describe("detectTreeFilterType", () => { - test("detects ltxtquery (contains &)", () => { - expect(detectTreeFilterType("api & v2")).toBe("ltxtquery"); - expect(detectTreeFilterType("work & !draft")).toBe("ltxtquery"); - expect(detectTreeFilterType("a & b & c")).toBe("ltxtquery"); - }); - - test("detects lquery (contains pattern chars)", () => { - // Wildcard - expect(detectTreeFilterType("work.*")).toBe("lquery"); - expect(detectTreeFilterType("*.api.*")).toBe("lquery"); - - // Quantifier - expect(detectTreeFilterType("work.*{2}")).toBe("lquery"); - expect(detectTreeFilterType("work.*{1,3}")).toBe("lquery"); - - // Negation - expect(detectTreeFilterType("*.!draft.*")).toBe("lquery"); - - // Alternation - expect(detectTreeFilterType("work|personal.*")).toBe("lquery"); - - // Other pattern chars - expect(detectTreeFilterType("work.@api")).toBe("lquery"); - expect(detectTreeFilterType("work.%")).toBe("lquery"); - }); - - test("detects ltree (plain paths)", () => { - expect(detectTreeFilterType("work")).toBe("ltree"); - expect(detectTreeFilterType("work.projects")).toBe("ltree"); - expect(detectTreeFilterType("work.projects.api")).toBe("ltree"); - expect(detectTreeFilterType("a.b.c.d.e")).toBe("ltree"); - expect(detectTreeFilterType("")).toBe("ltree"); - }); -}); - -describe("rrfFusion", () => { - test("combines results from both sources", () => { - const bm25Results = [{ id: "a" }, { id: "b" }, { id: "c" }]; - const semanticResults = [{ id: "b" }, { id: "d" }, { id: "a" }]; - - const fused = rrfFusion(bm25Results, semanticResults); - - // All unique IDs should be present - const ids = fused.map((r) => r.id); - expect(ids).toContain("a"); - expect(ids).toContain("b"); - expect(ids).toContain("c"); - expect(ids).toContain("d"); - expect(ids).toHaveLength(4); - }); - - test("ranks items appearing in both lists higher", () => { - const bm25Results = [{ id: "a" }, { id: "b" }]; - const semanticResults = [{ id: "b" }, { id: "c" }]; - - const fused = rrfFusion(bm25Results, semanticResults); - - // 'b' appears in both, should have highest score - expect(fused[0]!.id).toBe("b"); - }); - - test("respects rank order within each list", () => { - const bm25Results = [{ id: "a" }, { id: "b" }, { id: "c" }]; - const semanticResults: Array<{ id: string }> = []; - - const fused = rrfFusion(bm25Results, semanticResults); - - // Order should be preserved from BM25 - expect(fused[0]!.id).toBe("a"); - expect(fused[1]!.id).toBe("b"); - expect(fused[2]!.id).toBe("c"); - }); - - test("applies weights correctly", () => { - const bm25Results = [{ id: "a" }]; - const semanticResults = [{ id: "b" }]; - - // With equal weights - const fusedEqual = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: 1.0, - semantic: 1.0, - }); - expect(fusedEqual[0]!.score).toBe(fusedEqual[1]!.score); - - // With higher BM25 weight - const fusedBM25Heavy = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: 2.0, - semantic: 1.0, - }); - const aScore = fusedBM25Heavy.find((r) => r.id === "a")!.score; - const bScore = fusedBM25Heavy.find((r) => r.id === "b")!.score; - expect(aScore).toBeGreaterThan(bScore); - - // With higher semantic weight - const fusedSemanticHeavy = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: 1.0, - semantic: 2.0, - }); - const aScore2 = fusedSemanticHeavy.find((r) => r.id === "a")!.score; - const bScore2 = fusedSemanticHeavy.find((r) => r.id === "b")!.score; - expect(bScore2).toBeGreaterThan(aScore2); - }); - - test("handles empty inputs", () => { - expect(rrfFusion([], [])).toEqual([]); - expect(rrfFusion([{ id: "a" }], [])).toHaveLength(1); - expect(rrfFusion([], [{ id: "b" }])).toHaveLength(1); - }); - - test("uses k parameter correctly", () => { - const bm25Results = [{ id: "a" }]; - const semanticResults: Array<{ id: string }> = []; - - // RRF score = weight / (k + rank) - // With k=60, rank=1: score = 1 / 61 ≈ 0.0164 - const fusedK60 = rrfFusion(bm25Results, semanticResults, 60); - expect(fusedK60[0]!.score).toBeCloseTo(1 / 61, 5); - - // With k=10, rank=1: score = 1 / 11 ≈ 0.0909 - const fusedK10 = rrfFusion(bm25Results, semanticResults, 10); - expect(fusedK10[0]!.score).toBeCloseTo(1 / 11, 5); - }); -}); diff --git a/packages/engine/ops/memory.ts b/packages/engine/ops/memory.ts deleted file mode 100644 index fa30083..0000000 --- a/packages/engine/ops/memory.ts +++ /dev/null @@ -1,778 +0,0 @@ -import type { SQL } from "bun"; -import type { - CreateMemoryParams, - GetTreeParams, - Memory, - OpsContext, - SearchParams, - SearchResult, - SearchResultItem, - TemporalFilter, - TreeNode, - UpdateMemoryParams, -} from "../types"; -import { withTx } from "./_tx"; - -// ============================================================================= -// Row Types -// ============================================================================= - -interface MemoryRow { - id: string; - content: string; - meta: Record; - tree: string; - temporal: string | null; - has_embedding: boolean; - created_at: Date; - created_by: string | null; - updated_at: Date | null; -} - -interface SearchRow extends MemoryRow { - score: number | string; // Postgres may return as string -} - -interface TreeRow { - path: string; - count: number; -} - -// ============================================================================= -// Tree Filter Detection -// ============================================================================= - -type TreeFilterType = "ltree" | "lquery" | "ltxtquery"; - -/** - * Detect the type of tree filter based on the pattern. - * - ltxtquery: Contains & (label search with AND) - * - lquery: Contains pattern characters (*, {}, !, |, @, %) - * - ltree: Plain path (default) - */ -function detectTreeFilterType(value: string): TreeFilterType { - if (value.includes("&")) return "ltxtquery"; - if (/[*{}!|@%]/.test(value)) return "lquery"; - return "ltree"; -} - -// ============================================================================= -// RRF Fusion -// ============================================================================= - -interface RankedResult { - id: string; - score: number; -} - -/** - * Reciprocal Rank Fusion (RRF) algorithm for combining search results. - * - * RRF score = Σ (weight / (k + rank)) where rank is 1-indexed. - * - * @param bm25Results - Results from BM25 full-text search (ordered by relevance) - * @param semanticResults - Results from semantic/vector search (ordered by relevance) - * @param k - RRF constant (default 60, prevents high-ranked items from dominating) - * @param weights - Relative weights for each search type - */ -function rrfFusion( - bm25Results: Array<{ id: string }>, - semanticResults: Array<{ id: string }>, - k = 60, - weights = { fulltext: 1.0, semantic: 1.0 }, -): RankedResult[] { - const scores = new Map(); - - // Score from BM25 results - bm25Results.forEach((result, index) => { - const rank = index + 1; - const score = weights.fulltext / (k + rank); - scores.set(result.id, (scores.get(result.id) ?? 0) + score); - }); - - // Score from semantic results - semanticResults.forEach((result, index) => { - const rank = index + 1; - const score = weights.semantic / (k + rank); - scores.set(result.id, (scores.get(result.id) ?? 0) + score); - }); - - // Sort by combined RRF score (descending) - return Array.from(scores.entries()) - .map(([id, score]) => ({ id, score })) - .sort((a, b) => b.score - a.score); -} - -// ============================================================================= -// Temporal Parsing/Formatting -// ============================================================================= - -/** - * Parse a PostgreSQL tstzrange string into a temporal object - */ -function parseTemporal( - range: string | null, -): { start: Date; end: Date } | null { - if (!range) { - return null; - } - - // Parse format like ["2024-01-01 00:00:00+00","2024-01-02 00:00:00+00") - const match = range.match(/[[(]"?([^",]+)"?,"?([^",\])]+)"?[\])]/); - if (!match) { - return null; - } - - const startStr = match[1]; - const endStr = match[2]; - if (!startStr || !endStr) { - return null; - } - return { - start: new Date(startStr), - end: new Date(endStr), - }; -} - -/** - * Format a temporal object as a PostgreSQL tstzrange string - */ -function formatTemporal( - temporal: { start: Date; end?: Date } | null | undefined, -): string | null { - if (!temporal) { - return null; - } - - const start = temporal.start.toISOString(); - const end = temporal.end?.toISOString() ?? start; - - // Point-in-time: [same,same] (inclusive both ends) - // Range: [start,end) (inclusive-exclusive) - if (start === end) { - return `[${start},${end}]`; - } - return `[${start},${end})`; -} - -// ============================================================================= -// Row Conversion -// ============================================================================= - -function rowToMemory(row: MemoryRow): Memory { - return { - id: row.id, - content: row.content, - meta: row.meta, - tree: row.tree, - temporal: parseTemporal(row.temporal), - hasEmbedding: row.has_embedding, - createdAt: row.created_at, - createdBy: row.created_by, - updatedAt: row.updated_at, - }; -} - -function rowToSearchResult(row: SearchRow): SearchResultItem { - return { - ...rowToMemory(row), - score: typeof row.score === "string" ? parseFloat(row.score) : row.score, - }; -} - -// ============================================================================= -// Query Builders -// ============================================================================= - -interface FilterParams { - meta?: Record; - tree?: string; - temporal?: TemporalFilter; - grep?: string; -} - -/** - * Build common filter clauses for WHERE conditions - */ -function buildCommonFilters( - params: FilterParams, - valueOffset: number, -): { clauses: string[]; values: unknown[] } { - const clauses: string[] = []; - const values: unknown[] = []; - - // Metadata containment filter - if (params.meta && Object.keys(params.meta).length > 0) { - const paramIdx = valueOffset + values.length + 1; - clauses.push(`meta @> $${paramIdx}`); - values.push(params.meta); - } - - // Tree filter with auto-detection - if (params.tree) { - const paramIdx = valueOffset + values.length + 1; - const filterType = detectTreeFilterType(params.tree); - switch (filterType) { - case "ltxtquery": - clauses.push(`tree @ $${paramIdx}::ltxtquery`); - break; - case "lquery": - clauses.push(`tree ~ $${paramIdx}::lquery`); - break; - case "ltree": - clauses.push(`tree <@ $${paramIdx}::ltree`); - break; - } - values.push(params.tree); - } - - // Temporal filter - if (params.temporal) { - if (params.temporal.contains !== undefined) { - const paramIdx = valueOffset + values.length + 1; - clauses.push(`temporal @> $${paramIdx}::timestamptz`); - const ts = - params.temporal.contains instanceof Date - ? params.temporal.contains.toISOString() - : params.temporal.contains; - values.push(ts); - } else if (params.temporal.overlaps) { - const paramIdx1 = valueOffset + values.length + 1; - const paramIdx2 = valueOffset + values.length + 2; - clauses.push( - `temporal && tstzrange($${paramIdx1}::timestamptz, $${paramIdx2}::timestamptz, '[)')`, - ); - const [start, end] = params.temporal.overlaps; - values.push( - start instanceof Date ? start.toISOString() : start, - end instanceof Date ? end.toISOString() : end, - ); - } else if (params.temporal.within) { - const paramIdx1 = valueOffset + values.length + 1; - const paramIdx2 = valueOffset + values.length + 2; - clauses.push( - `temporal <@ tstzrange($${paramIdx1}::timestamptz, $${paramIdx2}::timestamptz, '[)')`, - ); - const [start, end] = params.temporal.within; - values.push( - start instanceof Date ? start.toISOString() : start, - end instanceof Date ? end.toISOString() : end, - ); - } - } - - // Content regex filter (POSIX, case-insensitive) - if (params.grep) { - const paramIdx = valueOffset + values.length + 1; - clauses.push(`content ~* $${paramIdx}`); - values.push(params.grep); - } - - return { clauses, values }; -} - -/** - * Build a BM25 full-text search query - */ -async function buildBM25Query( - sql: SQL, - schema: string, - params: FilterParams & { - query: string; - limit: number; - }, -): Promise { - const indexName = `${schema}.memory_content_bm25_idx`; - const { clauses, values } = buildCommonFilters(params, 2); // $1=query, $2=limit - - const whereClause = clauses.length > 0 ? `AND ${clauses.join(" AND ")}` : ""; - - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at, - -(content <@> to_bm25query($1, '${indexName}')) as score - FROM ${schema}.memory - WHERE content <@> to_bm25query($1, '${indexName}') < 0 - ${whereClause} - ORDER BY score DESC, created_at DESC - LIMIT $2 - `; - - return sql.unsafe(query, [ - params.query, - params.limit, - ...values, - ]); -} - -/** - * Build a semantic/vector similarity search query - */ -async function buildSemanticQuery( - sql: SQL, - schema: string, - params: FilterParams & { - embedding: number[]; - limit: number; - semanticThreshold?: number; - }, -): Promise { - const hasSemanticThreshold = typeof params.semanticThreshold === "number"; - const { clauses, values } = buildCommonFilters( - params, - hasSemanticThreshold ? 3 : 2, - ); // $1=embedding, $2=limit, optional $3=semanticThreshold - - const semanticThresholdClause = hasSemanticThreshold - ? "AND (1 - (embedding <=> $1::halfvec)) >= $3" - : "AND (embedding <=> $1::halfvec) < 1.0"; - const whereClause = clauses.length > 0 ? `AND ${clauses.join(" AND ")}` : ""; - - // Format embedding as PostgreSQL array literal - const embeddingLiteral = `[${params.embedding.join(",")}]`; - - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at, - (1 - (embedding <=> $1::halfvec)) as score - FROM ${schema}.memory - WHERE embedding IS NOT NULL - ${semanticThresholdClause} - ${whereClause} - ORDER BY score DESC, created_at DESC - LIMIT $2 - `; - - return sql.unsafe(query, [ - embeddingLiteral, - params.limit, - ...(hasSemanticThreshold ? [params.semanticThreshold] : []), - ...values, - ]); -} - -/** - * Build a filter-only query (no search ranking) - */ -async function buildFilterQuery( - sql: SQL, - schema: string, - params: FilterParams & { - limit: number; - orderBy: "asc" | "desc"; - }, -): Promise { - const { clauses, values } = buildCommonFilters(params, 1); // $1=limit - - const whereClause = - clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""; - const orderDirection = params.orderBy === "asc" ? "ASC" : "DESC"; - - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at, - 1.0 as score - FROM ${schema}.memory - ${whereClause} - ORDER BY created_at ${orderDirection} - LIMIT $1 - `; - - return sql.unsafe(query, [params.limit, ...values]); -} - -/** - * Fetch full memory rows by IDs, preserving order - */ -async function fetchByIds( - sql: SQL, - schema: string, - ids: string[], -): Promise { - if (ids.length === 0) { - return []; - } - - // Use array position to preserve order - const idsArray = `{${ids.join(",")}}`; - const query = ` - SELECT - id, content, meta, tree::text, temporal::text, - embedding IS NOT NULL as has_embedding, - created_at, created_by, updated_at - FROM ${schema}.memory - WHERE id = ANY($1::uuid[]) - ORDER BY array_position($1::uuid[], id) - `; - - return sql.unsafe(query, [idsArray]); -} - -// ============================================================================= -// Memory Ops -// ============================================================================= - -export function memoryOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Create a new memory - */ - async createMemory(params: CreateMemoryParams): Promise { - const { id, content, meta = {}, tree = "", temporal, createdBy } = params; - - const temporalStr = formatTemporal(temporal); - - return withTx(ctx, "write", "createMemory", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}.memory - (${id ? sql`id,` : sql``} content, meta, tree, temporal, created_by) - values - (${id ? sql`${id},` : sql``} ${content}, ${meta}::jsonb, ${tree}::ltree, ${temporalStr}::tstzrange, ${createdBy ?? null}) - returning - id, content, meta, tree::text, temporal::text, - embedding is not null as has_embedding, - created_at, created_by, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create memory"); - } - return rowToMemory(row); - }); - }, - - /** - * Batch create memories. - * - * Inserts skip duplicates by id (`ON CONFLICT (id) DO NOTHING`) so that - * a duplicate id in a single batch — or a retry of a partially-applied - * batch — does not abort the surrounding transaction. The returned ids - * may therefore be shorter than `params` when conflicts occur. Callers - * that care which inputs landed should compare ids against the input. - * - * This is safe because callers typically supply deterministic ids (e.g. - * the importer derives UUIDv7 from a stable hash of the source message), - * so a conflict means "we already have this row" rather than "two - * unrelated callers raced on a generated id." Callers without - * deterministic ids let the column default produce fresh UUIDv7s and - * never collide. - */ - async batchCreateMemories(params: CreateMemoryParams[]): Promise { - if (params.length === 0) { - return []; - } - - return withTx(ctx, "write", "batchCreateMemories", async (sql) => { - // TODO: Optimize with multi-row VALUES when Bun.sql supports it better - const ids: string[] = []; - for (const p of params) { - const temporalStr = formatTemporal(p.temporal); - const rows = await sql<{ id: string }[]>` - insert into ${sql.unsafe(schema)}.memory - (${p.id ? sql`id,` : sql``} content, meta, tree, temporal, created_by) - values - (${p.id ? sql`${p.id},` : sql``} ${p.content}, ${p.meta ?? {}}::jsonb, ${p.tree ?? ""}::ltree, ${temporalStr}::tstzrange, ${p.createdBy ?? null}) - on conflict (id) do nothing - returning id - `; - const row = rows[0]; - if (row) { - ids.push(row.id); - } - } - return ids; - }); - }, - - /** - * Get a memory by ID - */ - async getMemory(id: string): Promise { - return withTx(ctx, "read", "getMemory", async (sql) => { - const rows = await sql` - select - id, content, meta, tree::text, temporal::text, - embedding is not null as has_embedding, - created_at, created_by, updated_at - from ${sql.unsafe(schema)}.memory - where id = ${id} - `; - const row = rows[0]; - return row ? rowToMemory(row) : null; - }); - }, - - /** - * Update a memory - */ - async updateMemory( - id: string, - params: UpdateMemoryParams, - ): Promise { - const { content, meta, tree, temporal } = params; - - const updates: string[] = []; - const values: unknown[] = []; - let paramIndex = 1; - - if (content !== undefined) { - updates.push(`content = $${paramIndex++}`); - values.push(content); - } - if (meta !== undefined) { - updates.push(`meta = $${paramIndex++}::jsonb`); - values.push(meta); - } - if (tree !== undefined) { - updates.push(`tree = $${paramIndex++}::ltree`); - values.push(tree); - } - if (temporal !== undefined) { - updates.push(`temporal = $${paramIndex++}::tstzrange`); - values.push(formatTemporal(temporal)); - } - - if (updates.length === 0) { - return this.getMemory(id); - } - - values.push(id); - - return withTx(ctx, "write", "updateMemory", async (sql) => { - const query = ` - update ${schema}.memory - set ${updates.join(", ")} - where id = $${paramIndex} - returning - id, content, meta, tree::text, temporal::text, - embedding is not null as has_embedding, - created_at, created_by, updated_at - `; - - const rows = await sql.unsafe(query, values); - const row = rows[0]; - return row ? rowToMemory(row) : null; - }); - }, - - /** - * Delete a memory by ID - */ - async deleteMemory(id: string): Promise { - return withTx(ctx, "write", "deleteMemory", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.memory - where id = ${id} - `; - return result.count > 0; - }); - }, - - /** - * Delete all memories under a tree path - */ - async deleteTree(treePath: string): Promise<{ count: number }> { - return withTx(ctx, "write", "deleteTree", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.memory - where tree <@ ${treePath}::ltree - `; - return { count: result.count }; - }); - }, - - /** - * Count memories under a tree path (unbounded). - * - * Used to preview the impact of `deleteTree` / `moveTree` without - * being limited by a search `limit`. - */ - async countTree(treePath: string): Promise<{ count: number }> { - return withTx(ctx, "read", "countTree", async (sql) => { - const rows = await sql` - select count(*)::int as count - from ${sql.unsafe(schema)}.memory - where tree <@ ${treePath}::ltree - `; - return { count: Number(rows[0]?.count ?? 0) }; - }); - }, - - /** - * Move memories from one tree path to another - */ - async moveTree( - source: string, - destination: string, - ): Promise<{ count: number }> { - return withTx(ctx, "write", "moveTree", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}.memory - set tree = case - when tree = ${source}::ltree then ${destination}::ltree - else ${destination}::ltree || subpath(tree, nlevel(${source}::ltree)) - end - where tree <@ ${source}::ltree - `; - return { count: result.count }; - }); - }, - - /** - * Search memories with hybrid BM25 + semantic search and RRF fusion. - * - * Search modes: - * 1. Hybrid (fulltext + embedding): Both searches run in parallel, results fused with RRF - * 2. BM25-only (fulltext): Full-text search using pg_textsearch - * 3. Semantic-only (embedding): Vector similarity search using pgvector - * 4. Filter-only (no search): Just filters by meta/tree/temporal - * - * Note: If `semantic` text is provided but no `embedding`, the caller is responsible - * for generating the embedding first. This keeps the embedding provider decoupled. - */ - async searchMemories(params: SearchParams): Promise { - const { - fulltext, - embedding, - grep, - meta, - tree, - temporal, - limit = 10, - candidateLimit = 30, - semanticThreshold, - weights = { fulltext: 1.0, semantic: 1.0 }, - orderBy = "desc", - } = params; - - return withTx(ctx, "read", "searchMemories", async (sql) => { - let results: SearchResultItem[]; - - if (fulltext && embedding && embedding.length > 0) { - // Case 1: Hybrid search with RRF fusion - const [bm25Results, semanticResults] = await Promise.all([ - buildBM25Query(sql, schema, { - query: fulltext, - grep, - meta, - tree, - temporal, - limit: candidateLimit, - }), - buildSemanticQuery(sql, schema, { - embedding, - grep, - meta, - tree, - temporal, - limit: candidateLimit, - semanticThreshold, - }), - ]); - - // Fuse results using RRF - const fusedResults = rrfFusion(bm25Results, semanticResults, 60, { - fulltext: weights.fulltext ?? 1.0, - semantic: weights.semantic ?? 1.0, - }); - - // Take top N and fetch full records - const topIds = fusedResults.slice(0, limit).map((r) => r.id); - const scoreMap = new Map(fusedResults.map((r) => [r.id, r.score])); - - const rows = await fetchByIds(sql, schema, topIds); - results = rows.map((row) => ({ - ...rowToMemory(row), - score: scoreMap.get(row.id) ?? 0, - })); - } else if (fulltext) { - // Case 2: BM25-only search - const rows = await buildBM25Query(sql, schema, { - query: fulltext, - grep, - meta, - tree, - temporal, - limit, - }); - results = rows.map(rowToSearchResult); - } else if (embedding && embedding.length > 0) { - // Case 3: Semantic-only search - const rows = await buildSemanticQuery(sql, schema, { - embedding, - grep, - meta, - tree, - temporal, - limit, - semanticThreshold, - }); - results = rows.map(rowToSearchResult); - } else { - // Case 4: Filter-only (no search ranking) - const rows = await buildFilterQuery(sql, schema, { - grep, - meta, - tree, - temporal, - limit, - orderBy, - }); - results = rows.map(rowToSearchResult); - } - - return { - results, - total: results.length, - limit, - }; - }); - }, - - /** - * Get the tree structure with counts - */ - async getTree(params?: GetTreeParams): Promise { - const { tree: rootPath, levels } = params ?? {}; - - return withTx(ctx, "read", "getTree", async (sql) => { - if (rootPath) { - const rows = await sql` - select subpath(tree, 0, nlevel(${rootPath}::ltree) + g.lvl)::text as path, count(*)::int as count - from ${sql.unsafe(schema)}.memory - cross join lateral generate_series(1, ${levels ?? 100}) as g(lvl) - where tree <@ ${rootPath}::ltree - and nlevel(tree) >= nlevel(${rootPath}::ltree) + g.lvl - group by 1 - order by 1 - `; - return rows.map((r) => ({ path: r.path, count: r.count })); - } - - const rows = await sql` - select subpath(tree, 0, g.lvl)::text as path, count(*)::int as count - from ${sql.unsafe(schema)}.memory - cross join lateral generate_series(1, ${levels ?? 100}) as g(lvl) - where nlevel(tree) >= g.lvl - and tree <> ''::ltree - group by 1 - order by 1 - `; - return rows.map((r) => ({ path: r.path, count: r.count })); - }); - }, - }; -} - -export type MemoryOps = ReturnType; - -// Export for testing -export { detectTreeFilterType, rrfFusion }; diff --git a/packages/engine/ops/owner.ts b/packages/engine/ops/owner.ts deleted file mode 100644 index bc700e1..0000000 --- a/packages/engine/ops/owner.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { OpsContext, TreeOwner } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface TreeOwnerRow { - tree_path: string; - user_id: string; - user_name: string; - created_by: string | null; - created_by_name: string | null; - created_at: Date; -} - -function rowToTreeOwner(row: TreeOwnerRow): TreeOwner { - return { - treePath: row.tree_path, - userId: row.user_id, - userName: row.user_name, - createdBy: row.created_by, - createdByName: row.created_by_name, - createdAt: row.created_at, - }; -} - -export function ownerOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Set tree owner (upserts on tree_path) - */ - async setTreeOwner( - userId: string, - treePath: string, - createdBy?: string, - ): Promise { - await withTx(ctx, "admin", "setTreeOwner", async (sql) => { - await sql` - insert into ${sql.unsafe(schema)}.tree_owner - (tree_path, user_id, created_by) - values - (${treePath}::ltree, ${userId}, ${createdBy ?? null}) - on conflict (tree_path) - do update set - user_id = excluded.user_id, - created_by = excluded.created_by - `; - }); - }, - - /** - * Remove tree owner - */ - async removeTreeOwner(treePath: string): Promise { - return withTx(ctx, "admin", "removeTreeOwner", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.tree_owner - where tree_path = ${treePath}::ltree - `; - return result.count > 0; - }); - }, - - /** - * Get tree owner by path - */ - async getTreeOwner(treePath: string): Promise { - return withTx(ctx, "admin", "getTreeOwner", async (sql) => { - const rows = await sql` - select o.tree_path::text, o.user_id, u.name as user_name, o.created_by, cb.name as created_by_name, o.created_at - from ${sql.unsafe(schema)}.tree_owner o - join ${sql.unsafe(schema)}."user" u on u.id = o.user_id - left join ${sql.unsafe(schema)}."user" cb on cb.id = o.created_by - where o.tree_path = ${treePath}::ltree - `; - const row = rows[0]; - return row ? rowToTreeOwner(row) : null; - }); - }, - - /** - * List tree owners, optionally filtered by user - */ - async listTreeOwners(userId?: string): Promise { - return withTx(ctx, "admin", "listTreeOwners", async (sql) => { - if (userId) { - const rows = await sql` - select o.tree_path::text, o.user_id, u.name as user_name, o.created_by, cb.name as created_by_name, o.created_at - from ${sql.unsafe(schema)}.tree_owner o - join ${sql.unsafe(schema)}."user" u on u.id = o.user_id - left join ${sql.unsafe(schema)}."user" cb on cb.id = o.created_by - where o.user_id = ${userId} - order by o.tree_path - `; - return rows.map(rowToTreeOwner); - } - - const rows = await sql` - select o.tree_path::text, o.user_id, u.name as user_name, o.created_by, cb.name as created_by_name, o.created_at - from ${sql.unsafe(schema)}.tree_owner o - join ${sql.unsafe(schema)}."user" u on u.id = o.user_id - left join ${sql.unsafe(schema)}."user" cb on cb.id = o.created_by - order by o.tree_path - `; - return rows.map(rowToTreeOwner); - }); - }, - - /** - * Check if a user owns a tree path (or any ancestor) - */ - async isOwnerOf(userId: string, treePath: string): Promise { - return withTx(ctx, "admin", "isOwnerOf", async (sql) => { - const rows = await sql<{ is_owner: boolean }[]>` - select exists ( - select 1 - from ${sql.unsafe(schema)}.tree_owner - where user_id = ${userId} - and ${treePath}::ltree <@ tree_path - ) as is_owner - `; - return rows[0]?.is_owner ?? false; - }); - }, - }; -} - -export type OwnerOps = ReturnType; diff --git a/packages/engine/ops/role.ts b/packages/engine/ops/role.ts deleted file mode 100644 index 3e59b70..0000000 --- a/packages/engine/ops/role.ts +++ /dev/null @@ -1,142 +0,0 @@ -import type { OpsContext, RoleInfo, RoleMember } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface RoleMemberRow { - role_id: string; - member_id: string; - member_name: string; - with_admin_option: boolean; - created_at: Date; -} - -interface RoleInfoRow { - id: string; - name: string; - with_admin_option: boolean; -} - -function rowToRoleMember(row: RoleMemberRow): RoleMember { - return { - roleId: row.role_id, - memberId: row.member_id, - memberName: row.member_name, - withAdminOption: row.with_admin_option, - createdAt: row.created_at, - }; -} - -function rowToRoleInfo(row: RoleInfoRow): RoleInfo { - return { - id: row.id, - name: row.name, - withAdminOption: row.with_admin_option, - }; -} - -export function roleOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Add a member to a role (with cycle detection) - */ - async addRoleMember( - roleId: string, - memberId: string, - withAdminOption = false, - ): Promise { - await withTx(ctx, "admin", "addRoleMember", async (sql) => { - // Check for cycles first - const cycleRows = await sql<{ would_cycle: boolean }[]>` - select ${sql.unsafe(schema)}.would_create_cycle( - ${roleId}::uuid, - ${memberId}::uuid - ) as would_cycle - `; - - if (cycleRows[0]?.would_cycle) { - throw new Error( - `Adding member ${memberId} to role ${roleId} would create a cycle`, - ); - } - - await sql` - insert into ${sql.unsafe(schema)}.role_membership - (role_id, member_id, with_admin_option) - values - (${roleId}, ${memberId}, ${withAdminOption}) - on conflict (role_id, member_id) - do update set - with_admin_option = excluded.with_admin_option - `; - }); - }, - - /** - * Remove a member from a role - */ - async removeRoleMember(roleId: string, memberId: string): Promise { - return withTx(ctx, "admin", "removeRoleMember", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}.role_membership - where role_id = ${roleId} - and member_id = ${memberId} - `; - return result.count > 0; - }); - }, - - /** - * List members of a role - */ - async listRoleMembers(roleId: string): Promise { - return withTx(ctx, "admin", "listRoleMembers", async (sql) => { - const rows = await sql` - select rm.role_id, rm.member_id, u.name as member_name, rm.with_admin_option, rm.created_at - from ${sql.unsafe(schema)}.role_membership rm - join ${sql.unsafe(schema)}."user" u on u.id = rm.member_id - where rm.role_id = ${roleId} - order by rm.created_at - `; - return rows.map(rowToRoleMember); - }); - }, - - /** - * List roles that a user is a member of - */ - async listRolesForUser(userId: string): Promise { - return withTx(ctx, "admin", "listRolesForUser", async (sql) => { - const rows = await sql` - select u.id, u.name, rm.with_admin_option - from ${sql.unsafe(schema)}.role_membership rm - join ${sql.unsafe(schema)}."user" u on u.id = rm.role_id - where rm.member_id = ${userId} - order by u.name - `; - return rows.map(rowToRoleInfo); - }); - }, - - /** - * Check if a user has admin option on a role - */ - async hasAdminOption(userId: string, roleId: string): Promise { - return withTx(ctx, "admin", "hasAdminOption", async (sql) => { - const rows = await sql<{ has_admin: boolean }[]>` - select exists ( - select 1 - from ${sql.unsafe(schema)}.role_membership - where role_id = ${roleId} - and member_id = ${userId} - and with_admin_option = true - ) as has_admin - `; - return rows[0]?.has_admin ?? false; - }); - }, - }; -} - -export type RoleOps = ReturnType; diff --git a/packages/engine/ops/user.ts b/packages/engine/ops/user.ts deleted file mode 100644 index 8bebd05..0000000 --- a/packages/engine/ops/user.ts +++ /dev/null @@ -1,193 +0,0 @@ -import type { CreateUserParams, OpsContext, User } from "../types"; -import { withTx } from "./_tx"; - -// Row type from database -interface UserRow { - id: string; - name: string; - identity_id: string | null; - can_login: boolean; - superuser: boolean; - createrole: boolean; - created_at: Date; - updated_at: Date | null; -} - -function rowToUser(row: UserRow): User { - return { - id: row.id, - name: row.name, - identityId: row.identity_id, - canLogin: row.can_login, - superuser: row.superuser, - createrole: row.createrole, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; -} - -export function userOps(ctx: OpsContext) { - const { schema } = ctx; - - return { - /** - * Create a new user - */ - async createUser(params: CreateUserParams): Promise { - const { - id, - name, - identityId = null, - canLogin = true, - superuser = false, - createrole = false, - } = params; - - return withTx(ctx, "admin", "createUser", async (sql) => { - const rows = await sql` - insert into ${sql.unsafe(schema)}."user" - (id, name, identity_id, can_login, superuser, createrole) - values - (${id ? sql`${id}::uuid` : sql`uuidv7()`}, ${name}, ${identityId}, ${canLogin}, ${superuser}, ${createrole}) - returning id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - `; - const row = rows[0]; - if (!row) { - throw new Error("Failed to create user"); - } - return rowToUser(row); - }); - }, - - /** - * Create a role (user with can_login = false) - */ - async createRole(name: string, identityId?: string | null): Promise { - return this.createUser({ - name, - identityId, - canLogin: false, - superuser: false, - }); - }, - - /** - * Create a superuser - */ - async createSuperuser( - name: string, - id?: string, - identityId?: string | null, - ): Promise { - return this.createUser({ - id, - name, - identityId, - canLogin: true, - superuser: true, - }); - }, - - /** - * Get a user by ID - */ - async getUser(id: string): Promise { - return withTx(ctx, "admin", "getUser", async (sql) => { - const [row] = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where id = ${id} - `; - return row ? rowToUser(row) : null; - }); - }, - - /** - * Get a user by name - */ - async getUserByName(name: string): Promise { - return withTx(ctx, "admin", "getUserByName", async (sql) => { - const [row] = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where name = ${name} - `; - return row ? rowToUser(row) : null; - }); - }, - - /** - * List all users (optionally filter by can_login) - */ - async listUsers(canLogin?: boolean): Promise { - return withTx(ctx, "admin", "listUsers", async (sql) => { - const rows = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - ${canLogin !== undefined ? sql`where can_login = ${canLogin}` : sql``} - order by created_at - `; - return rows.map(rowToUser); - }); - }, - - /** - * List users linked to a specific identity - */ - async listUsersByIdentity(identityId: string): Promise { - return withTx(ctx, "admin", "listUsersByIdentity", async (sql) => { - const rows = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where identity_id = ${identityId} - order by created_at - `; - return rows.map(rowToUser); - }); - }, - - /** - * Find a user by identity ID (returns first match or null) - */ - async getUserByIdentity(identityId: string): Promise { - return withTx(ctx, "admin", "getUserByIdentity", async (sql) => { - const [row] = await sql` - select id, name, identity_id, can_login, superuser, createrole, created_at, updated_at - from ${sql.unsafe(schema)}."user" - where identity_id = ${identityId} - limit 1 - `; - return row ? rowToUser(row) : null; - }); - }, - - /** - * Rename a user - */ - async renameUser(id: string, newName: string): Promise { - return withTx(ctx, "admin", "renameUser", async (sql) => { - const result = await sql` - update ${sql.unsafe(schema)}."user" - set name = ${newName} - where id = ${id} - `; - return result.count > 0; - }); - }, - - /** - * Delete a user - */ - async deleteUser(id: string): Promise { - return withTx(ctx, "admin", "deleteUser", async (sql) => { - const result = await sql` - delete from ${sql.unsafe(schema)}."user" - where id = ${id} - `; - return result.count > 0; - }); - }, - }; -} - -export type UserOps = ReturnType; diff --git a/packages/engine/package.json b/packages/engine/package.json index 5e214a9..39f044f 100644 --- a/packages/engine/package.json +++ b/packages/engine/package.json @@ -4,6 +4,8 @@ "private": true, "type": "module", "dependencies": { - "@pydantic/logfire-node": "^0.13.1" + "@memory.build/database": "workspace:*", + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9" } } diff --git a/packages/engine/space/db.integration.test.ts b/packages/engine/space/db.integration.test.ts new file mode 100644 index 0000000..56e8532 --- /dev/null +++ b/packages/engine/space/db.integration.test.ts @@ -0,0 +1,273 @@ +// Integration tests for the space data-plane TS layer (spaceStore). +// +// Provisions a throwaway metest_ schema via migrateSpace (small embedding +// dims for speed) and exercises the wrappers against the real SQL functions. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/engine/space/db.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateSpace } from "@memory.build/database"; +import postgres, { type Sql } from "postgres"; +import { type SpaceStore, spaceStore } from "./db"; +import type { TreeAccess } from "./types"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"; +const randomSlug = () => { + const bytes = crypto.getRandomValues(new Uint8Array(12)); + let s = ""; + for (const b of bytes) s += ALPHABET[b % 36]; + return s; +}; + +// Full owner access at "work"; all test memories live under work.* +const FULL: TreeAccess = [{ tree_path: "work", access: 3 }]; +const READONLY: TreeAccess = [{ tree_path: "work", access: 1 }]; + +let sql: Sql; +let schema: string; +let db: SpaceStore; + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + const slug = randomSlug(); + schema = `metest_${slug}`; + await migrateSpace(sql, { slug, schema, embeddingDimensions: 4 }); + db = spaceStore(sql, schema); +}); + +afterAll(async () => { + if (schema) await sql.unsafe(`drop schema if exists ${schema} cascade`); + await sql.end(); +}); + +/** Directly set a memory's embedding (simulating the worker). */ +async function setEmbedding(id: string, vec: number[]): Promise { + await sql.unsafe( + `update ${schema}.memory set embedding = $1::halfvec where id = $2`, + [`[${vec.join(",")}]`, id], + ); +} + +/** createMemory asserting a fresh insert happened (no skip/replace). */ +async function mustCreate( + access: TreeAccess, + params: Parameters[1], +): Promise { + const created = await db.createMemory(access, params); + if (created === null) throw new Error("unexpected duplicate-id skip"); + if (!created.inserted) throw new Error("unexpected replace"); + return created.id; +} + +test("createMemory + getMemory round-trips", async () => { + const id = await mustCreate(FULL, { + tree: "work.note", + content: "hello world", + meta: { kind: "note" }, + }); + const m = await db.getMemory(FULL, id); + expect(m?.id).toBe(id); + expect(m?.tree).toBe("work.note"); + expect(m?.content).toBe("hello world"); + expect(m?.meta).toEqual({ kind: "note" }); + expect(m?.hasEmbedding).toBe(false); +}); + +test("createMemory returns null for a duplicate explicit id", async () => { + const id = "01900000-0000-7000-8000-0000000000d0"; + const first = await db.createMemory(FULL, { + id, + tree: "work.dup", + content: "original", + }); + expect(first).toEqual({ id, inserted: true }); + + // Re-submitting the same id is a no-op skip, not an error. + const second = await db.createMemory(FULL, { + id, + tree: "work.dup", + content: "replacement", + }); + expect(second).toBeNull(); + expect((await db.getMemory(FULL, id))?.content).toBe("original"); +}); + +test("createMemory with replaceIfMetaDiffers rewrites stale rows in place", async () => { + const id = "01900000-0000-7000-8000-0000000000d1"; + await db.createMemory(FULL, { + id, + tree: "work.upsert", + content: "render v1", + meta: { importer_version: "1" }, + }); + + // Same version → skip. + const same = await db.createMemory(FULL, { + id, + tree: "work.upsert", + content: "render v1 again", + meta: { importer_version: "1" }, + replaceIfMetaDiffers: "importer_version", + }); + expect(same).toBeNull(); + expect((await db.getMemory(FULL, id))?.content).toBe("render v1"); + + // Bumped version → replaced, reported as an update (inserted: false). + const bumped = await db.createMemory(FULL, { + id, + tree: "work.upsert", + content: "render v2", + meta: { importer_version: "2" }, + replaceIfMetaDiffers: "importer_version", + }); + expect(bumped).toEqual({ id, inserted: false }); + const after = await db.getMemory(FULL, id); + expect(after?.content).toBe("render v2"); + expect(after?.meta).toEqual({ importer_version: "2" }); +}); + +test("batchCreateMemories upserts a batch in one call", async () => { + const stale = "01900000-0000-7000-8000-0000000000b1"; + const fresh = "01900000-0000-7000-8000-0000000000b2"; + await db.batchCreateMemories(FULL, [ + { id: stale, tree: "work.batch", content: "old", meta: { v: "1" } }, + { id: fresh, tree: "work.batch", content: "current", meta: { v: "2" } }, + ]); + + const rows = await db.batchCreateMemories( + FULL, + [ + { id: stale, tree: "work.batch", content: "new", meta: { v: "2" } }, + { id: fresh, tree: "work.batch", content: "untouched", meta: { v: "2" } }, + { tree: "work.batch", content: "generated id" }, + ], + "v", + ); + const byId = new Map(rows.map((r) => [r.id, r.inserted])); + expect(rows).toHaveLength(2); // fresh skipped → absent + expect(byId.get(stale)).toBe(false); + expect((await db.getMemory(FULL, stale))?.content).toBe("new"); + expect((await db.getMemory(FULL, fresh))?.content).toBe("current"); + const generated = rows.find((r) => r.id !== stale); + expect(generated?.inserted).toBe(true); + expect((await db.getMemory(FULL, generated?.id as string))?.content).toBe( + "generated id", + ); +}); + +test("access is enforced by the tree_access argument", async () => { + // create requires write (>=2): read-only access is rejected + await expect( + db.createMemory(READONLY, { tree: "work.x", content: "nope" }), + ).rejects.toThrow(); + + // a memory is invisible to a tree_access set that doesn't cover its path + const id = await mustCreate(FULL, { + tree: "work.secret", + content: "shh", + }); + const other: TreeAccess = [{ tree_path: "other", access: 3 }]; + expect(await db.getMemory(other, id)).toBeNull(); +}); + +test("patchMemory updates fields; deleteMemory removes", async () => { + const id = await mustCreate(FULL, { + tree: "work.p", + content: "before", + }); + expect(await db.patchMemory(FULL, id, { content: "after" })).toBe(true); + expect((await db.getMemory(FULL, id))?.content).toBe("after"); + + expect(await db.deleteMemory(FULL, id)).toBe(true); + expect(await db.getMemory(FULL, id)).toBeNull(); +}); + +test("bm25 search ranks by full-text relevance", async () => { + await db.createMemory(FULL, { + tree: "work.a", + content: "the quick brown fox", + }); + await db.createMemory(FULL, { tree: "work.b", content: "lorem ipsum dolor" }); + + const results = await db.search(FULL, { bm25: "fox", limit: 5 }); + expect(results.length).toBeGreaterThanOrEqual(1); + expect(results[0]?.content).toContain("fox"); +}); + +test("unranked (filter-only) search orders by id, newest-first by default", async () => { + // Explicit, strictly-increasing uuidv7 ids under a dedicated subtree. + const ids = [ + "01900000-0000-7000-8000-000000000001", + "01900000-0000-7000-8000-000000000002", + "01900000-0000-7000-8000-000000000003", + ]; + for (const id of ids) { + await db.createMemory(FULL, { id, tree: "work.ord", content: `c-${id}` }); + } + + // Default → newest id first (desc); results[0] is the high-water entry. + const def = await db.search(FULL, { ltree: "work.ord", limit: 10 }); + expect(def.map((r) => r.id)).toEqual([...ids].reverse()); + + // Explicit asc → oldest first. + const asc = await db.search(FULL, { + ltree: "work.ord", + order: "asc", + limit: 10, + }); + expect(asc.map((r) => r.id)).toEqual(ids); + + // Explicit desc matches the default. + const desc = await db.search(FULL, { + ltree: "work.ord", + order: "desc", + limit: 10, + }); + expect(desc.map((r) => r.id)).toEqual([...ids].reverse()); +}); + +test("vector search ranks by embedding similarity", async () => { + const near = await mustCreate(FULL, { + tree: "work.v1", + content: "near", + }); + const far = await mustCreate(FULL, { tree: "work.v2", content: "far" }); + await setEmbedding(near, [1, 0, 0, 0]); + await setEmbedding(far, [0, 1, 0, 0]); + + const results = await db.search(FULL, { vec: [1, 0, 0, 0], limit: 5 }); + expect(results[0]?.id).toBe(near); +}); + +test("hybridSearch fuses bm25 + vector", async () => { + const id = await mustCreate(FULL, { + tree: "work.h", + content: "hybrid pineapple", + }); + await setEmbedding(id, [0, 0, 1, 0]); + + const results = await db.hybridSearch(FULL, { + bm25: "pineapple", + vec: [0, 0, 1, 0], + limit: 5, + }); + expect(results.some((r) => r.id === id)).toBe(true); +}); + +test("moveTree, countTree, listTree", async () => { + await db.createMemory(FULL, { tree: "work.src.one", content: "1" }); + await db.createMemory(FULL, { tree: "work.src.two", content: "2" }); + + expect(await db.countTree(FULL, { tree: "work.src" }, 1)).toBe(2); + + const moved = await db.moveTree(FULL, "work.src", "work.dst"); + expect(moved).toBe(2); + expect(await db.countTree(FULL, { tree: "work.src" }, 1)).toBe(0); + expect(await db.countTree(FULL, { tree: "work.dst" }, 1)).toBe(2); + + const listed = await db.listTree(FULL, "work.dst.*"); + expect(listed.some((e) => e.tree === "work.dst")).toBe(true); +}); diff --git a/packages/engine/space/db.ts b/packages/engine/space/db.ts new file mode 100644 index 0000000..1f9092d --- /dev/null +++ b/packages/engine/space/db.ts @@ -0,0 +1,302 @@ +import type { Sql } from "postgres"; +import type { AccessLevel } from "../core/types"; +import type { + CreateMemoryParams, + HybridSearchOptions, + Memory, + MemoryPatch, + SearchOptions, + SearchResultItem, + TreeAccess, + TreeListEntry, +} from "./types"; + +/** + * The space data-plane layer for one space schema (me_). + * + * Thin wrappers over the space SQL functions — each method calls a function and + * passes the `treeAccess` set (from core.buildTreeAccess) for access enforcement. + * No table queries in TS; no RLS (access is the jsonb argument). + */ +export interface SpaceStore { + /** + * Insert one memory. When an explicit `params.id` already exists the + * outcome depends on `params.replaceIfMetaDiffers`: unset → skip (null); + * set to a meta key → the existing row is replaced when its value for that + * key differs from the new record's (`inserted: false`), else skipped. + * Deterministic-id importers use this to re-submit idempotently and push + * version-bump re-renders in the same call. + */ + createMemory( + treeAccess: TreeAccess, + params: CreateMemoryParams, + ): Promise<{ id: string; inserted: boolean } | null>; + /** + * Set-based createMemory for a whole batch: one statement, one round + * trip, same per-row conflict semantics. Returns one row per + * insert/replace — skipped rows are absent — and an explicit id repeated + * within the batch collapses to its first occurrence. Atomic. + */ + batchCreateMemories( + treeAccess: TreeAccess, + memories: CreateMemoryParams[], + replaceIfMetaDiffers?: string, + ): Promise>; + getMemory(treeAccess: TreeAccess, id: string): Promise; + patchMemory( + treeAccess: TreeAccess, + id: string, + patch: MemoryPatch, + ): Promise; + deleteMemory(treeAccess: TreeAccess, id: string): Promise; + + moveTree( + treeAccess: TreeAccess, + src: string, + dst: string, + dryRun?: boolean, + ): Promise; + copyTree( + treeAccess: TreeAccess, + src: string, + dst: string, + dryRun?: boolean, + ): Promise; + deleteTree( + treeAccess: TreeAccess, + tree: string, + dryRun?: boolean, + ): Promise; + countTree( + treeAccess: TreeAccess, + query: { tree?: string; lquery?: string; ltxtquery?: string }, + access: AccessLevel, + ): Promise; + listTree(treeAccess: TreeAccess, lquery: string): Promise; + + search( + treeAccess: TreeAccess, + options?: SearchOptions, + ): Promise; + hybridSearch( + treeAccess: TreeAccess, + options: HybridSearchOptions, + ): Promise; + + /** Run operations atomically against the same transaction. */ + withTransaction(fn: (store: SpaceStore) => Promise): Promise; +} + +function mapMemory(row: Record): Memory { + return { + id: row.id as string, + tree: row.tree as string, + meta: (row.meta as Record) ?? {}, + temporal: (row.temporal as string | null) ?? null, + content: row.content as string, + hasEmbedding: Boolean(row.has_embedding), + createdAt: row.created_at as Date, + updatedAt: (row.updated_at as Date | null) ?? null, + }; +} + +function mapSearchItem(row: Record): SearchResultItem { + return { ...mapMemory(row), score: Number(row.score) }; +} + +export function spaceStore(sql: Sql, schema: string): SpaceStore { + const sch = sql(schema); + const bm25Index = `${schema}.memory_content_bm25_idx`; + + /** + * jsonb param fragment (null-safe). Uses sql.json so postgres.js serializes + * the value as json — passing a pre-stringified string double-encodes it into + * a jsonb string scalar, and passing a raw JS array would be sent as a Postgres + * array; both break jsonb_to_recordset in the SQL functions. + */ + const jb = (v: unknown) => + v === null || v === undefined + ? sql`null::jsonb` + : sql`${sql.json(v as never)}::jsonb`; + + /** bm25query fragment from a query string (or null). */ + const bm25 = (q: string | undefined) => + q === undefined + ? sql`null::bm25query` + : sql`to_bm25query(${q}::text, ${bm25Index}::text)`; + + /** halfvec fragment from an embedding (or null). */ + const halfvec = (v: number[] | undefined) => + v === undefined ? sql`null::halfvec` : sql`${`[${v.join(",")}]`}::halfvec`; + + return { + async createMemory(treeAccess, p) { + const [row] = await sql` + select id, inserted from ${sch}.create_memory( + ${jb(treeAccess)}, + ${p.tree}::ltree, + ${p.content}, + ${p.id ?? null}, + ${jb(p.meta)}, + ${p.temporal ?? null}::tstzrange, + ${p.replaceIfMetaDiffers ?? null} + )`; + // Zero rows = the explicit id already exists and was skipped (version + // match, no replace key, or no write access on the existing row's tree). + if (!row) return null; + return { id: row.id as string, inserted: Boolean(row.inserted) }; + }, + + async batchCreateMemories(treeAccess, memories, replaceIfMetaDiffers) { + if (memories.length === 0) return []; + // Parallel arrays aligned by position. Metas travel as ONE jsonb array + // via sql.json — a jsonb[] parameter would double-encode each element + // into a string scalar (see the jb() note above). + const rows = await sql` + select id, inserted from ${sch}.batch_create_memory( + ${jb(treeAccess)}, + ${memories.map((m) => m.id ?? null)}::uuid[], + ${memories.map((m) => m.tree)}::ltree[], + ${memories.map((m) => m.content)}::text[], + ${jb(memories.map((m) => m.meta ?? {}))}, + ${memories.map((m) => m.temporal ?? null)}::tstzrange[], + ${replaceIfMetaDiffers ?? null} + )`; + return rows.map((r) => ({ + id: r.id as string, + inserted: Boolean(r.inserted), + })); + }, + + async getMemory(treeAccess, id) { + const [row] = await sql` + select id, tree::text as tree, meta, temporal::text as temporal, + content, has_embedding, created_at, updated_at + from ${sch}.get_memory(${jb(treeAccess)}, ${id})`; + return row ? mapMemory(row) : null; + }, + + async patchMemory(treeAccess, id, patch) { + const obj: Record = {}; + if (patch.tree !== undefined) obj.tree = patch.tree; + if (patch.meta !== undefined) obj.meta = patch.meta; + if (patch.temporal !== undefined) obj.temporal = patch.temporal; + if (patch.content !== undefined) obj.content = patch.content; + const [row] = await sql` + select ${sch}.patch_memory(${jb(treeAccess)}, ${id}, ${jb(obj)}) as ok`; + return Boolean(row?.ok); + }, + + async deleteMemory(treeAccess, id) { + const [row] = await sql` + select ${sch}.delete_memory(${jb(treeAccess)}, ${id}) as ok`; + return Boolean(row?.ok); + }, + + async moveTree(treeAccess, src, dst, dryRun = false) { + const [row] = await sql` + select ${sch}.move_tree(${jb(treeAccess)}, ${src}::ltree, ${dst}::ltree, ${dryRun}) as n`; + return Number(row?.n); + }, + + async copyTree(treeAccess, src, dst, dryRun = false) { + const [row] = await sql` + select ${sch}.copy_tree(${jb(treeAccess)}, ${src}::ltree, ${dst}::ltree, ${dryRun}) as n`; + return Number(row?.n); + }, + + async deleteTree(treeAccess, tree, dryRun = false) { + const [row] = await sql` + select ${sch}.delete_tree(${jb(treeAccess)}, ${tree}::ltree, ${dryRun}) as n`; + return Number(row?.n); + }, + + async countTree(treeAccess, query, access) { + let row: { n?: unknown } | undefined; + if (query.tree !== undefined) { + [row] = await sql` + select ${sch}.count_tree(${jb(treeAccess)}, ${query.tree}::ltree, ${access}) as n`; + } else if (query.lquery !== undefined) { + [row] = await sql` + select ${sch}.count_tree(${jb(treeAccess)}, ${query.lquery}::lquery, ${access}) as n`; + } else if (query.ltxtquery !== undefined) { + [row] = await sql` + select ${sch}.count_tree(${jb(treeAccess)}, ${query.ltxtquery}::ltxtquery, ${access}) as n`; + } else { + throw new Error("countTree requires one of tree / lquery / ltxtquery"); + } + return Number(row?.n); + }, + + async listTree(treeAccess, lquery) { + const rows = await sql` + select tree::text as tree, count + from ${sch}.list_tree(${jb(treeAccess)}, ${lquery}::lquery)`; + return rows.map((r) => ({ + tree: r.tree as string, + count: Number(r.count), + })); + }, + + async search(treeAccess, options = {}) { + const o = options; + const rows = await sql` + select id, meta, tree::text as tree, temporal::text as temporal, + content, has_embedding, created_at, updated_at, score + from ${sch}.search_memory( + ${jb(treeAccess)}, + ${bm25(o.bm25)}, + ${halfvec(o.vec)}, + ${o.maxVecDist ?? null}, + ${o.ltree ?? null}::ltree, + ${o.lquery ?? null}::lquery, + ${o.ltxtquery ?? null}::ltxtquery, + ${jb(o.metaContains)}, + ${o.temporalWithin ?? null}::tstzrange, + ${o.temporalOverlaps ?? null}::tstzrange, + ${o.temporalBefore ?? null}::timestamptz, + ${o.temporalAfter ?? null}::timestamptz, + ${o.regexp ?? null}, + ${o.limit ?? 10}, + ${o.order ?? "desc"} + )`; + return rows.map(mapSearchItem); + }, + + async hybridSearch(treeAccess, options) { + const o = options; + const rows = await sql` + select id, meta, tree::text as tree, temporal::text as temporal, + content, has_embedding, created_at, updated_at, score + from ${sch}.hybrid_search_memory( + ${jb(treeAccess)}, + ${bm25(o.bm25)}, + ${halfvec(o.vec)}, + ${o.maxVecDist ?? null}, + ${o.ltree ?? null}::ltree, + ${o.lquery ?? null}::lquery, + ${o.ltxtquery ?? null}::ltxtquery, + ${jb(o.metaContains)}, + ${o.temporalWithin ?? null}::tstzrange, + ${o.temporalOverlaps ?? null}::tstzrange, + ${o.temporalBefore ?? null}::timestamptz, + ${o.temporalAfter ?? null}::timestamptz, + ${o.regexp ?? null}, + ${o.k ?? 60.0}, + ${o.candidateLimit ?? 30}, + ${o.fulltextWeight ?? 1.0}, + ${o.semanticWeight ?? 1.0}, + ${o.limit ?? 10} + )`; + return rows.map(mapSearchItem); + }, + + async withTransaction( + fn: (store: SpaceStore) => Promise, + ): Promise { + return sql.begin((tx) => + fn(spaceStore(tx as unknown as Sql, schema)), + ) as Promise; + }, + }; +} diff --git a/packages/engine/space/index.ts b/packages/engine/space/index.ts new file mode 100644 index 0000000..c210533 --- /dev/null +++ b/packages/engine/space/index.ts @@ -0,0 +1,13 @@ +export { type SpaceStore, spaceStore } from "./db"; +export type { + CreateMemoryParams, + HybridSearchOptions, + Memory, + MemoryFilters, + MemoryPatch, + SearchOptions, + SearchResultItem, + TemporalRange, + TreeAccess, + TreeListEntry, +} from "./types"; diff --git a/packages/engine/space/types.ts b/packages/engine/space/types.ts new file mode 100644 index 0000000..215c17f --- /dev/null +++ b/packages/engine/space/types.ts @@ -0,0 +1,104 @@ +/** + * Types for the space data-plane TS layer. + * + * Thin wrappers over the space SQL functions (packages/database/space/migrate/ + * idempotent/*.sql). Every method takes a `treeAccess` set — the jsonb produced + * by core.buildTreeAccess — which the SQL functions use to enforce access. + */ + +import type { TreeAccess } from "../core/types"; + +export type { TreeAccess }; + +/** tstzrange rendered as its text form, e.g. "[2024-01-01,2024-01-02)". */ +export type TemporalRange = string; + +export interface Memory { + id: string; + tree: string; + meta: Record; + temporal: TemporalRange | null; + content: string; + hasEmbedding: boolean; + createdAt: Date; + updatedAt: Date | null; +} + +export interface SearchResultItem extends Memory { + score: number; +} + +export interface CreateMemoryParams { + tree: string; + content: string; + id?: string; + meta?: Record; + temporal?: TemporalRange; + /** + * Meta key for conditional replace: when an explicit `id` already exists, + * replace the row iff its meta value for this key differs from the new + * record (e.g. importer_version). Default: duplicates are skipped. + */ + replaceIfMetaDiffers?: string; +} + +export interface MemoryPatch { + tree?: string; + meta?: Record; + temporal?: TemporalRange | null; + content?: string; +} + +/** Filters shared by search (and the count/list tree helpers where relevant). */ +export interface MemoryFilters { + /** ancestor-or-self match: only memories at/under this path. */ + ltree?: string; + /** ltree lquery pattern. */ + lquery?: string; + /** ltree full-text ltxtquery. */ + ltxtquery?: string; + /** meta @> this object. */ + metaContains?: Record; + temporalWithin?: TemporalRange; + temporalOverlaps?: TemporalRange; + temporalBefore?: string; + temporalAfter?: string; + /** case-insensitive regexp on content (must be combined with another filter). */ + regexp?: string; +} + +export interface SearchOptions extends MemoryFilters { + /** BM25 full-text query string. Mutually exclusive with `vec`. */ + bm25?: string; + /** Pre-computed query embedding. Mutually exclusive with `bm25`. */ + vec?: number[]; + /** Max cosine distance (only with `vec`). */ + maxVecDist?: number; + limit?: number; + /** + * Result order for the **unranked** (filter-only) path: by id (chronological), + * `"desc"` (default, newest first) or `"asc"` (oldest first). Ignored when a + * `bm25`/`vec` query is present — those are ordered by relevance score. + */ + order?: "asc" | "desc"; +} + +export interface HybridSearchOptions extends MemoryFilters { + /** BM25 full-text query string (required). */ + bm25: string; + /** Pre-computed query embedding (required). */ + vec: number[]; + maxVecDist?: number; + /** RRF constant (default 60). */ + k?: number; + /** Per-arm candidate pool size (default 30). */ + candidateLimit?: number; + fulltextWeight?: number; + semanticWeight?: number; + limit?: number; +} + +export interface TreeListEntry { + tree: string; + count: number; +} diff --git a/packages/engine/types.ts b/packages/engine/types.ts deleted file mode 100644 index da3c59a..0000000 --- a/packages/engine/types.ts +++ /dev/null @@ -1,258 +0,0 @@ -import type { SQL } from "bun"; - -// ============================================================================= -// Errors -// ============================================================================= - -/** - * Thrown when a feature is not yet implemented - */ -export class NotImplementedError extends Error { - constructor(message: string) { - super(message); - this.name = "NotImplementedError"; - } -} - -// ============================================================================= -// Context -// ============================================================================= - -/** - * Context passed to all ops functions - */ -export interface OpsContext { - /** Database connection or transaction handle */ - sql: SQL; - /** Schema name (e.g., "me_abc123xyz789") */ - schema: string; - /** Shard number for pgDog routing (optional, future use) */ - shard?: number; - /** Whether we're inside a transaction (controls whether withTx opens a new one) */ - inTransaction: boolean; - /** Get the current user ID (for RLS context) */ - getUserId: () => string | null; -} - -// ============================================================================= -// User -// ============================================================================= - -/** - * User: thing that accesses memories within an engine. - * Can be linked to an identity (soft FK to accounts.identity) or standalone. - * If can_login = false, it's a role (grant container for RBAC). - */ -export interface User { - id: string; - name: string; - identityId: string | null; - canLogin: boolean; - superuser: boolean; - createrole: boolean; - createdAt: Date; - updatedAt: Date | null; -} - -export interface CreateUserParams { - id?: string; - name: string; - identityId?: string | null; - canLogin?: boolean; - superuser?: boolean; - createrole?: boolean; -} - -// ============================================================================= -// API Key -// ============================================================================= - -/** - * API key for authenticating to an engine. - * Scoped to a user within this engine. - */ -export interface ApiKey { - id: string; - userId: string; - lookupId: string; - name: string; - expiresAt: Date | null; - createdAt: Date; - revokedAt: Date | null; -} - -export interface CreateApiKeyParams { - userId: string; - name: string; - expiresAt?: Date | null; -} - -export interface CreateApiKeyResult { - apiKey: ApiKey; - /** The full API key string (only returned on creation) */ - rawKey: string; -} - -export interface ValidateApiKeyResult { - valid: boolean; - userId?: string; - apiKeyId?: string; - error?: string; -} - -// ============================================================================= -// Tree Grant -// ============================================================================= - -export interface TreeGrant { - id: string; - userId: string; - userName: string; - treePath: string; - actions: string[]; - grantedBy: string | null; - withGrantOption: boolean; - createdAt: Date; -} - -export interface GrantTreeAccessParams { - userId: string; - treePath: string; - actions: string[]; - grantedBy?: string | null; - withGrantOption?: boolean; -} - -// ============================================================================= -// Tree Owner -// ============================================================================= - -export interface TreeOwner { - treePath: string; - userId: string; - userName: string; - createdBy: string | null; - createdByName: string | null; - createdAt: Date; -} - -// ============================================================================= -// Role -// ============================================================================= - -export interface RoleMember { - roleId: string; - memberId: string; - memberName: string; - withAdminOption: boolean; - createdAt: Date; -} - -export interface RoleInfo { - id: string; - name: string; - withAdminOption: boolean; -} - -// ============================================================================= -// Memory -// ============================================================================= - -export interface Memory { - id: string; - content: string; - meta: Record; - tree: string; - temporal: { start: Date; end: Date } | null; - hasEmbedding: boolean; - createdAt: Date; - createdBy: string | null; - updatedAt: Date | null; -} - -export interface CreateMemoryParams { - id?: string; - content: string; - meta?: Record; - tree?: string; - temporal?: { start: Date; end?: Date } | null; - createdBy?: string | null; -} - -export interface UpdateMemoryParams { - content?: string; - meta?: Record; - tree?: string; - temporal?: { start: Date; end?: Date } | null; -} - -// ============================================================================= -// Search -// ============================================================================= - -export interface SearchParams { - /** Semantic search query (text - embedding must be generated by caller) */ - semantic?: string; - /** Pre-computed embedding vector for semantic search */ - embedding?: number[]; - /** Full-text (BM25) search query */ - fulltext?: string; - /** Regex filter applied to content (POSIX, case-insensitive) */ - grep?: string; - /** Filter by tree path (ltree, lquery, or ltxtquery) */ - tree?: string; - /** Filter by metadata (JSONB containment) */ - meta?: Record; - /** Temporal filter */ - temporal?: TemporalFilter; - /** Maximum results (default: 10) */ - limit?: number; - /** Candidates per search mode before RRF fusion */ - candidateLimit?: number; - /** Minimum semantic similarity score (0-1) for vector candidates */ - semanticThreshold?: number; - /** Weights for hybrid search */ - weights?: SearchWeights; - /** Sort direction for filter-only searches */ - orderBy?: "asc" | "desc"; -} - -export interface TemporalFilter { - /** Find memories containing this point in time */ - contains?: Date | string; - /** Find memories overlapping this range [start, end] */ - overlaps?: [Date | string, Date | string]; - /** Find memories fully within this range [start, end] */ - within?: [Date | string, Date | string]; -} - -export interface SearchWeights { - semantic?: number; - fulltext?: number; -} - -export interface SearchResult { - results: SearchResultItem[]; - total: number; - limit: number; -} - -export interface SearchResultItem extends Memory { - score: number; -} - -// ============================================================================= -// Tree -// ============================================================================= - -export interface GetTreeParams { - /** Root path to start from (default: root) */ - tree?: string; - /** Maximum depth to return */ - levels?: number; -} - -export interface TreeNode { - path: string; - count: number; -} diff --git a/packages/engine/util/api-key.test.ts b/packages/engine/util/api-key.test.ts deleted file mode 100644 index 3bc1942..0000000 --- a/packages/engine/util/api-key.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { - extractEngineSlug, - formatApiKey, - generateLookupId, - generateSecret, - hashSecret, - parseApiKey, - verifySecret, -} from "./api-key"; - -describe("generateLookupId", () => { - test("generates 16-char string", () => { - const id = generateLookupId(); - expect(id).toHaveLength(16); - }); - - test("only contains valid characters", () => { - const id = generateLookupId(); - expect(id).toMatch(/^[A-Za-z0-9_-]{16}$/); - }); - - test("generates unique values", () => { - const ids = new Set(Array.from({ length: 100 }, () => generateLookupId())); - expect(ids.size).toBe(100); - }); -}); - -describe("generateSecret", () => { - test("generates 32-char string", () => { - const secret = generateSecret(); - expect(secret).toHaveLength(32); - }); - - test("only contains base64url characters", () => { - const secret = generateSecret(); - expect(secret).toMatch(/^[A-Za-z0-9_-]{32}$/); - }); - - test("generates unique values", () => { - const secrets = new Set( - Array.from({ length: 100 }, () => generateSecret()), - ); - expect(secrets.size).toBe(100); - }); -}); - -describe("hashSecret / verifySecret", () => { - test("hash and verify round-trip", async () => { - const secret = generateSecret(); - const hash = await hashSecret(secret); - - expect(await verifySecret(secret, hash)).toBe(true); - expect(await verifySecret("wrong-secret", hash)).toBe(false); - }); - - test("different secrets produce different hashes", async () => { - const secret1 = generateSecret(); - const secret2 = generateSecret(); - const hash1 = await hashSecret(secret1); - const hash2 = await hashSecret(secret2); - - expect(hash1).not.toBe(hash2); - }); -}); - -describe("formatApiKey", () => { - test("formats key with all parts", () => { - const key = formatApiKey( - "abc123xyz789", - "Sh00uLs5rmSHHun3", - "secret32charslong_______________", - ); - expect(key).toBe( - "me.abc123xyz789.Sh00uLs5rmSHHun3.secret32charslong_______________", - ); - }); -}); - -describe("parseApiKey", () => { - // 32-char secret for tests - const validSecret = "pREy3xfnbCpgUXiaBcDeFgHiJkLm1234"; - - test("parses valid key", () => { - const key = `me.abc123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - const parsed = parseApiKey(key); - - expect(parsed).toEqual({ - engineSlug: "abc123xyz789", - lookupId: "Sh00uLs5rmSHHun3", - secret: validSecret, - }); - }); - - test("returns null for wrong prefix", () => { - const key = `xx.abc123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for invalid engineSlug (uppercase)", () => { - const key = `me.ABC123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for short engineSlug", () => { - const key = `me.abc123.Sh00uLs5rmSHHun3.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for invalid lookupId", () => { - const key = `me.abc123xyz789.short.${validSecret}`; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for wrong secret length", () => { - const key = "me.abc123xyz789.Sh00uLs5rmSHHun3.tooshort"; - expect(parseApiKey(key)).toBeNull(); - }); - - test("returns null for wrong number of parts", () => { - expect(parseApiKey("me.abc123xyz789.Sh00uLs5rmSHHun3")).toBeNull(); - expect(parseApiKey("me.abc123xyz789")).toBeNull(); - expect(parseApiKey("invalid")).toBeNull(); - }); -}); - -describe("extractEngineSlug", () => { - const validSecret = "pREy3xfnbCpgUXiaBcDeFgHiJkLm1234"; - - test("extracts slug from valid key", () => { - const key = `me.abc123xyz789.Sh00uLs5rmSHHun3.${validSecret}`; - expect(extractEngineSlug(key)).toBe("abc123xyz789"); - }); - - test("returns null for invalid key", () => { - expect(extractEngineSlug("invalid")).toBeNull(); - expect(extractEngineSlug("me.INVALID.x.y")).toBeNull(); - }); -}); diff --git a/packages/engine/util/api-key.ts b/packages/engine/util/api-key.ts deleted file mode 100644 index 09b4f77..0000000 --- a/packages/engine/util/api-key.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * API key generation and parsing utilities - * - * Key format: me.{engineSlug}.{lookupId}.{secret} - * Example: me.k8xf2nq4mp7a.Sh00uLs5rmSHHun3.pREy3xfnbCpgUXiaBcD... - * - * - me: Fixed prefix for all memory engine keys - * - engineSlug: 12-char alphanumeric identifier for routing - * - lookupId: 16-char alphanumeric identifier for database lookup - * - secret: 32-char random secret, verified against hash - */ - -const LOOKUP_ID_LENGTH = 16; -const SECRET_LENGTH = 32; -const LOOKUP_ID_CHARSET = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-"; - -/** - * Generate a random lookup ID (16 chars, URL-safe) - */ -export function generateLookupId(): string { - const bytes = crypto.getRandomValues(new Uint8Array(LOOKUP_ID_LENGTH)); - let result = ""; - for (const byte of bytes) { - result += LOOKUP_ID_CHARSET[byte % LOOKUP_ID_CHARSET.length]; - } - return result; -} - -/** - * Generate a random secret (32 chars, base64url) - */ -export function generateSecret(): string { - const bytes = crypto.getRandomValues(new Uint8Array(SECRET_LENGTH)); - return btoa(String.fromCharCode(...bytes)) - .replace(/\+/g, "-") - .replace(/\//g, "_") - .replace(/=/g, "") - .slice(0, SECRET_LENGTH); -} - -/** - * Hash a secret for storage using Argon2id - */ -export async function hashSecret(secret: string): Promise { - return Bun.password.hash(secret, { - algorithm: "argon2id", - memoryCost: 19456, - timeCost: 2, - }); -} - -/** - * Verify a secret against its hash - */ -export async function verifySecret( - secret: string, - hash: string, -): Promise { - return Bun.password.verify(secret, hash); -} - -/** - * Format a complete API key from its parts - */ -export function formatApiKey( - engineSlug: string, - lookupId: string, - secret: string, -): string { - return `me.${engineSlug}.${lookupId}.${secret}`; -} - -/** - * Parse an API key into its components - * Returns null if the key format is invalid - */ -export function parseApiKey( - key: string, -): { engineSlug: string; lookupId: string; secret: string } | null { - const parts = key.split("."); - if (parts.length !== 4) { - return null; - } - - const [prefix, engineSlug, lookupId, secret] = parts; - - // Validate prefix - if (prefix !== "me") { - return null; - } - - // Validate engineSlug format (12 lowercase alphanumeric chars) - if (!engineSlug || !/^[a-z0-9]{12}$/.test(engineSlug)) { - return null; - } - - // Validate lookupId format (16 chars from our charset) - if (!lookupId || !/^[A-Za-z0-9_-]{16}$/.test(lookupId)) { - return null; - } - - // Validate secret (32 chars, base64url) - if (!secret || secret.length !== SECRET_LENGTH) { - return null; - } - - return { engineSlug, lookupId, secret }; -} - -/** - * Extract the engine slug from an API key without full parsing - * Useful for routing before validation - */ -export function extractEngineSlug(key: string): string | null { - const parts = key.split("."); - if (parts.length !== 4 || parts[0] !== "me") { - return null; - } - const slug = parts[1]; - if (!slug || !/^[a-z0-9]{12}$/.test(slug)) { - return null; - } - return slug; -} diff --git a/packages/engine/util/index.ts b/packages/engine/util/index.ts deleted file mode 100644 index ea06279..0000000 --- a/packages/engine/util/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { - extractEngineSlug, - formatApiKey, - generateLookupId, - generateSecret, - hashSecret, - parseApiKey, - verifySecret, -} from "./api-key"; diff --git a/packages/protocol/accounts/engine.ts b/packages/protocol/accounts/engine.ts deleted file mode 100644 index f4d5205..0000000 --- a/packages/protocol/accounts/engine.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Engine method schemas (accounts side) — params and results for engine.* RPC methods. - * - * These are the engine management methods on the accounts RPC endpoint, - * not to be confused with the engine's own RPC methods. - */ -import { z } from "zod"; -import { engineStatusSchema, nameSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * engine.create params. - */ -export const engineCreateParams = z.object({ - orgId: uuidv7Schema, - name: nameSchema, - language: z - .string() - .regex(/^[a-z_]+$/) - .optional() - .default("english"), -}); - -export type EngineCreateParams = z.infer; - -/** - * engine.list params. - */ -export const engineListParams = z.object({ - orgId: uuidv7Schema, -}); - -export type EngineListParams = z.infer; - -/** - * engine.get params. - */ -export const engineGetParams = z.object({ - id: uuidv7Schema, -}); - -export type EngineGetParams = z.infer; - -/** - * engine.update params. - */ -export const engineUpdateParams = z.object({ - id: uuidv7Schema, - name: nameSchema.optional(), - status: engineStatusSchema.optional(), -}); - -export type EngineUpdateParams = z.infer; - -/** - * engine.delete params. - */ -export const engineDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type EngineDeleteParams = z.infer; - -/** - * engine.delete result. - */ -export const engineDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type EngineDeleteResult = z.infer; - -/** - * engine.setupAccess params. - * Bootstraps engine access for a session-authenticated identity. - */ -export const engineSetupAccessParams = z.object({ - engineId: uuidv7Schema, - apiKeyName: z.string().min(1).optional(), -}); - -export type EngineSetupAccessParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single engine response — returned by create, get, update. - */ -export const engineResponse = z.object({ - id: z.string(), - orgId: z.string(), - slug: z.string(), - name: z.string(), - shardId: z.number(), - status: z.string(), - language: z.string(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type EngineResponse = z.infer; - -/** - * engine.list result. - */ -export const engineListResult = z.object({ - engines: z.array(engineResponse), -}); - -export type EngineListResult = z.infer; - -/** - * engine.setupAccess result. - */ -export const engineSetupAccessResult = z.object({ - rawKey: z.string(), - engineSlug: z.string(), - userId: z.string(), - engineName: z.string(), - orgName: z.string(), -}); - -export type EngineSetupAccessResult = z.infer; diff --git a/packages/protocol/accounts/identity.test.ts b/packages/protocol/accounts/identity.test.ts deleted file mode 100644 index a59e628..0000000 --- a/packages/protocol/accounts/identity.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Tests for identity protocol schemas. - */ -import { describe, expect, test } from "bun:test"; -import { - identityGetByEmailParams, - identityGetByEmailResult, - identityResponse, -} from "./identity.ts"; - -describe("identityGetByEmailParams", () => { - test("accepts valid email", () => { - expect( - identityGetByEmailParams.safeParse({ email: "a@b.com" }).success, - ).toBe(true); - }); - - test("rejects invalid email", () => { - expect(identityGetByEmailParams.safeParse({ email: "nope" }).success).toBe( - false, - ); - }); - - test("rejects missing email", () => { - expect(identityGetByEmailParams.safeParse({}).success).toBe(false); - }); -}); - -describe("identityGetByEmailResult", () => { - test("accepts identity with all fields", () => { - const result = identityGetByEmailResult.safeParse({ - identity: { - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "a@b.com", - name: "Alice", - createdAt: "2026-01-15T00:00:00.000Z", - updatedAt: null, - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts null identity", () => { - const result = identityGetByEmailResult.safeParse({ identity: null }); - expect(result.success).toBe(true); - }); -}); - -describe("identityResponse", () => { - test("accepts valid identity", () => { - const result = identityResponse.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "a@b.com", - name: "Alice", - createdAt: "2026-01-15T00:00:00.000Z", - updatedAt: null, - }); - expect(result.success).toBe(true); - }); -}); diff --git a/packages/protocol/accounts/identity.ts b/packages/protocol/accounts/identity.ts deleted file mode 100644 index a7e2874..0000000 --- a/packages/protocol/accounts/identity.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Identity method schemas — params and results for me.* RPC methods. - */ -import { z } from "zod"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * me.get params — no params needed, uses session identity. - */ -export const meGetParams = z.object({}); - -export type MeGetParams = z.infer; - -/** - * identity.getByEmail params. - */ -export const identityGetByEmailParams = z.object({ - email: z.string().email(), -}); - -export type IdentityGetByEmailParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Identity response — returned by me.get. - */ -export const identityResponse = z.object({ - id: z.string(), - email: z.string(), - name: z.string(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type IdentityResponse = z.infer; - -/** - * identity.getByEmail result — nullable (identity may not exist). - */ -export const identityGetByEmailResult = z.object({ - identity: identityResponse.nullable(), -}); - -export type IdentityGetByEmailResult = z.infer; diff --git a/packages/protocol/accounts/index.ts b/packages/protocol/accounts/index.ts deleted file mode 100644 index 509c4fd..0000000 --- a/packages/protocol/accounts/index.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Accounts RPC contract — maps method names to params/result schema pairs. - * - * Covers all 21 methods on POST /api/v1/accounts/rpc (session token auth). - */ -import type { z } from "zod"; - -// Domain schemas -import { - engineCreateParams, - engineDeleteParams, - engineDeleteResult, - engineGetParams, - engineListParams, - engineListResult, - engineResponse, - engineSetupAccessParams, - engineSetupAccessResult, - engineUpdateParams, -} from "./engine.ts"; -import { - identityGetByEmailParams, - identityGetByEmailResult, - identityResponse, - meGetParams, -} from "./identity.ts"; -import { - invitationAcceptParams, - invitationAcceptResult, - invitationCreateParams, - invitationCreateResult, - invitationListParams, - invitationListResult, - invitationRevokeParams, - invitationRevokeResult, -} from "./invitation.ts"; -import { - orgCreateParams, - orgDeleteParams, - orgDeleteResult, - orgGetParams, - orgListParams, - orgListResult, - orgResponse, - orgUpdateParams, -} from "./org.ts"; -import { - orgMemberAddParams, - orgMemberListParams, - orgMemberListResult, - orgMemberRemoveParams, - orgMemberRemoveResult, - orgMemberResponse, - orgMemberUpdateRoleParams, - orgMemberUpdateRoleResult, -} from "./org-member.ts"; -import { sessionRevokeParams, sessionRevokeResult } from "./session.ts"; - -export * from "./engine.ts"; -// Re-export all domain schemas -export * from "./identity.ts"; -export * from "./invitation.ts"; -export * from "./org.ts"; -export * from "./org-member.ts"; -export * from "./session.ts"; - -// ============================================================================= -// RPC Contract -// ============================================================================= - -/** - * Define a method with its params schema and result schema. - */ -function method( - params: TParams, - result: TResult, -) { - return { params, result }; -} - -/** - * Accounts RPC method contract — all 21 methods. - * - * Each entry maps a method name to its params and result Zod schemas. - * The client library uses this for type inference and optional response validation. - * The server uses the params schemas for input validation. - */ -export const accountsMethods = { - // Identity (2) - "me.get": method(meGetParams, identityResponse), - "identity.getByEmail": method( - identityGetByEmailParams, - identityGetByEmailResult, - ), - - // Session (1) - "session.revoke": method(sessionRevokeParams, sessionRevokeResult), - - // Org (5) - "org.create": method(orgCreateParams, orgResponse), - "org.list": method(orgListParams, orgListResult), - "org.get": method(orgGetParams, orgResponse), - "org.update": method(orgUpdateParams, orgResponse), - "org.delete": method(orgDeleteParams, orgDeleteResult), - - // Org Member (4) - "org.member.list": method(orgMemberListParams, orgMemberListResult), - "org.member.add": method(orgMemberAddParams, orgMemberResponse), - "org.member.remove": method(orgMemberRemoveParams, orgMemberRemoveResult), - "org.member.updateRole": method( - orgMemberUpdateRoleParams, - orgMemberUpdateRoleResult, - ), - - // Engine (6) - "engine.create": method(engineCreateParams, engineResponse), - "engine.list": method(engineListParams, engineListResult), - "engine.get": method(engineGetParams, engineResponse), - "engine.update": method(engineUpdateParams, engineResponse), - "engine.delete": method(engineDeleteParams, engineDeleteResult), - "engine.setupAccess": method( - engineSetupAccessParams, - engineSetupAccessResult, - ), - - // Invitation (4) - "invitation.create": method(invitationCreateParams, invitationCreateResult), - "invitation.list": method(invitationListParams, invitationListResult), - "invitation.revoke": method(invitationRevokeParams, invitationRevokeResult), - "invitation.accept": method(invitationAcceptParams, invitationAcceptResult), -} as const; - -// ============================================================================= -// Type Utilities -// ============================================================================= - -/** Union of all accounts method names. */ -export type AccountsMethodName = keyof typeof accountsMethods; - -/** Extract the params type for a given accounts method. */ -export type AccountsParams = z.infer< - (typeof accountsMethods)[M]["params"] ->; - -/** Extract the result type for a given accounts method. */ -export type AccountsResult = z.infer< - (typeof accountsMethods)[M]["result"] ->; - -/** Get the params schema for runtime validation. */ -export function getAccountsParamsSchema( - method: M, -) { - return accountsMethods[method].params; -} - -/** Get the result schema for runtime response validation. */ -export function getAccountsResultSchema( - method: M, -) { - return accountsMethods[method].result; -} diff --git a/packages/protocol/accounts/invitation.ts b/packages/protocol/accounts/invitation.ts deleted file mode 100644 index e82f8e1..0000000 --- a/packages/protocol/accounts/invitation.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Invitation method schemas — params and results for invitation.* RPC methods. - */ -import { z } from "zod"; -import { emailSchema, orgRoleSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * invitation.create params. - */ -export const invitationCreateParams = z.object({ - orgId: uuidv7Schema, - email: emailSchema, - role: orgRoleSchema, - expiresInDays: z.number().int().min(1).max(30).optional(), -}); - -export type InvitationCreateParams = z.infer; - -/** - * invitation.list params. - */ -export const invitationListParams = z.object({ - orgId: uuidv7Schema, -}); - -export type InvitationListParams = z.infer; - -/** - * invitation.revoke params. - */ -export const invitationRevokeParams = z.object({ - id: uuidv7Schema, -}); - -export type InvitationRevokeParams = z.infer; - -/** - * invitation.accept params. - * Token is the raw invitation token from the email link. - */ -export const invitationAcceptParams = z.object({ - token: z.string().min(1, "token is required"), -}); - -export type InvitationAcceptParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Invitation response — returned by list. - */ -export const invitationResponse = z.object({ - id: z.string(), - orgId: z.string(), - email: z.string(), - role: z.string(), - invitedBy: z.string(), - expiresAt: z.string(), - acceptedAt: z.string().nullable(), - createdAt: z.string(), -}); - -export type InvitationResponse = z.infer; - -/** - * invitation.create result — includes the raw token (only returned on creation). - */ -export const invitationCreateResult = invitationResponse.extend({ - token: z.string(), -}); - -export type InvitationCreateResult = z.infer; - -/** - * invitation.list result. - */ -export const invitationListResult = z.object({ - invitations: z.array(invitationResponse), -}); - -export type InvitationListResult = z.infer; - -/** - * invitation.revoke result. - */ -export const invitationRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type InvitationRevokeResult = z.infer; - -/** - * invitation.accept result. - */ -export const invitationAcceptResult = z.object({ - accepted: z.boolean(), - orgId: z.string(), -}); - -export type InvitationAcceptResult = z.infer; diff --git a/packages/protocol/accounts/org-member.test.ts b/packages/protocol/accounts/org-member.test.ts deleted file mode 100644 index f1b18aa..0000000 --- a/packages/protocol/accounts/org-member.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Tests for org-member protocol response schemas. - * - * Verifies the response schemas accept name and email fields. - */ -import { describe, expect, test } from "bun:test"; -import { orgMemberResponse } from "./org-member.ts"; - -describe("orgMemberResponse", () => { - test("accepts response with name and email", () => { - const result = orgMemberResponse.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "admin", - name: "Alice Smith", - email: "alice@example.com", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects response missing name", () => { - const result = orgMemberResponse.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "member", - email: "alice@example.com", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); - - test("rejects response missing email", () => { - const result = orgMemberResponse.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "member", - name: "Alice", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/protocol/accounts/org-member.ts b/packages/protocol/accounts/org-member.ts deleted file mode 100644 index 5a0444a..0000000 --- a/packages/protocol/accounts/org-member.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Org member method schemas — params and results for org.member.* RPC methods. - */ -import { z } from "zod"; -import { orgRoleSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * org.member.list params. - */ -export const orgMemberListParams = z.object({ - orgId: uuidv7Schema, -}); - -export type OrgMemberListParams = z.infer; - -/** - * org.member.add params. - */ -export const orgMemberAddParams = z.object({ - orgId: uuidv7Schema, - identityId: uuidv7Schema, - role: orgRoleSchema, -}); - -export type OrgMemberAddParams = z.infer; - -/** - * org.member.remove params. - */ -export const orgMemberRemoveParams = z.object({ - orgId: uuidv7Schema, - identityId: uuidv7Schema, -}); - -export type OrgMemberRemoveParams = z.infer; - -/** - * org.member.updateRole params. - */ -export const orgMemberUpdateRoleParams = z.object({ - orgId: uuidv7Schema, - identityId: uuidv7Schema, - role: orgRoleSchema, -}); - -export type OrgMemberUpdateRoleParams = z.infer< - typeof orgMemberUpdateRoleParams ->; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single org member response — returned by add, used in list. - */ -export const orgMemberResponse = z.object({ - orgId: z.string(), - identityId: z.string(), - role: z.string(), - name: z.string(), - email: z.string(), - createdAt: z.string(), -}); - -export type OrgMemberResponse = z.infer; - -/** - * org.member.list result. - */ -export const orgMemberListResult = z.object({ - members: z.array(orgMemberResponse), -}); - -export type OrgMemberListResult = z.infer; - -/** - * org.member.remove result. - */ -export const orgMemberRemoveResult = z.object({ - removed: z.boolean(), -}); - -export type OrgMemberRemoveResult = z.infer; - -/** - * org.member.updateRole result. - */ -export const orgMemberUpdateRoleResult = z.object({ - updated: z.boolean(), -}); - -export type OrgMemberUpdateRoleResult = z.infer< - typeof orgMemberUpdateRoleResult ->; diff --git a/packages/protocol/accounts/org.ts b/packages/protocol/accounts/org.ts deleted file mode 100644 index 5bba03b..0000000 --- a/packages/protocol/accounts/org.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Org method schemas — params and results for org.* RPC methods. - */ -import { z } from "zod"; -import { nameSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * org.create params. - */ -export const orgCreateParams = z.object({ - name: nameSchema, -}); - -export type OrgCreateParams = z.infer; - -/** - * org.list params — no params needed, lists orgs for session identity. - */ -export const orgListParams = z.object({}); - -export type OrgListParams = z.infer; - -/** - * org.get params. - */ -export const orgGetParams = z.object({ - id: uuidv7Schema, -}); - -export type OrgGetParams = z.infer; - -/** - * org.update params. - */ -export const orgUpdateParams = z.object({ - id: uuidv7Schema, - name: nameSchema.optional(), -}); - -export type OrgUpdateParams = z.infer; - -/** - * org.delete params. - */ -export const orgDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type OrgDeleteParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single org response — returned by create, get, update. - */ -export const orgResponse = z.object({ - id: z.string(), - slug: z.string(), - name: z.string(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type OrgResponse = z.infer; - -/** - * org.list result. - */ -export const orgListResult = z.object({ - orgs: z.array(orgResponse), -}); - -export type OrgListResult = z.infer; - -/** - * org.delete result. - */ -export const orgDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type OrgDeleteResult = z.infer; diff --git a/packages/protocol/accounts/session.ts b/packages/protocol/accounts/session.ts deleted file mode 100644 index 97ce82d..0000000 --- a/packages/protocol/accounts/session.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Session method schemas — params and results for session.* RPC methods. - */ -import { z } from "zod"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * session.revoke params — revokes the current session (logout). - * No params needed — uses the session from the auth token. - */ -export const sessionRevokeParams = z.object({}); - -export type SessionRevokeParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * session.revoke result. - */ -export const sessionRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type SessionRevokeResult = z.infer; diff --git a/packages/protocol/engine/api-key.ts b/packages/protocol/engine/api-key.ts deleted file mode 100644 index ba17a23..0000000 --- a/packages/protocol/engine/api-key.ts +++ /dev/null @@ -1,112 +0,0 @@ -/** - * API Key method schemas — params and results for apiKey.* RPC methods. - */ -import { z } from "zod"; -import { timestampSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * apiKey.create params. - */ -export const apiKeyCreateParams = z.object({ - userId: uuidv7Schema, - name: z.string().min(1, "name is required"), - expiresAt: timestampSchema.optional().nullable(), -}); - -export type ApiKeyCreateParams = z.infer; - -/** - * apiKey.get params. - */ -export const apiKeyGetParams = z.object({ - id: uuidv7Schema, -}); - -export type ApiKeyGetParams = z.infer; - -/** - * apiKey.list params. - */ -export const apiKeyListParams = z.object({ - userId: uuidv7Schema, -}); - -export type ApiKeyListParams = z.infer; - -/** - * apiKey.revoke params. - */ -export const apiKeyRevokeParams = z.object({ - id: uuidv7Schema, -}); - -export type ApiKeyRevokeParams = z.infer; - -/** - * apiKey.delete params. - */ -export const apiKeyDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type ApiKeyDeleteParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single API key response — returned by get, included in list and create. - */ -export const apiKeyResponse = z.object({ - id: z.string(), - userId: z.string(), - lookupId: z.string(), - name: z.string(), - expiresAt: z.string().nullable(), - createdAt: z.string(), - revokedAt: z.string().nullable(), -}); - -export type ApiKeyResponse = z.infer; - -/** - * apiKey.create result — includes the raw key (only returned on creation). - */ -export const apiKeyCreateResult = z.object({ - apiKey: apiKeyResponse, - rawKey: z.string(), -}); - -export type ApiKeyCreateResult = z.infer; - -/** - * apiKey.list result. - */ -export const apiKeyListResult = z.object({ - apiKeys: z.array(apiKeyResponse), -}); - -export type ApiKeyListResult = z.infer; - -/** - * apiKey.revoke result. - */ -export const apiKeyRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type ApiKeyRevokeResult = z.infer; - -/** - * apiKey.delete result. - */ -export const apiKeyDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type ApiKeyDeleteResult = z.infer; diff --git a/packages/protocol/engine/grant.test.ts b/packages/protocol/engine/grant.test.ts deleted file mode 100644 index 839dfa3..0000000 --- a/packages/protocol/engine/grant.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Tests for grant protocol response schemas. - * - * Verifies the response schemas accept the userName field added via JOINs. - */ -import { describe, expect, test } from "bun:test"; -import { grantResponse } from "./grant.ts"; - -describe("grantResponse", () => { - test("accepts response with userName", () => { - const result = grantResponse.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - userId: "019d694f-79f6-7595-8faf-b70b01c11f99", - userName: "alice", - treePath: "work.projects", - actions: ["read", "create"], - grantedBy: null, - withGrantOption: false, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects response missing userName", () => { - const result = grantResponse.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - userId: "019d694f-79f6-7595-8faf-b70b01c11f99", - treePath: "work.projects", - actions: ["read"], - grantedBy: null, - withGrantOption: false, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/protocol/engine/grant.ts b/packages/protocol/engine/grant.ts deleted file mode 100644 index 4973e4e..0000000 --- a/packages/protocol/engine/grant.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Grant method schemas — params and results for grant.* RPC methods. - */ -import { z } from "zod"; -import { grantActionSchema, treePathSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * grant.create params. - */ -export const grantCreateParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, - actions: z.array(grantActionSchema).min(1, "at least one action required"), - withGrantOption: z.boolean().optional(), -}); - -export type GrantCreateParams = z.infer; - -/** - * grant.list params. - */ -export const grantListParams = z.object({ - userId: uuidv7Schema.optional(), -}); - -export type GrantListParams = z.infer; - -/** - * grant.get params. - */ -export const grantGetParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, -}); - -export type GrantGetParams = z.infer; - -/** - * grant.revoke params. - */ -export const grantRevokeParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, -}); - -export type GrantRevokeParams = z.infer; - -/** - * grant.check params. - */ -export const grantCheckParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, - action: grantActionSchema, -}); - -export type GrantCheckParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single grant response — returned by get. - */ -export const grantResponse = z.object({ - id: z.string(), - userId: z.string(), - userName: z.string(), - treePath: z.string(), - actions: z.array(z.string()), - grantedBy: z.string().nullable(), - withGrantOption: z.boolean(), - createdAt: z.string(), -}); - -export type GrantResponse = z.infer; - -/** - * grant.create result. - */ -export const grantCreateResult = z.object({ - created: z.boolean(), -}); - -export type GrantCreateResult = z.infer; - -/** - * grant.list result. - */ -export const grantListResult = z.object({ - grants: z.array(grantResponse), -}); - -export type GrantListResult = z.infer; - -/** - * grant.revoke result. - */ -export const grantRevokeResult = z.object({ - revoked: z.boolean(), -}); - -export type GrantRevokeResult = z.infer; - -/** - * grant.check result. - */ -export const grantCheckResult = z.object({ - allowed: z.boolean(), -}); - -export type GrantCheckResult = z.infer; diff --git a/packages/protocol/engine/index.ts b/packages/protocol/engine/index.ts deleted file mode 100644 index c035bda..0000000 --- a/packages/protocol/engine/index.ts +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Engine RPC contract — maps method names to params/result schema pairs. - * - * Covers all 35 methods on POST /api/v1/engine/rpc (API key auth). - */ -import type { z } from "zod"; - -// Domain schemas -import { - apiKeyCreateParams, - apiKeyCreateResult, - apiKeyDeleteParams, - apiKeyDeleteResult, - apiKeyGetParams, - apiKeyListParams, - apiKeyListResult, - apiKeyResponse, - apiKeyRevokeParams, - apiKeyRevokeResult, -} from "./api-key.ts"; -import { - grantCheckParams, - grantCheckResult, - grantCreateParams, - grantCreateResult, - grantGetParams, - grantListParams, - grantListResult, - grantResponse, - grantRevokeParams, - grantRevokeResult, -} from "./grant.ts"; -import { - memoryBatchCreateParams, - memoryBatchCreateResult, - memoryCountTreeParams, - memoryCountTreeResult, - memoryCreateParams, - memoryDeleteParams, - memoryDeleteResult, - memoryDeleteTreeParams, - memoryDeleteTreeResult, - memoryGetParams, - memoryMoveParams, - memoryMoveResult, - memoryResponse, - memorySearchParams, - memorySearchResult, - memoryTreeParams, - memoryTreeResult, - memoryUpdateParams, -} from "./memory.ts"; -import { - ownerGetParams, - ownerListParams, - ownerListResult, - ownerRemoveParams, - ownerRemoveResult, - ownerResponse, - ownerSetParams, - ownerSetResult, -} from "./owner.ts"; -import { - roleAddMemberParams, - roleAddMemberResult, - roleCreateParams, - roleListForUserParams, - roleListForUserResult, - roleListMembersParams, - roleListMembersResult, - roleRemoveMemberParams, - roleRemoveMemberResult, - roleResponse, -} from "./role.ts"; -import { - userCreateParams, - userDeleteParams, - userDeleteResult, - userGetByNameParams, - userGetParams, - userListParams, - userListResult, - userRenameParams, - userRenameResult, - userResponse, -} from "./user.ts"; - -export * from "./api-key.ts"; -export * from "./grant.ts"; -// Re-export all domain schemas -export * from "./memory.ts"; -export * from "./owner.ts"; -export * from "./role.ts"; -export * from "./user.ts"; - -// ============================================================================= -// RPC Contract -// ============================================================================= - -/** - * Define a method with its params schema and result schema. - */ -function method( - params: TParams, - result: TResult, -) { - return { params, result }; -} - -/** - * Engine RPC method contract — all 35 methods. - * - * Each entry maps a method name to its params and result Zod schemas. - * The client library uses this for type inference and optional response validation. - * The server uses the params schemas for input validation. - */ -export const engineMethods = { - // Memory (10) - "memory.create": method(memoryCreateParams, memoryResponse), - "memory.batchCreate": method( - memoryBatchCreateParams, - memoryBatchCreateResult, - ), - "memory.get": method(memoryGetParams, memoryResponse), - "memory.update": method(memoryUpdateParams, memoryResponse), - "memory.delete": method(memoryDeleteParams, memoryDeleteResult), - "memory.search": method(memorySearchParams, memorySearchResult), - "memory.tree": method(memoryTreeParams, memoryTreeResult), - "memory.move": method(memoryMoveParams, memoryMoveResult), - "memory.deleteTree": method(memoryDeleteTreeParams, memoryDeleteTreeResult), - "memory.countTree": method(memoryCountTreeParams, memoryCountTreeResult), - - // User (6) - "user.create": method(userCreateParams, userResponse), - "user.get": method(userGetParams, userResponse), - "user.getByName": method(userGetByNameParams, userResponse), - "user.list": method(userListParams, userListResult), - "user.rename": method(userRenameParams, userRenameResult), - "user.delete": method(userDeleteParams, userDeleteResult), - - // Grant (5) - "grant.create": method(grantCreateParams, grantCreateResult), - "grant.list": method(grantListParams, grantListResult), - "grant.get": method(grantGetParams, grantResponse), - "grant.revoke": method(grantRevokeParams, grantRevokeResult), - "grant.check": method(grantCheckParams, grantCheckResult), - - // Role (5) - "role.create": method(roleCreateParams, roleResponse), - "role.addMember": method(roleAddMemberParams, roleAddMemberResult), - "role.removeMember": method(roleRemoveMemberParams, roleRemoveMemberResult), - "role.listMembers": method(roleListMembersParams, roleListMembersResult), - "role.listForUser": method(roleListForUserParams, roleListForUserResult), - - // Owner (4) - "owner.set": method(ownerSetParams, ownerSetResult), - "owner.get": method(ownerGetParams, ownerResponse), - "owner.remove": method(ownerRemoveParams, ownerRemoveResult), - "owner.list": method(ownerListParams, ownerListResult), - - // API Key (5) - "apiKey.create": method(apiKeyCreateParams, apiKeyCreateResult), - "apiKey.get": method(apiKeyGetParams, apiKeyResponse), - "apiKey.list": method(apiKeyListParams, apiKeyListResult), - "apiKey.revoke": method(apiKeyRevokeParams, apiKeyRevokeResult), - "apiKey.delete": method(apiKeyDeleteParams, apiKeyDeleteResult), -} as const; - -// ============================================================================= -// Type Utilities -// ============================================================================= - -/** Union of all engine method names. */ -export type EngineMethodName = keyof typeof engineMethods; - -/** Extract the params type for a given engine method. */ -export type EngineParams = z.infer< - (typeof engineMethods)[M]["params"] ->; - -/** Extract the result type for a given engine method. */ -export type EngineResult = z.infer< - (typeof engineMethods)[M]["result"] ->; - -/** Get the params schema for runtime validation. */ -export function getEngineParamsSchema(method: M) { - return engineMethods[method].params; -} - -/** Get the result schema for runtime response validation. */ -export function getEngineResultSchema(method: M) { - return engineMethods[method].result; -} diff --git a/packages/protocol/engine/owner.test.ts b/packages/protocol/engine/owner.test.ts deleted file mode 100644 index 661949e..0000000 --- a/packages/protocol/engine/owner.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Tests for owner protocol response schemas. - * - * Verifies the response schemas accept userName and createdByName. - */ -import { describe, expect, test } from "bun:test"; -import { ownerResponse } from "./owner.ts"; - -describe("ownerResponse", () => { - test("accepts response with userName and createdByName", () => { - const result = ownerResponse.safeParse({ - treePath: "work.projects", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: "019d694f-79f6-7595-8faf-b70b01c11f99", - createdByName: "admin", - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("accepts null createdBy and createdByName", () => { - const result = ownerResponse.safeParse({ - treePath: "work", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: null, - createdByName: null, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects response missing userName", () => { - const result = ownerResponse.safeParse({ - treePath: "work", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - createdBy: null, - createdByName: null, - createdAt: "2026-01-15T00:00:00.000Z", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/protocol/engine/owner.ts b/packages/protocol/engine/owner.ts deleted file mode 100644 index be0da06..0000000 --- a/packages/protocol/engine/owner.ts +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Owner method schemas — params and results for owner.* RPC methods. - */ -import { z } from "zod"; -import { treePathSchema, uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * owner.set params. - */ -export const ownerSetParams = z.object({ - userId: uuidv7Schema, - treePath: treePathSchema, -}); - -export type OwnerSetParams = z.infer; - -/** - * owner.remove params. - */ -export const ownerRemoveParams = z.object({ - treePath: treePathSchema, -}); - -export type OwnerRemoveParams = z.infer; - -/** - * owner.get params. - */ -export const ownerGetParams = z.object({ - treePath: treePathSchema, -}); - -export type OwnerGetParams = z.infer; - -/** - * owner.list params. - */ -export const ownerListParams = z.object({ - userId: uuidv7Schema.optional(), -}); - -export type OwnerListParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single owner response. - */ -export const ownerResponse = z.object({ - treePath: z.string(), - userId: z.string(), - userName: z.string(), - createdBy: z.string().nullable(), - createdByName: z.string().nullable(), - createdAt: z.string(), -}); - -export type OwnerResponse = z.infer; - -/** - * owner.set result. - */ -export const ownerSetResult = z.object({ - set: z.boolean(), -}); - -export type OwnerSetResult = z.infer; - -/** - * owner.remove result. - */ -export const ownerRemoveResult = z.object({ - removed: z.boolean(), -}); - -export type OwnerRemoveResult = z.infer; - -/** - * owner.list result. - */ -export const ownerListResult = z.object({ - owners: z.array(ownerResponse), -}); - -export type OwnerListResult = z.infer; diff --git a/packages/protocol/engine/role.ts b/packages/protocol/engine/role.ts deleted file mode 100644 index 4fa3b0e..0000000 --- a/packages/protocol/engine/role.ts +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Role method schemas — params and results for role.* RPC methods. - */ -import { z } from "zod"; -import { uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * role.create params. - * Creates a user with canLogin=false (a role for grouping grants). - */ -export const roleCreateParams = z.object({ - name: z.string().min(1, "name is required"), - identityId: uuidv7Schema.optional().nullable(), -}); - -export type RoleCreateParams = z.infer; - -/** - * role.addMember params. - */ -export const roleAddMemberParams = z.object({ - roleId: uuidv7Schema, - memberId: uuidv7Schema, - withAdminOption: z.boolean().optional(), -}); - -export type RoleAddMemberParams = z.infer; - -/** - * role.removeMember params. - */ -export const roleRemoveMemberParams = z.object({ - roleId: uuidv7Schema, - memberId: uuidv7Schema, -}); - -export type RoleRemoveMemberParams = z.infer; - -/** - * role.listMembers params. - */ -export const roleListMembersParams = z.object({ - roleId: uuidv7Schema, -}); - -export type RoleListMembersParams = z.infer; - -/** - * role.listForUser params. - */ -export const roleListForUserParams = z.object({ - userId: uuidv7Schema, -}); - -export type RoleListForUserParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single role response — returned by create. - */ -export const roleResponse = z.object({ - id: z.string(), - name: z.string(), - identityId: z.string().nullable(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type RoleResponse = z.infer; - -/** - * Role member response — used in listMembers result. - */ -export const roleMemberResponse = z.object({ - roleId: z.string(), - memberId: z.string(), - memberName: z.string(), - withAdminOption: z.boolean(), - createdAt: z.string(), -}); - -export type RoleMemberResponse = z.infer; - -/** - * Role info response — used in listForUser result. - */ -export const roleInfoResponse = z.object({ - id: z.string(), - name: z.string(), - withAdminOption: z.boolean(), -}); - -export type RoleInfoResponse = z.infer; - -/** - * role.addMember result. - */ -export const roleAddMemberResult = z.object({ - added: z.boolean(), -}); - -export type RoleAddMemberResult = z.infer; - -/** - * role.removeMember result. - */ -export const roleRemoveMemberResult = z.object({ - removed: z.boolean(), -}); - -export type RoleRemoveMemberResult = z.infer; - -/** - * role.listMembers result. - */ -export const roleListMembersResult = z.object({ - members: z.array(roleMemberResponse), -}); - -export type RoleListMembersResult = z.infer; - -/** - * role.listForUser result. - */ -export const roleListForUserResult = z.object({ - roles: z.array(roleInfoResponse), -}); - -export type RoleListForUserResult = z.infer; diff --git a/packages/protocol/engine/user.ts b/packages/protocol/engine/user.ts deleted file mode 100644 index ed2ee88..0000000 --- a/packages/protocol/engine/user.ts +++ /dev/null @@ -1,116 +0,0 @@ -/** - * User method schemas — params and results for user.* RPC methods. - */ -import { z } from "zod"; -import { uuidv7Schema } from "../fields.ts"; - -// ============================================================================= -// Params Schemas -// ============================================================================= - -/** - * user.create params. - */ -export const userCreateParams = z.object({ - id: uuidv7Schema.optional().nullable(), - name: z.string().min(1, "name is required"), - identityId: uuidv7Schema.optional().nullable(), - canLogin: z.boolean().optional(), - superuser: z.boolean().optional(), - createrole: z.boolean().optional(), -}); - -export type UserCreateParams = z.infer; - -/** - * user.get params. - */ -export const userGetParams = z.object({ - id: uuidv7Schema, -}); - -export type UserGetParams = z.infer; - -/** - * user.getByName params. - */ -export const userGetByNameParams = z.object({ - name: z.string().min(1), -}); - -export type UserGetByNameParams = z.infer; - -/** - * user.list params. - */ -export const userListParams = z.object({ - canLogin: z.boolean().optional(), -}); - -export type UserListParams = z.infer; - -/** - * user.rename params. - */ -export const userRenameParams = z.object({ - id: uuidv7Schema, - name: z.string().min(1, "name is required"), -}); - -export type UserRenameParams = z.infer; - -/** - * user.delete params. - */ -export const userDeleteParams = z.object({ - id: uuidv7Schema, -}); - -export type UserDeleteParams = z.infer; - -// ============================================================================= -// Result Schemas -// ============================================================================= - -/** - * Single user response — returned by create, get, getByName. - */ -export const userResponse = z.object({ - id: z.string(), - name: z.string(), - identityId: z.string().nullable(), - canLogin: z.boolean(), - superuser: z.boolean(), - createrole: z.boolean(), - createdAt: z.string(), - updatedAt: z.string().nullable(), -}); - -export type UserResponse = z.infer; - -/** - * user.list result. - */ -export const userListResult = z.object({ - users: z.array(userResponse), -}); - -export type UserListResult = z.infer; - -/** - * user.rename result. - */ -export const userRenameResult = z.object({ - renamed: z.boolean(), -}); - -export type UserRenameResult = z.infer; - -/** - * user.delete result. - */ -export const userDeleteResult = z.object({ - deleted: z.boolean(), -}); - -export type UserDeleteResult = z.infer; diff --git a/packages/protocol/errors.ts b/packages/protocol/errors.ts index 7579b37..3a16872 100644 --- a/packages/protocol/errors.ts +++ b/packages/protocol/errors.ts @@ -39,6 +39,8 @@ export const APP_ERROR_CODES = { FORBIDDEN: "FORBIDDEN", NOT_FOUND: "NOT_FOUND", CONFLICT: "CONFLICT", + /** Operation would leave a space without any admin (the last-admin safeguard). */ + LAST_ADMIN: "LAST_ADMIN", RATE_LIMITED: "RATE_LIMITED", VALIDATION_ERROR: "VALIDATION_ERROR", EMBEDDING_NOT_CONFIGURED: "EMBEDDING_NOT_CONFIGURED", diff --git a/packages/protocol/fields.ts b/packages/protocol/fields.ts index bb68d1d..1e93f4e 100644 --- a/packages/protocol/fields.ts +++ b/packages/protocol/fields.ts @@ -26,20 +26,41 @@ export const timestampSchema = z.iso.datetime({ offset: true }); // ============================================================================= /** - * ltree path pattern (alphanumeric and underscores, dot-separated). + * User-facing tree-path input pattern. This is the *lenient* wire form, not the + * canonical ltree: separators may be `.` or `/`, a leading `~` is the home + * shortcut, and labels are ltree labels (`[A-Za-z0-9_-]`). The empty string is + * the root. Every handler that accepts this normalizes it server-side via + * `normalizeTreePath` (see packages/database/space/path.ts), which is the + * authoritative validator — it rejects malformed labels and a misplaced `~` + * with a TreePathError mapped to a validation error. This regex is only a cheap + * shape gate so obviously-bad characters (spaces, etc.) fail fast. + * + * Keeping this lenient (rather than the strict canonical ltree) is required so + * the documented `~`/`share` conventions and slash separators actually work + * over the wire on create/update/move/tree/grant — all of which normalize. */ -const ltreePattern = /^([A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*)?$/; +const treePathInputPattern = /^[A-Za-z0-9_~./-]*$/; /** - * Tree path schema (ltree format, allows empty string for root). + * Tree path schema (lenient user-facing input; allows empty string for root). */ export const treePathSchema = z .string() .regex( - ltreePattern, - "must be a valid ltree path (alphanumeric/underscore, dot-separated)", + treePathInputPattern, + "must be a tree path (labels [A-Za-z0-9_-], '.' or '/' separated, optional leading '~')", ); +/** + * The reserved shared-tree root. This is the single source of truth for the + * `"share"` literal across the codebase (the database boundary re-exports it). + * It is the conventional default for memories that should be visible to the + * whole space — `memory.create`/`batchCreate` now require an explicit `tree`, + * so callers that previously relied on a server-side default (the file + * importers) default to this. + */ +export const SHARE_NAMESPACE = "share"; + /** * Tree filter schema (ltree, lquery, or ltxtquery). * More permissive than treePathSchema since it allows query operators. diff --git a/packages/protocol/headers.ts b/packages/protocol/headers.ts new file mode 100644 index 0000000..fa1377c --- /dev/null +++ b/packages/protocol/headers.ts @@ -0,0 +1,17 @@ +/** + * HTTP header names shared between the server and clients. + */ + +/** + * Header the client uses to advertise its CLIENT_VERSION on every RPC. + * + * The server short-circuits requests with an incompatible client version + * before dispatching them to a handler. + */ +export const CLIENT_VERSION_HEADER = "X-Client-Version"; + +/** + * Header the client sends to select which space a memory-endpoint request + * targets (both session and api-key auth). The value is the active space slug. + */ +export const SPACE_HEADER = "X-Me-Space"; diff --git a/packages/protocol/index.ts b/packages/protocol/index.ts index 6081843..67377a1 100644 --- a/packages/protocol/index.ts +++ b/packages/protocol/index.ts @@ -5,22 +5,24 @@ * client and server. Both the server (validation) and client libraries * (type inference + optional response validation) import from here. * - * Two RPC endpoints, two contracts: - * - Engine RPC (POST /api/v1/engine/rpc) — API key auth, 30 methods - * - Accounts RPC (POST /api/v1/accounts/rpc) — session token auth, 19 methods + * RPC endpoints / contracts: + * - Memory RPC (POST /api/v1/memory/rpc) — session or api-key auth; the memory + * data plane (./memory) + the space management contract (./space). + * - User RPC (POST /api/v1/user/rpc) — session auth; whoami + agent + space + * discovery (./user). */ -// Accounts RPC contract + all accounts schemas -export * from "./accounts/index.ts"; // Device flow auth schemas export * from "./auth/device-flow.ts"; -// Engine RPC contract + all engine schemas -export * from "./engine/index.ts"; // Error codes and AppError export * from "./errors.ts"; // Shared field validators export * from "./fields.ts"; +// HTTP header names +export * from "./headers.ts"; // JSON-RPC 2.0 envelope types export * from "./jsonrpc.ts"; +// Memory data-plane schemas +export * from "./memory.ts"; // Version compatibility schemas export * from "./version.ts"; diff --git a/packages/protocol/engine/memory.ts b/packages/protocol/memory.ts similarity index 87% rename from packages/protocol/engine/memory.ts rename to packages/protocol/memory.ts index da65f12..42286ac 100644 --- a/packages/protocol/engine/memory.ts +++ b/packages/protocol/memory.ts @@ -10,7 +10,7 @@ import { treeFilterSchema, treePathSchema, uuidv7Schema, -} from "../fields.ts"; +} from "./fields.ts"; // ============================================================================= // Params Schemas @@ -23,7 +23,7 @@ export const memoryCreateParams = z.object({ id: uuidv7Schema.optional().nullable(), content: z.string().min(1, "content is required"), meta: metaSchema.optional().nullable(), - tree: treePathSchema.optional().nullable(), + tree: treePathSchema.min(1, "tree path is required"), temporal: temporalSchema.optional().nullable(), }); @@ -31,6 +31,12 @@ export type MemoryCreateParams = z.infer; /** * memory.batchCreate params. + * + * `replaceIfMetaDiffers` names a meta key for conditional replace: a memory + * whose explicit id already exists is rewritten in place when the stored + * row's value for that key differs from the submitted one (deterministic-id + * importers pass e.g. "importer_version" so version bumps re-render existing + * rows), and skipped when it matches. Without it, duplicates are skipped. */ export const memoryBatchCreateParams = z.object({ memories: z @@ -39,12 +45,13 @@ export const memoryBatchCreateParams = z.object({ id: uuidv7Schema.optional().nullable(), content: z.string().min(1, "content is required"), meta: metaSchema.optional().nullable(), - tree: treePathSchema.optional().nullable(), + tree: treePathSchema.min(1, "tree path is required"), temporal: temporalSchema.optional().nullable(), }), ) .min(1, "at least one memory required") .max(1000, "maximum 1000 memories per batch"), + replaceIfMetaDiffers: z.string().min(1).optional().nullable(), }); export type MemoryBatchCreateParams = z.infer; @@ -176,9 +183,15 @@ export type MemoryWithScoreResponse = z.infer; /** * memory.batchCreate result. + * + * `ids` are the freshly inserted memories; `updatedIds` are existing rows + * rewritten in place via `replaceIfMetaDiffers`. A submitted explicit id in + * neither array (and not in a failed request) was skipped — it already + * existed at the same meta-key value. */ export const memoryBatchCreateResult = z.object({ ids: z.array(z.string()), + updatedIds: z.array(z.string()), }); export type MemoryBatchCreateResult = z.infer; diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 52b3670..32638b8 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -29,30 +29,40 @@ "types": "./errors.ts", "import": "./dist/errors.js" }, + "./headers": { + "bun": "./headers.ts", + "types": "./headers.ts", + "import": "./dist/headers.js" + }, "./version": { "bun": "./version.ts", "types": "./version.ts", "import": "./dist/version.js" }, - "./engine": { - "bun": "./engine/index.ts", - "types": "./engine/index.ts", - "import": "./dist/engine/index.js" + "./memory": { + "bun": "./memory.ts", + "types": "./memory.ts", + "import": "./dist/memory.js" + }, + "./space": { + "bun": "./space/index.ts", + "types": "./space/index.ts", + "import": "./dist/space/index.js" }, - "./engine/*": { - "bun": "./engine/*.ts", - "types": "./engine/*.ts", - "import": "./dist/engine/*.js" + "./space/*": { + "bun": "./space/*.ts", + "types": "./space/*.ts", + "import": "./dist/space/*.js" }, - "./accounts": { - "bun": "./accounts/index.ts", - "types": "./accounts/index.ts", - "import": "./dist/accounts/index.js" + "./user": { + "bun": "./user/index.ts", + "types": "./user/index.ts", + "import": "./dist/user/index.js" }, - "./accounts/*": { - "bun": "./accounts/*.ts", - "types": "./accounts/*.ts", - "import": "./dist/accounts/*.js" + "./user/*": { + "bun": "./user/*.ts", + "types": "./user/*.ts", + "import": "./dist/user/*.js" }, "./auth/*": { "bun": "./auth/*.ts", @@ -63,13 +73,13 @@ "files": [ "dist", "*.ts", - "engine/*.ts", - "accounts/*.ts", + "space/*.ts", + "user/*.ts", "auth/*.ts", "!*.test.ts" ], "scripts": { - "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts version.ts engine/index.ts engine/memory.ts engine/user.ts engine/grant.ts engine/owner.ts engine/role.ts engine/api-key.ts accounts/index.ts accounts/engine.ts accounts/identity.ts accounts/invitation.ts accounts/org.ts accounts/org-member.ts accounts/session.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", + "build:js": "../../bun build index.ts fields.ts jsonrpc.ts errors.ts headers.ts version.ts memory.ts space/index.ts space/principal.ts space/group.ts space/grant.ts space/api-key.ts user/index.ts user/agent.ts user/space.ts user/whoami.ts auth/device-flow.ts --outdir dist --format esm --target node --packages external --splitting", "build:types": "tsc -p tsconfig.build.json", "build": "../../bun run build:js && ../../bun run build:types", "prepublishOnly": "../../bun run build" diff --git a/packages/protocol/space/grant.ts b/packages/protocol/space/grant.ts new file mode 100644 index 0000000..62bed10 --- /dev/null +++ b/packages/protocol/space/grant.ts @@ -0,0 +1,88 @@ +/** + * Tree-access grant method schemas (grant.*). + * + * The core model uses three additive levels (1 = read, 2 = write, 3 = owner); + * there are no per-action grants and no deny entries. Owner listing is grant.list + * filtered to access = 3. + */ +import { z } from "zod"; +import { treePathSchema, uuidv7Schema } from "../fields.ts"; + +/** Access level: 1 = read, 2 = write, 3 = owner. */ +export const accessLevelSchema = z.union([ + z.literal(1), + z.literal(2), + z.literal(3), +]); +export type AccessLevel = z.infer; + +/** The canonical name for an access level (1 → "read", 2 → "write", 3 → "owner"). */ +export function accessLevelName( + level: AccessLevel, +): "read" | "write" | "owner" { + return level === 1 ? "read" : level === 2 ? "write" : "owner"; +} + +/** + * Parse a human access-level string to its numeric level, or null if unknown. + * Accepts the full names (read/write/owner) and single-letter forms (r/w/o), + * case-insensitively. The "none" sentinel (no grant) is the caller's concern. + */ +export function parseAccessLevel(input: string): AccessLevel | null { + switch (input.trim().toLowerCase()) { + case "r": + case "read": + return 1; + case "w": + case "write": + return 2; + case "o": + case "owner": + return 3; + default: + return null; + } +} + +export const treeGrantResponse = z.object({ + principalId: z.string(), + treePath: z.string(), + access: accessLevelSchema, + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type TreeGrantResponse = z.infer; + +// grant.set — grant or update a principal's access at a tree path +export const grantSetParams = z.object({ + principalId: uuidv7Schema, + treePath: treePathSchema, + access: accessLevelSchema, +}); +export type GrantSetParams = z.infer; + +export const grantSetResult = z.object({ granted: z.boolean() }); +export type GrantSetResult = z.infer; + +// grant.remove +export const grantRemoveParams = z.object({ + principalId: uuidv7Schema, + treePath: treePathSchema, +}); +export type GrantRemoveParams = z.infer; + +export const grantRemoveResult = z.object({ removed: z.boolean() }); +export type GrantRemoveResult = z.infer; + +// grant.list — optionally filtered to a principal and/or a subtree path +export const grantListParams = z.object({ + principalId: uuidv7Schema.optional().nullable(), + /** Only grants at or below this tree path (requires owning the path). */ + treePath: treePathSchema.optional().nullable(), +}); +export type GrantListParams = z.infer; + +export const grantListResult = z.object({ + grants: z.array(treeGrantResponse), +}); +export type GrantListResult = z.infer; diff --git a/packages/protocol/space/group.ts b/packages/protocol/space/group.ts new file mode 100644 index 0000000..87018f6 --- /dev/null +++ b/packages/protocol/space/group.ts @@ -0,0 +1,105 @@ +/** + * Group method schemas (group.*). + * + * Groups are space-scoped principals used to bundle members for tree-access + * grants. Group membership confers space access (a group member is a space + * member, flagged direct=false in principal.list). + */ +import { z } from "zod"; +import { nameSchema, uuidv7Schema } from "../fields.ts"; +import { principalKindSchema } from "./principal.ts"; + +export const groupResponse = z.object({ + id: z.string(), + name: z.string(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type GroupResponse = z.infer; + +export const groupMemberResponse = z.object({ + memberId: z.string(), + kind: principalKindSchema, + name: z.string(), + admin: z.boolean(), + createdAt: z.string(), +}); +export type GroupMemberResponse = z.infer; + +export const groupMembershipResponse = z.object({ + groupId: z.string(), + name: z.string(), + admin: z.boolean(), + createdAt: z.string(), +}); +export type GroupMembershipResponse = z.infer; + +// group.create +export const groupCreateParams = z.object({ name: nameSchema }); +export type GroupCreateParams = z.infer; + +export const groupCreateResult = z.object({ id: z.string() }); +export type GroupCreateResult = z.infer; + +// group.list +export const groupListParams = z.object({}); +export type GroupListParams = z.infer; + +export const groupListResult = z.object({ groups: z.array(groupResponse) }); +export type GroupListResult = z.infer; + +// group.rename +export const groupRenameParams = z.object({ + id: uuidv7Schema, + name: nameSchema, +}); +export type GroupRenameParams = z.infer; + +export const groupRenameResult = z.object({ renamed: z.boolean() }); +export type GroupRenameResult = z.infer; + +// group.delete +export const groupDeleteParams = z.object({ id: uuidv7Schema }); +export type GroupDeleteParams = z.infer; + +export const groupDeleteResult = z.object({ deleted: z.boolean() }); +export type GroupDeleteResult = z.infer; + +// group.addMember +export const groupAddMemberParams = z.object({ + groupId: uuidv7Schema, + memberId: uuidv7Schema, + admin: z.boolean().optional(), +}); +export type GroupAddMemberParams = z.infer; + +export const groupAddMemberResult = z.object({ added: z.boolean() }); +export type GroupAddMemberResult = z.infer; + +// group.removeMember +export const groupRemoveMemberParams = z.object({ + groupId: uuidv7Schema, + memberId: uuidv7Schema, +}); +export type GroupRemoveMemberParams = z.infer; + +export const groupRemoveMemberResult = z.object({ removed: z.boolean() }); +export type GroupRemoveMemberResult = z.infer; + +// group.listMembers +export const groupListMembersParams = z.object({ groupId: uuidv7Schema }); +export type GroupListMembersParams = z.infer; + +export const groupListMembersResult = z.object({ + members: z.array(groupMemberResponse), +}); +export type GroupListMembersResult = z.infer; + +// group.listForMember +export const groupListForMemberParams = z.object({ memberId: uuidv7Schema }); +export type GroupListForMemberParams = z.infer; + +export const groupListForMemberResult = z.object({ + groups: z.array(groupMembershipResponse), +}); +export type GroupListForMemberResult = z.infer; diff --git a/packages/protocol/space/index.ts b/packages/protocol/space/index.ts new file mode 100644 index 0000000..40dea9f --- /dev/null +++ b/packages/protocol/space/index.ts @@ -0,0 +1,115 @@ +/** + * Space management RPC contract — the control-plane methods served on + * POST /api/v1/memory/rpc alongside the memory.* data-plane methods. + * + * Follows the core model: principals (users/agents/groups), space membership, + * group membership, and 3-level tree-access grants. (Agent lifecycle and api + * keys are user-scoped and live on the user endpoint; here agents are only + * referenced as members.) + */ +import type { z } from "zod"; +import { + grantListParams, + grantListResult, + grantRemoveParams, + grantRemoveResult, + grantSetParams, + grantSetResult, +} from "./grant.ts"; +import { + groupAddMemberParams, + groupAddMemberResult, + groupCreateParams, + groupCreateResult, + groupDeleteParams, + groupDeleteResult, + groupListForMemberParams, + groupListForMemberResult, + groupListMembersParams, + groupListMembersResult, + groupListParams, + groupListResult, + groupRemoveMemberParams, + groupRemoveMemberResult, + groupRenameParams, + groupRenameResult, +} from "./group.ts"; +import { + inviteCreateParams, + inviteCreateResult, + inviteListParams, + inviteListResult, + inviteRevokeParams, + inviteRevokeResult, +} from "./invitation.ts"; +import { + principalAddParams, + principalAddResult, + principalListParams, + principalListResult, + principalLookupParams, + principalLookupResult, + principalRemoveParams, + principalRemoveResult, + principalResolveParams, + principalResolveResult, +} from "./principal.ts"; + +export * from "./grant.ts"; +export * from "./group.ts"; +export * from "./invitation.ts"; +export * from "./principal.ts"; + +function method( + params: TParams, + result: TResult, +) { + return { params, result }; +} + +/** + * Space management RPC method contract (member/group/grant/invite). + * Served on the memory endpoint together with the memory.* methods. + */ +export const spaceMethods = { + // Membership (4) — the space roster holds principals (user | agent | group) + "principal.list": method(principalListParams, principalListResult), + "principal.add": method(principalAddParams, principalAddResult), + "principal.remove": method(principalRemoveParams, principalRemoveResult), + "principal.resolve": method(principalResolveParams, principalResolveResult), + "principal.lookup": method(principalLookupParams, principalLookupResult), + + // Groups (8) + "group.create": method(groupCreateParams, groupCreateResult), + "group.list": method(groupListParams, groupListResult), + "group.rename": method(groupRenameParams, groupRenameResult), + "group.delete": method(groupDeleteParams, groupDeleteResult), + "group.addMember": method(groupAddMemberParams, groupAddMemberResult), + "group.removeMember": method( + groupRemoveMemberParams, + groupRemoveMemberResult, + ), + "group.listMembers": method(groupListMembersParams, groupListMembersResult), + "group.listForMember": method( + groupListForMemberParams, + groupListForMemberResult, + ), + + // Grants (3) + "grant.set": method(grantSetParams, grantSetResult), + "grant.remove": method(grantRemoveParams, grantRemoveResult), + "grant.list": method(grantListParams, grantListResult), + + // Invitations (3) — email-keyed; adds existing users now, else pending + "invite.create": method(inviteCreateParams, inviteCreateResult), + "invite.list": method(inviteListParams, inviteListResult), + "invite.revoke": method(inviteRevokeParams, inviteRevokeResult), +} as const; + +export type SpaceMethodName = keyof typeof spaceMethods; +export type SpaceParams = z.infer< + (typeof spaceMethods)[M]["params"] +>; +export type SpaceResult = z.infer< + (typeof spaceMethods)[M]["result"] +>; diff --git a/packages/protocol/space/invitation.ts b/packages/protocol/space/invitation.ts new file mode 100644 index 0000000..d3f8fa3 --- /dev/null +++ b/packages/protocol/space/invitation.ts @@ -0,0 +1,61 @@ +/** + * Space invitation method schemas (invite.*). + * + * Invitations are keyed by invitee email so an invite can be issued before the + * user registers. Inviting an *already-registered* user adds them to the space + * immediately (instant access on their existing session); inviting a not-yet- + * registered email records a pending invitation, redeemed at their first + * verified login. Each invite carries whether to make the user a space admin and + * an optional share level (read/write/owner at the shared root; null = none). + */ +import { z } from "zod"; +import { emailSchema } from "../fields.ts"; +import { accessLevelSchema } from "./grant.ts"; + +/** A pending invitation to the space. */ +export const spaceInvitationResponse = z.object({ + id: z.string(), + email: z.string(), + admin: z.boolean(), + /** Share-root access granted on redemption; null = no share grant. */ + shareAccess: accessLevelSchema.nullable(), + invitedBy: z.string().nullable(), + invitedByName: z.string().nullable(), + createdAt: z.string(), +}); +export type SpaceInvitationResponse = z.infer; + +// invite.create — invite by email; adds an existing user now, else records a +// pending invite. `admin` defaults false; `shareAccess` null/omitted = no share. +export const inviteCreateParams = z.object({ + email: emailSchema, + admin: z.boolean().optional(), + shareAccess: accessLevelSchema.nullable().optional(), +}); +export type InviteCreateParams = z.infer; + +export const inviteCreateResult = z.object({ + /** True when the invitee was an existing user and was added to the space now. */ + applied: z.boolean(), + /** The pending invitation id (null when applied immediately). */ + invitationId: z.string().nullable(), + /** The principal added now (null when deferred to a pending invitation). */ + principalId: z.string().nullable(), +}); +export type InviteCreateResult = z.infer; + +// invite.list — pending invitations for the space +export const inviteListParams = z.object({}); +export type InviteListParams = z.infer; + +export const inviteListResult = z.object({ + invitations: z.array(spaceInvitationResponse), +}); +export type InviteListResult = z.infer; + +// invite.revoke — delete a pending invitation by email +export const inviteRevokeParams = z.object({ email: emailSchema }); +export type InviteRevokeParams = z.infer; + +export const inviteRevokeResult = z.object({ revoked: z.boolean() }); +export type InviteRevokeResult = z.infer; diff --git a/packages/protocol/space/principal.ts b/packages/protocol/space/principal.ts new file mode 100644 index 0000000..2318df7 --- /dev/null +++ b/packages/protocol/space/principal.ts @@ -0,0 +1,102 @@ +/** + * Space membership method schemas (principal.*). + * + * The space management API, served on POST /api/v1/memory/rpc, follows the core + * model: principals (users/agents/groups), space membership, group membership, + * 3-level tree-access grants, and agent api keys. All methods are scoped to the + * space selected by the X-Me-Space header and require space-owner authority. + * + * "Principal" is the union (user | agent | group); the space roster holds + * principals. "Member" is reserved for the user/agent sense (group members, + * api-key holders). + */ +import { z } from "zod"; +import { nameSchema, uuidv7Schema } from "../fields.ts"; + +/** Principal kind: user / group / agent. */ +export const principalKindSchema = z.enum(["u", "g", "a"]); +export type PrincipalKind = z.infer; + +/** + * A principal that belongs to a space — directly or via a group. + * `direct` is true for a direct membership; `admin` is the direct-membership + * admin flag (false for group-only members). + */ +export const spacePrincipalResponse = z.object({ + id: z.string(), + kind: principalKindSchema, + name: z.string(), + ownerId: z.string().nullable(), + direct: z.boolean(), + admin: z.boolean(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type SpacePrincipalResponse = z.infer; + +/** A principal reference: the minimal shape returned by resolve / lookup. */ +export const principalRef = z.object({ + id: z.string(), + kind: principalKindSchema, + name: z.string(), +}); +export type PrincipalRef = z.infer; + +// principal.list +export const principalListParams = z.object({ + kind: principalKindSchema.optional().nullable(), +}); +export type PrincipalListParams = z.infer; + +export const principalListResult = z.object({ + principals: z.array(spacePrincipalResponse), +}); +export type PrincipalListResult = z.infer; + +// principal.add +export const principalAddParams = z.object({ + principalId: uuidv7Schema, + admin: z.boolean().optional(), +}); +export type PrincipalAddParams = z.infer; + +export const principalAddResult = z.object({ added: z.boolean() }); +export type PrincipalAddResult = z.infer; + +// principal.remove +export const principalRemoveParams = z.object({ principalId: uuidv7Schema }); +export type PrincipalRemoveParams = z.infer; + +export const principalRemoveResult = z.object({ removed: z.boolean() }); +export type PrincipalRemoveResult = z.infer; + +// principal.resolve — resolve principals in this space by exact name +// (case-insensitive), optionally constrained to a kind. Available to any space +// member: a targeted name->id lookup, not roster enumeration (that is +// principal.list). Returns all matches so the caller can detect ambiguity. +export const principalResolveParams = z.object({ + name: z.string().min(1), + kind: principalKindSchema.optional().nullable(), +}); +export type PrincipalResolveParams = z.infer; + +export const principalResolveResult = z.object({ + principals: z.array(principalRef), +}); +export type PrincipalResolveResult = z.infer; + +// principal.lookup — reverse lookup: resolve a batch of principal ids to their +// names/kinds (for display, e.g. grant listings). Available to any space member; +// only ids that are in the space come back (you cannot enumerate by guessing). +export const principalLookupParams = z.object({ + ids: z.array(uuidv7Schema), +}); +export type PrincipalLookupParams = z.infer; + +export const principalLookupResult = z.object({ + principals: z.array(principalRef), +}); +export type PrincipalLookupResult = z.infer; + +// shared by agent.* / group.* mutation results +export { nameSchema }; diff --git a/packages/protocol/user/agent.ts b/packages/protocol/user/agent.ts new file mode 100644 index 0000000..90e3bfe --- /dev/null +++ b/packages/protocol/user/agent.ts @@ -0,0 +1,49 @@ +/** + * Agent method schemas (agent.*) for the user RPC. + * + * Agents are a user's global service accounts (owned by the user, names unique + * per user, not scoped to a space). Their lifecycle lives on the session-only + * user endpoint (POST /api/v1/user/rpc); bringing an agent into a space and + * minting its (space-bound) api key are space-endpoint operations. + */ +import { z } from "zod"; +import { nameSchema, uuidv7Schema } from "../fields.ts"; + +export const agentResponse = z.object({ + id: z.string(), + name: z.string(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type AgentResponse = z.infer; + +// agent.create — create an agent owned by the calling user +export const agentCreateParams = z.object({ name: nameSchema }); +export type AgentCreateParams = z.infer; + +export const agentCreateResult = z.object({ id: z.string() }); +export type AgentCreateResult = z.infer; + +// agent.list — the caller's agents +export const agentListParams = z.object({}); +export type AgentListParams = z.infer; + +export const agentListResult = z.object({ agents: z.array(agentResponse) }); +export type AgentListResult = z.infer; + +// agent.rename +export const agentRenameParams = z.object({ + id: uuidv7Schema, + name: nameSchema, +}); +export type AgentRenameParams = z.infer; + +export const agentRenameResult = z.object({ renamed: z.boolean() }); +export type AgentRenameResult = z.infer; + +// agent.delete +export const agentDeleteParams = z.object({ id: uuidv7Schema }); +export type AgentDeleteParams = z.infer; + +export const agentDeleteResult = z.object({ deleted: z.boolean() }); +export type AgentDeleteResult = z.infer; diff --git a/packages/protocol/user/api-key.ts b/packages/protocol/user/api-key.ts new file mode 100644 index 0000000..5325ad8 --- /dev/null +++ b/packages/protocol/user/api-key.ts @@ -0,0 +1,60 @@ +/** + * Api key method schemas (apiKey.*). + * + * Keys are agent-only (humans authenticate via session) and global per-principal + * — not bound to a space. The plaintext key is returned exactly once, by + * apiKey.create. There is no soft-revoke state: apiKey.delete is the only + * removal (revoke ≡ delete). + */ +import { z } from "zod"; +import { nameSchema, timestampSchema, uuidv7Schema } from "../fields.ts"; + +export const apiKeyInfoResponse = z.object({ + id: z.string(), + memberId: z.string(), + lookupId: z.string(), + name: z.string(), + createdAt: z.string(), + expiresAt: z.string().nullable(), +}); +export type ApiKeyInfoResponse = z.infer; + +// apiKey.create — mint a key for an agent the caller owns +export const apiKeyCreateParams = z.object({ + agentId: uuidv7Schema, + name: nameSchema, + expiresAt: timestampSchema.optional().nullable(), +}); +export type ApiKeyCreateParams = z.infer; + +export const apiKeyCreateResult = z.object({ + id: z.string(), + /** The full api key string — returned once; only its hash is stored. */ + key: z.string(), +}); +export type ApiKeyCreateResult = z.infer; + +// apiKey.list — a member's keys (metadata only) +export const apiKeyListParams = z.object({ memberId: uuidv7Schema }); +export type ApiKeyListParams = z.infer; + +export const apiKeyListResult = z.object({ + apiKeys: z.array(apiKeyInfoResponse), +}); +export type ApiKeyListResult = z.infer; + +// apiKey.get +export const apiKeyGetParams = z.object({ id: uuidv7Schema }); +export type ApiKeyGetParams = z.infer; + +export const apiKeyGetResult = z.object({ + apiKey: apiKeyInfoResponse.nullable(), +}); +export type ApiKeyGetResult = z.infer; + +// apiKey.delete (revoke ≡ delete) +export const apiKeyDeleteParams = z.object({ id: uuidv7Schema }); +export type ApiKeyDeleteParams = z.infer; + +export const apiKeyDeleteResult = z.object({ deleted: z.boolean() }); +export type ApiKeyDeleteResult = z.infer; diff --git a/packages/protocol/user/index.ts b/packages/protocol/user/index.ts new file mode 100644 index 0000000..99d58da --- /dev/null +++ b/packages/protocol/user/index.ts @@ -0,0 +1,82 @@ +/** + * User RPC contract — session-only, user-scoped methods served on + * POST /api/v1/user/rpc. Covers the lifecycle of a user's global service + * accounts (agents) and their global api keys; space membership lives on the + * space endpoint. + */ +import type { z } from "zod"; + +import { + agentCreateParams, + agentCreateResult, + agentDeleteParams, + agentDeleteResult, + agentListParams, + agentListResult, + agentRenameParams, + agentRenameResult, +} from "./agent.ts"; +import { + apiKeyCreateParams, + apiKeyCreateResult, + apiKeyDeleteParams, + apiKeyDeleteResult, + apiKeyGetParams, + apiKeyGetResult, + apiKeyListParams, + apiKeyListResult, +} from "./api-key.ts"; +import { + spaceCreateParams, + spaceCreateResult, + spaceDeleteParams, + spaceDeleteResult, + spaceListParams, + spaceListResult, + spaceRenameParams, + spaceRenameResult, +} from "./space.ts"; +import { whoamiParams, whoamiResult } from "./whoami.ts"; + +export * from "./agent.ts"; +export * from "./api-key.ts"; +export * from "./space.ts"; +export * from "./whoami.ts"; + +function method( + params: TParams, + result: TResult, +) { + return { params, result }; +} + +/** + * User RPC method contract (identity + agent lifecycle + api keys + space + * discovery). + */ +export const userMethods = { + whoami: method(whoamiParams, whoamiResult), + + "agent.create": method(agentCreateParams, agentCreateResult), + "agent.list": method(agentListParams, agentListResult), + "agent.rename": method(agentRenameParams, agentRenameResult), + "agent.delete": method(agentDeleteParams, agentDeleteResult), + + "apiKey.create": method(apiKeyCreateParams, apiKeyCreateResult), + "apiKey.list": method(apiKeyListParams, apiKeyListResult), + "apiKey.get": method(apiKeyGetParams, apiKeyGetResult), + "apiKey.delete": method(apiKeyDeleteParams, apiKeyDeleteResult), + + "space.list": method(spaceListParams, spaceListResult), + "space.create": method(spaceCreateParams, spaceCreateResult), + "space.rename": method(spaceRenameParams, spaceRenameResult), + "space.delete": method(spaceDeleteParams, spaceDeleteResult), +} as const; + +export type UserMethodName = keyof typeof userMethods; +export type UserParams = z.infer< + (typeof userMethods)[M]["params"] +>; +export type UserResult = z.infer< + (typeof userMethods)[M]["result"] +>; diff --git a/packages/protocol/user/space.ts b/packages/protocol/user/space.ts new file mode 100644 index 0000000..1f48e5b --- /dev/null +++ b/packages/protocol/user/space.ts @@ -0,0 +1,60 @@ +/** + * Space method schemas (space.*) for the user RPC. + * + * Lets a logged-in user discover the spaces they belong to — used by the CLI to + * select the X-Me-Space the rest of the commands are scoped to. + */ +import { z } from "zod"; +import { nameSchema } from "../fields.ts"; + +export const memberSpaceResponse = z.object({ + id: z.string(), + slug: z.string(), + name: z.string(), + language: z.string(), + /** Whether the user is a (direct) admin of the space. */ + admin: z.boolean(), + createdAt: z.string(), + updatedAt: z.string().nullable(), +}); +export type MemberSpaceResponse = z.infer; + +// space.list — the caller's spaces +export const spaceListParams = z.object({}); +export type SpaceListParams = z.infer; + +export const spaceListResult = z.object({ + spaces: z.array(memberSpaceResponse), +}); +export type SpaceListResult = z.infer; + +// space.create — create a new space; the caller becomes admin + owner@root +export const spaceCreateParams = z.object({ name: nameSchema }); +export type SpaceCreateParams = z.infer; + +export const spaceCreateResult = z.object({ + id: z.string(), + slug: z.string(), +}); +export type SpaceCreateResult = z.infer; + +/** A space's slug (12-char routing key). */ +const spaceSlugSchema = z.string().regex(/^[a-z0-9]{12}$/); + +// space.rename — change a space's display name (admin only). The slug (and +// thus the me_ schema, api keys, and routing) is immutable. +export const spaceRenameParams = z.object({ + slug: spaceSlugSchema, + name: nameSchema, +}); +export type SpaceRenameParams = z.infer; + +export const spaceRenameResult = z.object({ renamed: z.boolean() }); +export type SpaceRenameResult = z.infer; + +// space.delete — delete a space + its data schema (admin only) +export const spaceDeleteParams = z.object({ slug: spaceSlugSchema }); +export type SpaceDeleteParams = z.infer; + +export const spaceDeleteResult = z.object({ deleted: z.boolean() }); +export type SpaceDeleteResult = z.infer; diff --git a/packages/protocol/user/whoami.ts b/packages/protocol/user/whoami.ts new file mode 100644 index 0000000..385bd20 --- /dev/null +++ b/packages/protocol/user/whoami.ts @@ -0,0 +1,19 @@ +/** + * whoami method schema for the user RPC. + * + * Returns the identity behind the session token — used by the CLI for `me login` + * confirmation and `me whoami`. Session-only (an api key never authenticates the + * user endpoint). + */ +import { z } from "zod"; + +// whoami — the authenticated user's identity +export const whoamiParams = z.object({}); +export type WhoamiParams = z.infer; + +export const whoamiResult = z.object({ + id: z.string(), + email: z.string(), + name: z.string(), +}); +export type WhoamiResult = z.infer; diff --git a/packages/protocol/version.ts b/packages/protocol/version.ts index 85b426e..2ddf659 100644 --- a/packages/protocol/version.ts +++ b/packages/protocol/version.ts @@ -13,17 +13,7 @@ */ import { z } from "zod"; -// ============================================================================= -// Headers -// ============================================================================= - -/** - * Header name the client uses to advertise its CLIENT_VERSION on every RPC. - * - * The server short-circuits requests with an incompatible client version - * before dispatching them to a handler. - */ -export const CLIENT_VERSION_HEADER = "X-Client-Version"; +// Header constants (CLIENT_VERSION_HEADER, SPACE_HEADER) live in ./headers.ts. // ============================================================================= // GET /api/v1/version diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 1f5a878..f2e01b0 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -9,9 +9,10 @@ EXPOSE 3000 # workspace declared in bun.lock is missing from the build context, so any # new workspace added under packages/* must be copied here too. COPY package.json bun.lock ./ -COPY packages/accounts/package.json packages/accounts/ +COPY packages/auth/package.json packages/auth/ COPY packages/cli/package.json packages/cli/ COPY packages/client/package.json packages/client/ +COPY packages/database/package.json packages/database/ COPY packages/docs-site/package.json packages/docs-site/ COPY packages/embedding/package.json packages/embedding/ COPY packages/engine/package.json packages/engine/ @@ -20,6 +21,7 @@ COPY packages/server/package.json packages/server/ COPY packages/web/package.json packages/web/ COPY packages/worker/package.json packages/worker/ COPY scripts/package.json scripts/ +COPY e2e/package.json e2e/ # --filter limits installation to the server and its transitive workspace deps, # avoiding pulling in docs-site's heavy runtime deps (next/react/tailwind/...). @@ -32,8 +34,8 @@ COPY version.ts ./ # Copy server source + all workspace dependencies COPY packages/server/ packages/server/ -COPY packages/accounts/ packages/accounts/ -COPY packages/client/ packages/client/ +COPY packages/auth/ packages/auth/ +COPY packages/database/ packages/database/ COPY packages/engine/ packages/engine/ COPY packages/embedding/ packages/embedding/ COPY packages/protocol/ packages/protocol/ diff --git a/packages/server/auth/device-flow.test.ts b/packages/server/auth/device-flow.test.ts deleted file mode 100644 index 05be81e..0000000 --- a/packages/server/auth/device-flow.test.ts +++ /dev/null @@ -1,369 +0,0 @@ -/** - * Tests for OAuth device flow state management. - * - * Uses an in-memory mock of the device auth database operations. - */ -import { beforeEach, describe, expect, test } from "bun:test"; -import type { - AccountsDB, - CreateDeviceAuthParams, - DeviceAuthorization, -} from "@memory.build/accounts"; -import { - authorizeDevice, - checkPollRateLimit, - cleanupDeviceState, - cleanupExpiredStates, - createDeviceAuthorization, - denyDevice, - getDeviceStateByDeviceCode, - getDeviceStateByOAuthState, - getDeviceStateByUserCode, -} from "./device-flow"; - -/** - * Create a mock AccountsDB with in-memory device auth storage. - * Only implements the device auth methods needed for testing. - */ -function createMockDb(): AccountsDB { - const store = new Map(); - const userCodeIndex = new Map(); - const oauthStateIndex = new Map(); - - return { - // Device auth operations - create: async (params: CreateDeviceAuthParams) => { - const auth: DeviceAuthorization = { - deviceCode: params.deviceCode, - userCode: params.userCode, - provider: params.provider, - oauthState: params.oauthState, - expiresAt: params.expiresAt, - lastPoll: null, - identityId: null, - denied: false, - createdAt: new Date(), - }; - store.set(params.deviceCode, auth); - userCodeIndex.set(params.userCode, params.deviceCode); - oauthStateIndex.set(params.oauthState, params.deviceCode); - return auth; - }, - - getByDeviceCode: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - return auth; - }, - - getByUserCode: async (userCode: string) => { - // Normalize: uppercase, remove hyphen, reconstruct XXXX-XXXX - const normalized = userCode.toUpperCase().replace(/-/g, ""); - const formatted = `${normalized.slice(0, 4)}-${normalized.slice(4)}`; - const deviceCode = userCodeIndex.get(formatted); - if (!deviceCode) return null; - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - return auth; - }, - - getByOAuthState: async (oauthState: string) => { - const deviceCode = oauthStateIndex.get(oauthState); - if (!deviceCode) return null; - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - return auth; - }, - - updateLastPoll: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt) { - return null; - } - const previousPoll = auth.lastPoll; - auth.lastPoll = new Date(); - if (!previousPoll) { - return null; - } - return Date.now() - previousPoll.getTime(); - }, - - authorize: async (deviceCode: string, identityId: string) => { - const auth = store.get(deviceCode); - if ( - !auth || - new Date() > auth.expiresAt || - auth.identityId !== null || - auth.denied - ) { - return false; - } - auth.identityId = identityId; - return true; - }, - - deny: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth || new Date() > auth.expiresAt || auth.identityId !== null) { - return false; - } - auth.denied = true; - return true; - }, - - delete: async (deviceCode: string) => { - const auth = store.get(deviceCode); - if (!auth) return false; - userCodeIndex.delete(auth.userCode); - oauthStateIndex.delete(auth.oauthState); - store.delete(deviceCode); - return true; - }, - - deleteExpired: async () => { - const now = new Date(); - let count = 0; - for (const [deviceCode, auth] of store) { - if (now > auth.expiresAt) { - userCodeIndex.delete(auth.userCode); - oauthStateIndex.delete(auth.oauthState); - store.delete(deviceCode); - count++; - } - } - return count; - }, - - // Expose store for testing (to manually expire entries) - _store: store, - } as unknown as AccountsDB; -} - -describe("device-flow", () => { - let db: AccountsDB; - - beforeEach(() => { - db = createMockDb(); - }); - - describe("createDeviceAuthorization", () => { - test("creates authorization with required fields", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - expect(auth.deviceCode).toBeDefined(); - expect(auth.deviceCode.length).toBeGreaterThan(20); - - expect(auth.userCode).toBeDefined(); - expect(auth.userCode).toMatch(/^[A-Z0-9]{4}-[A-Z0-9]{4}$/); - - expect(auth.oauthState).toBeDefined(); - expect(auth.oauthState.length).toBeGreaterThan(10); - - expect(auth.expiresIn).toBe(900); // 15 minutes - expect(auth.interval).toBe(5); - }); - - test("creates unique codes", async () => { - const auth1 = await createDeviceAuthorization(db, "google"); - const auth2 = await createDeviceAuthorization(db, "google"); - - expect(auth1.deviceCode).not.toBe(auth2.deviceCode); - expect(auth1.userCode).not.toBe(auth2.userCode); - expect(auth1.oauthState).not.toBe(auth2.oauthState); - }); - }); - - describe("getDeviceStateByUserCode", () => { - test("finds state by user code", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const state = await getDeviceStateByUserCode(db, auth.userCode); - - expect(state).not.toBeNull(); - expect(state?.userCode).toBe(auth.userCode); - expect(state?.provider).toBe("google"); - }); - - test("normalizes user code (case insensitive, with/without hyphen)", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - // Original format - expect(await getDeviceStateByUserCode(db, auth.userCode)).not.toBeNull(); - - // Lowercase - expect( - await getDeviceStateByUserCode(db, auth.userCode.toLowerCase()), - ).not.toBeNull(); - - // Without hyphen - expect( - await getDeviceStateByUserCode(db, auth.userCode.replace("-", "")), - ).not.toBeNull(); - - // Lowercase without hyphen - expect( - await getDeviceStateByUserCode( - db, - auth.userCode.toLowerCase().replace("-", ""), - ), - ).not.toBeNull(); - }); - - test("returns null for unknown code", async () => { - expect(await getDeviceStateByUserCode(db, "XXXX-XXXX")).toBeNull(); - }); - }); - - describe("getDeviceStateByOAuthState", () => { - test("finds state by OAuth state", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const state = await getDeviceStateByOAuthState(db, auth.oauthState); - - expect(state).not.toBeNull(); - expect(state?.oauthState).toBe(auth.oauthState); - }); - - test("returns null for unknown state", async () => { - expect(await getDeviceStateByOAuthState(db, "unknown-state")).toBeNull(); - }); - }); - - describe("getDeviceStateByDeviceCode", () => { - test("finds state by device code", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const state = await getDeviceStateByDeviceCode(db, auth.deviceCode); - - expect(state).not.toBeNull(); - expect(state?.deviceCode).toBe(auth.deviceCode); - }); - - test("returns null for unknown code", async () => { - expect(await getDeviceStateByDeviceCode(db, "unknown-code")).toBeNull(); - }); - }); - - describe("checkPollRateLimit", () => { - test("returns false on first poll", async () => { - const auth = await createDeviceAuthorization(db, "google"); - expect(await checkPollRateLimit(db, auth.deviceCode)).toBe(false); - }); - - test("returns true when polling too fast", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - // First poll - await checkPollRateLimit(db, auth.deviceCode); - - // Immediate second poll should be rate limited - expect(await checkPollRateLimit(db, auth.deviceCode)).toBe(true); - }); - - test("returns false for unknown device code", async () => { - expect(await checkPollRateLimit(db, "unknown-code")).toBe(false); - }); - }); - - describe("authorizeDevice", () => { - test("marks device as authorized", async () => { - const auth = await createDeviceAuthorization(db, "google"); - const identityId = "019d694f-79f6-7595-8faf-b70b01c11f98"; - - const result = await authorizeDevice(db, auth.deviceCode, identityId); - expect(result).toBe(true); - - const state = await getDeviceStateByDeviceCode(db, auth.deviceCode); - expect(state?.identityId).toBe(identityId); - }); - - test("returns false for unknown device code", async () => { - const result = await authorizeDevice( - db, - "unknown-code", - "019d694f-79f6-7595-8faf-b70b01c11f98", - ); - expect(result).toBe(false); - }); - }); - - describe("denyDevice", () => { - test("marks device as denied", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - const result = await denyDevice(db, auth.deviceCode); - expect(result).toBe(true); - - const state = await getDeviceStateByDeviceCode(db, auth.deviceCode); - expect(state?.denied).toBe(true); - }); - - test("returns false for unknown device code", async () => { - const result = await denyDevice(db, "unknown-code"); - expect(result).toBe(false); - }); - }); - - describe("cleanupDeviceState", () => { - test("removes device state", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - await cleanupDeviceState(db, auth.deviceCode); - - expect(await getDeviceStateByDeviceCode(db, auth.deviceCode)).toBeNull(); - expect(await getDeviceStateByUserCode(db, auth.userCode)).toBeNull(); - expect(await getDeviceStateByOAuthState(db, auth.oauthState)).toBeNull(); - }); - - test("handles unknown device code gracefully", async () => { - // Should not throw - await cleanupDeviceState(db, "unknown-code"); - }); - }); - - describe("cleanupExpiredStates", () => { - test("removes expired states", async () => { - // Create a state - const auth = await createDeviceAuthorization(db, "google"); - - // Manually expire it by modifying the store - const store = ( - db as unknown as { _store: Map } - )._store; - const state = store.get(auth.deviceCode); - if (state) { - state.expiresAt = new Date(Date.now() - 1000); // Expired 1 second ago - } - - // Cleanup - const cleaned = await cleanupExpiredStates(db); - expect(cleaned).toBeGreaterThanOrEqual(1); - - // State should be gone - expect(await getDeviceStateByDeviceCode(db, auth.deviceCode)).toBeNull(); - }); - }); - - describe("state expiration", () => { - test("expired state returns null on lookup", async () => { - const auth = await createDeviceAuthorization(db, "google"); - - // Manually expire it - const store = ( - db as unknown as { _store: Map } - )._store; - const state = store.get(auth.deviceCode); - if (state) { - state.expiresAt = new Date(Date.now() - 1000); - } - - // Lookup should return null - expect(await getDeviceStateByDeviceCode(db, auth.deviceCode)).toBeNull(); - expect(await getDeviceStateByUserCode(db, auth.userCode)).toBeNull(); - expect(await getDeviceStateByOAuthState(db, auth.oauthState)).toBeNull(); - }); - }); -}); diff --git a/packages/server/auth/device-flow.ts b/packages/server/auth/device-flow.ts deleted file mode 100644 index ea1ee4a..0000000 --- a/packages/server/auth/device-flow.ts +++ /dev/null @@ -1,182 +0,0 @@ -/** - * OAuth device flow state management. - * - * Manages device authorization state in PostgreSQL for multi-node support. - * State is persisted to database and cleaned up via cron. - */ - -import type { AccountsDB, DeviceAuthorization } from "@memory.build/accounts"; -import type { OAuthProvider } from "./types"; - -/** Device code expiration (15 minutes) */ -const DEVICE_CODE_EXPIRY_MS = 15 * 60 * 1000; - -/** Minimum polling interval (5 seconds) */ -const MIN_POLL_INTERVAL_MS = 5 * 1000; - -/** User code length (8 characters, alphanumeric, easy to type) */ -const USER_CODE_LENGTH = 8; - -/** Device code length (32 bytes, URL-safe base64) */ -const DEVICE_CODE_LENGTH = 32; - -/** OAuth state length (16 bytes, URL-safe base64) */ -const OAUTH_STATE_LENGTH = 16; - -/** - * Generate a cryptographically secure random string. - */ -function generateRandomString(length: number): string { - const bytes = new Uint8Array(length); - crypto.getRandomValues(bytes); - return Buffer.from(bytes).toString("base64url"); -} - -/** - * Generate a user-friendly code (uppercase alphanumeric, no ambiguous chars). - * Format: XXXX-XXXX (8 chars with hyphen separator for readability) - */ -function generateUserCode(): string { - // Exclude ambiguous characters: 0, O, 1, I, L - const chars = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"; - let code = ""; - const bytes = new Uint8Array(USER_CODE_LENGTH); - crypto.getRandomValues(bytes); - for (const byte of bytes) { - code += chars[byte % chars.length]; - } - // Insert hyphen for readability: XXXX-XXXX - return `${code.slice(0, 4)}-${code.slice(4)}`; -} - -/** - * Create a new device authorization. - * - * @returns Device code, user code, and expiry info - */ -export async function createDeviceAuthorization( - db: AccountsDB, - provider: OAuthProvider, -): Promise<{ - deviceCode: string; - userCode: string; - oauthState: string; - expiresIn: number; - interval: number; -}> { - const deviceCode = generateRandomString(DEVICE_CODE_LENGTH); - const userCode = generateUserCode(); - const oauthState = generateRandomString(OAUTH_STATE_LENGTH); - const expiresAt = new Date(Date.now() + DEVICE_CODE_EXPIRY_MS); - - await db.create({ - deviceCode, - userCode, - provider, - oauthState, - expiresAt, - }); - - return { - deviceCode, - userCode, - oauthState, - expiresIn: Math.floor(DEVICE_CODE_EXPIRY_MS / 1000), - interval: Math.floor(MIN_POLL_INTERVAL_MS / 1000), - }; -} - -/** - * Get device state by user code. - * Used when user enters code in browser. - */ -export async function getDeviceStateByUserCode( - db: AccountsDB, - userCode: string, -): Promise { - return db.getByUserCode(userCode); -} - -/** - * Get device state by OAuth state parameter. - * Used in OAuth callback to find the device being authorized. - */ -export async function getDeviceStateByOAuthState( - db: AccountsDB, - oauthState: string, -): Promise { - return db.getByOAuthState(oauthState); -} - -/** - * Get device state by device code. - * Used for polling. - */ -export async function getDeviceStateByDeviceCode( - db: AccountsDB, - deviceCode: string, -): Promise { - return db.getByDeviceCode(deviceCode); -} - -/** - * Check if polling is too fast (rate limiting). - * Also updates the last poll timestamp. - * - * @returns true if client should slow down - */ -export async function checkPollRateLimit( - db: AccountsDB, - deviceCode: string, -): Promise { - const elapsedMs = await db.updateLastPoll(deviceCode); - - // First poll or not found - if (elapsedMs === null) { - return false; - } - - // Too fast if less than minimum interval - return elapsedMs < MIN_POLL_INTERVAL_MS; -} - -/** - * Mark device as authorized with an identity. - * Called after successful OAuth callback. - */ -export async function authorizeDevice( - db: AccountsDB, - deviceCode: string, - identityId: string, -): Promise { - return db.authorize(deviceCode, identityId); -} - -/** - * Mark device as denied. - * Called if user denies access. - */ -export async function denyDevice( - db: AccountsDB, - deviceCode: string, -): Promise { - return db.deny(deviceCode); -} - -/** - * Clean up device state after completion or expiry. - */ -export async function cleanupDeviceState( - db: AccountsDB, - deviceCode: string, -): Promise { - await db.delete(deviceCode); -} - -/** - * Clean up all expired device states. - * Called by cron job. - */ -export async function cleanupExpiredStates(db: AccountsDB): Promise { - return db.deleteExpired(); -} diff --git a/packages/server/auth/index.ts b/packages/server/auth/index.ts index c7079e6..516b778 100644 --- a/packages/server/auth/index.ts +++ b/packages/server/auth/index.ts @@ -2,6 +2,5 @@ * OAuth authentication module exports. */ -export * from "./device-flow"; export * from "./providers"; export * from "./types"; diff --git a/packages/server/auth/providers/github.ts b/packages/server/auth/providers/github.ts index b4578ac..a790506 100644 --- a/packages/server/auth/providers/github.ts +++ b/packages/server/auth/providers/github.ts @@ -145,36 +145,32 @@ export async function fetchGitHubUserInfo( id: number; login: string; name: string | null; - email: string | null; }; - // If email is not public, fetch from emails endpoint - let email = userData.email; - if (!email) { - email = await fetchGitHubPrimaryEmail(accessToken); - } - - if (!email) { + // Always resolve the email via /user/emails so we get its `verified` flag + // (the public profile email field carries no verification signal). + const primary = await fetchGitHubPrimaryEmail(accessToken); + if (!primary) { throw new Error( - "GitHub account does not have a verified email address. Please add and verify an email in your GitHub settings.", + "GitHub account does not have an email address. Please add one in your GitHub settings.", ); } return { providerAccountId: String(userData.id), - email, + email: primary.email, + emailVerified: primary.verified, name: userData.name || userData.login, }; } /** - * Fetch user's primary verified email from GitHub. - * - * @param accessToken - OAuth access token + * Fetch the user's primary email from GitHub, with its verified flag. + * (Returns the primary email if present, else the first; null if none.) */ async function fetchGitHubPrimaryEmail( accessToken: string, -): Promise { +): Promise<{ email: string; verified: boolean } | null> { const response = await fetch("https://api.github.com/user/emails", { headers: { Authorization: `Bearer ${accessToken}`, @@ -184,7 +180,6 @@ async function fetchGitHubPrimaryEmail( }); if (!response.ok) { - // If we can't fetch emails, return null and let caller handle it return null; } @@ -194,13 +189,6 @@ async function fetchGitHubPrimaryEmail( verified: boolean; }>; - // Find primary verified email - const primary = emails.find((e) => e.primary && e.verified); - if (primary) { - return primary.email; - } - - // Fall back to any verified email - const verified = emails.find((e) => e.verified); - return verified?.email ?? null; + const chosen = emails.find((e) => e.primary) ?? emails[0]; + return chosen ? { email: chosen.email, verified: chosen.verified } : null; } diff --git a/packages/server/auth/providers/google.ts b/packages/server/auth/providers/google.ts index 1306cd1..c130cef 100644 --- a/packages/server/auth/providers/google.ts +++ b/packages/server/auth/providers/google.ts @@ -40,14 +40,14 @@ export function getGoogleConfig(): OAuthProviderConfig { */ export function buildGoogleAuthUrl(state: string, redirectUri: string): string { const config = getGoogleConfig(); + // Login-only: we use the access token once (to read the profile) and never + // store it, so we don't request offline access or force a consent prompt. const params = new URLSearchParams({ client_id: config.clientId, redirect_uri: redirectUri, response_type: "code", scope: config.scopes.join(" "), state, - access_type: "offline", - prompt: "consent", }); return `${config.authorizationUrl}?${params.toString()}`; @@ -137,6 +137,7 @@ export async function fetchGoogleUserInfo( return { providerAccountId: data.id, email: data.email, + emailVerified: Boolean(data.verified_email), name: data.name || data.email.split("@")[0] || "User", }; } diff --git a/packages/server/auth/types.ts b/packages/server/auth/types.ts index b67b189..2b8bb3f 100644 --- a/packages/server/auth/types.ts +++ b/packages/server/auth/types.ts @@ -114,6 +114,8 @@ export interface OAuthUserInfo { providerAccountId: string; /** User's email */ email: string; + /** Whether the provider has verified the user controls this email. */ + emailVerified: boolean; /** User's display name */ name: string; } diff --git a/packages/server/context.ts b/packages/server/context.ts index e2ceb3f..1223165 100644 --- a/packages/server/context.ts +++ b/packages/server/context.ts @@ -1,18 +1,23 @@ -import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { SQL } from "bun"; +import type { CoreStore } from "@memory.build/engine/core"; +import type { Sql } from "postgres"; /** * Server-wide context containing database connections. * Passed to createRouter() at startup. */ export interface ServerContext { - /** Accounts database operations */ - accountsDb: AccountsDB; - /** Accounts database pool (for health checks) */ - accountsSql: SQL; - /** Engine database pool (EngineDB created per-request based on slug) */ - engineSql: SQL; + /** Pool (postgres.js): auth + core + per-space schemas, one DB */ + db: Sql; + /** Auth store (auth schema): me/session/identity/device + OAuth accounts */ + auth: AuthStore; + /** Core control-plane store (core schema): spaces/principals/grants/api-keys */ + core: CoreStore; + /** The auth schema name */ + authSchema: string; + /** The core control-plane schema name */ + coreSchema: string; /** Embedding config for semantic search */ embeddingConfig: EmbeddingConfig; /** Base URL for API callbacks (e.g., "https://memory.build") */ diff --git a/packages/server/handlers/auth.integration.test.ts b/packages/server/handlers/auth.integration.test.ts new file mode 100644 index 0000000..73eb755 --- /dev/null +++ b/packages/server/handlers/auth.integration.test.ts @@ -0,0 +1,108 @@ +// Integration test for the OAuth callback's invitation-redemption hook (INV-3): +// redeemInvitationsForVerifiedLogin joins a user to every space they were invited +// to, and swallows failures so a redemption hiccup never breaks the sign-in. The +// redeem SQL/store itself is covered in the engine + migrate suites; here we cover +// the login-side glue against a real core schema. (Importing ./auth pulls in the +// OAuth handler but triggers no provider network calls — those only fire inside +// oauthCallbackHandler, which this test does not invoke.) +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/server/handlers/auth.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { migrateCore } from "@memory.build/database"; +import { ACCESS, type CoreStore, coreStore } from "@memory.build/engine/core"; +import postgres, { type Sql } from "postgres"; +import { redeemInvitationsForVerifiedLogin } from "./auth"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; +const randomSlug = () => rand(12); + +let sql: Sql; +let coreSchema: string; +let core: CoreStore; + +async function v7(): Promise { + const [row] = await sql`select uuidv7() as id`; + return row?.id as string; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + await migrateCore(sql, { schema: coreSchema }); + core = coreStore(sql, coreSchema); +}); + +afterAll(async () => { + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +test("redeems pending invitations for a verified-login email", async () => { + const spaceId = await core.createSpace(randomSlug(), "Invited Space"); + const inviterId = await v7(); + await core.createUser(inviterId, `inviter_${rand(8)}@example.com`); + + const email = `invitee_${rand(8)}@example.com`; + await core.createSpaceInvitation(spaceId, email, { + admin: true, + shareAccess: ACCESS.write, + invitedBy: inviterId, + }); + + // the user already exists in core (the OAuth callback resolves/provisions the + // user before reaching the redemption hook) + const userId = await v7(); + await core.createUser(userId, email); + + const joined = await redeemInvitationsForVerifiedLogin(core, userId, email); + expect(joined).toBe(1); + + // joined the space as admin, with owner@home + write@share + const principals = await core.listSpacePrincipals(spaceId); + expect(principals.find((p) => p.id === userId)?.admin).toBe(true); + const ta = await core.buildTreeAccess(userId, spaceId); + expect(ta).toContainEqual({ + tree_path: `home.${userId.replace(/-/g, "")}`, + access: ACCESS.owner, + }); + expect(ta).toContainEqual({ tree_path: "share", access: ACCESS.write }); + + // invitation consumed; a second login is a no-op + expect(await core.listSpaceInvitations(spaceId)).toHaveLength(0); + expect(await redeemInvitationsForVerifiedLogin(core, userId, email)).toBe(0); +}); + +test("best-effort: a redemption failure is swallowed (does not throw)", async () => { + const spaceId = await core.createSpace(randomSlug(), "Space"); + const inviterId = await v7(); + await core.createUser(inviterId, `inviter_${rand(8)}@example.com`); + const email = `ghost_${rand(8)}@example.com`; + await core.createSpaceInvitation(spaceId, email, { + admin: false, + shareAccess: ACCESS.read, + invitedBy: inviterId, + }); + + // a user id that is not a core principal → add_principal_to_space FK fails + // inside redeem; the helper must swallow the error and report zero joins. + const orphanUserId = await v7(); + const joined = await redeemInvitationsForVerifiedLogin( + core, + orphanUserId, + email, + ); + expect(joined).toBe(0); + + // the failed redemption rolled back atomically: the invite is still pending + expect(await core.listSpaceInvitations(spaceId)).toHaveLength(1); +}); diff --git a/packages/server/handlers/auth.ts b/packages/server/handlers/auth.ts index 791a2d3..c047cd5 100644 --- a/packages/server/handlers/auth.ts +++ b/packages/server/handlers/auth.ts @@ -1,55 +1,45 @@ /** - * OAuth device flow HTTP handlers. + * OAuth device flow HTTP handlers (new model: authStore + provisionUser). * * Endpoints: - * - POST /api/v1/auth/device/code - CLI initiates device flow - * - POST /api/v1/auth/device/token - CLI polls for token - * - GET /api/v1/auth/device/verify - User enters code (HTML form) - * - POST /api/v1/auth/device/verify - User submits code - * - GET /api/v1/auth/callback/:provider - OAuth callback + * - POST /api/v1/auth/device/code - CLI initiates device flow + * - POST /api/v1/auth/device/token - CLI polls for token + * - GET /api/v1/auth/device/verify - User enters code (HTML form) + * - POST /api/v1/auth/device/verify - User submits code -> OAuth redirect + * - GET /api/v1/auth/callback/:provider - OAuth callback -> consent page + * - POST /api/v1/auth/device/approve - User approves/denies (consent) */ -import type { AccountsDB, Identity } from "@memory.build/accounts"; -import { type EngineConfig, provisionEngine } from "@memory.build/engine"; -import { setLocalEngineTimeouts } from "@memory.build/engine/ops/_tx"; +import type { AuthStore, OAuthProvider } from "@memory.build/auth"; +import { type CoreStore, coreStore } from "@memory.build/engine/core"; import { info, reportError } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import { - authorizeDevice, - checkPollRateLimit, - cleanupDeviceState, - createDeviceAuthorization, - denyDevice, - getDeviceStateByDeviceCode, - getDeviceStateByOAuthState, - getDeviceStateByUserCode, -} from "../auth/device-flow"; +import type { Sql } from "postgres"; import { buildAuthUrl, exchangeCode, fetchUserInfo } from "../auth/providers"; -import type { OAuthProvider } from "../auth/types"; -import { embeddingConstants } from "../config"; +import { provisionUser } from "../provision"; import type { RouteParams } from "../router"; import { error, html, json } from "../util/response"; +/** Min CLI poll interval (seconds) — matches poll_device's default. */ +const POLL_INTERVAL_SECONDS = 5; + /** - * Context needed for auth handlers. + * Context for the auth handlers. `auth` is bound to the auth schema; `db` + + * schema names are for provisionUser's atomic cross-schema transaction. */ export interface AuthHandlerContext { - /** AccountsDB instance */ - db: AccountsDB; - /** Base URL for callbacks (e.g., "https://memory.build") */ + auth: AuthStore; + db: Sql; + authSchema: string; + coreSchema: string; baseUrl: string; - /** Engine database pool (for provisioning default engine on signup) */ - engineSql: SQL; - /** Application version for migration tracking */ - serverVersion: string; +} + +function isProvider(p: string | undefined): p is OAuthProvider { + return p === "google" || p === "github"; } /** - * POST /api/v1/auth/device/code - * - * CLI initiates device flow. - * Request: { provider: "google" } - * Response: { deviceCode, userCode, verificationUri, expiresIn, interval } + * POST /api/v1/auth/device/code — CLI initiates the device flow. */ export async function deviceCodeHandler( request: Request, @@ -58,16 +48,13 @@ export async function deviceCodeHandler( if (request.method !== "POST") { return error("Method not allowed", 405, "METHOD_NOT_ALLOWED"); } - let body: { provider?: string }; try { body = (await request.json()) as { provider?: string }; } catch { return error("Invalid JSON body", 400, "INVALID_REQUEST"); } - - const provider = body.provider; - if (provider !== "google" && provider !== "github") { + if (!isProvider(body.provider)) { return error( "Invalid provider. Must be 'google' or 'github'", 400, @@ -75,24 +62,18 @@ export async function deviceCodeHandler( ); } - const auth = await createDeviceAuthorization(ctx.db, provider); - + const device = await ctx.auth.createDeviceAuth(body.provider); return json({ - deviceCode: auth.deviceCode, - userCode: auth.userCode, + deviceCode: device.deviceCode, + userCode: device.userCode, verificationUri: `${ctx.baseUrl}/api/v1/auth/device/verify`, - expiresIn: auth.expiresIn, - interval: auth.interval, + expiresIn: device.expiresIn, + interval: POLL_INTERVAL_SECONDS, }); } /** - * POST /api/v1/auth/device/token - * - * CLI polls for token. - * Request: { deviceCode: "..." } - * Response (pending): { error: "authorization_pending" } - * Response (success): { sessionToken, identity: { id, email, name } } + * POST /api/v1/auth/device/token — CLI polls for the session token. */ export async function deviceTokenHandler( request: Request, @@ -101,156 +82,82 @@ export async function deviceTokenHandler( if (request.method !== "POST") { return error("Method not allowed", 405, "METHOD_NOT_ALLOWED"); } - let body: { deviceCode?: string }; try { body = (await request.json()) as { deviceCode?: string }; } catch { return error("Invalid JSON body", 400, "INVALID_REQUEST"); } - const deviceCode = body.deviceCode; if (!deviceCode || typeof deviceCode !== "string") { return error("Missing deviceCode", 400, "INVALID_REQUEST"); } - // Check rate limit first (also updates last_poll timestamp) - const tooFast = await checkPollRateLimit(ctx.db, deviceCode); - if (tooFast) { - return json({ error: "slow_down" }, 400); - } - - // Check if device code exists - const state = await getDeviceStateByDeviceCode(ctx.db, deviceCode); - if (!state) { - return json({ error: "expired_token" }, 400); - } - - // Check if denied - if (state.denied) { - await cleanupDeviceState(ctx.db, deviceCode); - return json({ error: "access_denied" }, 400); - } - - // Check if authorized - if (!state.identityId) { - return json({ error: "authorization_pending" }, 400); - } - - // Get identity and create session - const identity = await ctx.db.getIdentity(state.identityId); - if (!identity) { - return error("Identity not found", 500, "INTERNAL_ERROR"); + const poll = await ctx.auth.pollDevice(deviceCode); + switch (poll.status) { + case "slow_down": + return json({ error: "slow_down" }, 400); + case "expired": + return json({ error: "expired_token" }, 400); + case "denied": + await ctx.auth.deleteDevice(deviceCode); + return json({ error: "access_denied" }, 400); + case "pending": + return json({ error: "authorization_pending" }, 400); + case "approved": { + if (!poll.userId) { + return error("Approved device has no user", 500, "INTERNAL_ERROR"); + } + const user = await ctx.auth.getUser(poll.userId); + if (!user) { + return error("User not found", 500, "INTERNAL_ERROR"); + } + const session = await ctx.auth.createSession(poll.userId); + await ctx.auth.deleteDevice(deviceCode); + return json({ + sessionToken: session.token, + identity: { id: user.id, email: user.email, name: user.name }, + }); + } } - - // Create session - const sessionResult = await ctx.db.createSession({ - identityId: identity.id, - }); - - // Cleanup device state - await cleanupDeviceState(ctx.db, deviceCode); - - return json({ - sessionToken: sessionResult.rawToken, - identity: { - id: identity.id, - email: identity.email, - name: identity.name, - }, - }); } /** - * GET /api/v1/auth/device/verify - * - * User visits this page to enter their code. - * Returns an HTML form. + * GET /api/v1/auth/device/verify — the page where the user enters their code. */ -export async function deviceVerifyGetHandler( +export function deviceVerifyGetHandler( _request: Request, _ctx: AuthHandlerContext, -): Promise { +): Response { const htmlContent = ` Sign in to Memory Engine - + ${PAGE_STYLE}

Sign in to Memory Engine

Enter the code shown in your CLI

-
`; - - // The form POSTs to 'self', but the response is a 302 redirect to the - // OAuth provider. Browsers enforce form-action on the full redirect chain, - // so we must whitelist the OAuth provider origins here. + // The form POSTs to self, then we 302 to the OAuth provider; browsers enforce + // form-action across the redirect chain, so whitelist the provider origins. const csp = "default-src 'none'; style-src 'unsafe-inline'; form-action 'self' https://accounts.google.com https://github.com"; - return html(htmlContent, 200, csp); } /** - * POST /api/v1/auth/device/verify - * - * User submits code. If valid, redirect to OAuth provider. + * POST /api/v1/auth/device/verify — user submitted a code; redirect to OAuth. */ export async function deviceVerifyPostHandler( request: Request, @@ -258,233 +165,286 @@ export async function deviceVerifyPostHandler( ): Promise { const formData = await request.formData(); const userCode = formData.get("user_code"); - if (!userCode || typeof userCode !== "string") { return html(errorPage("Please enter a code"), 400); } - // Find device state - const state = await getDeviceStateByUserCode(ctx.db, userCode); - if (!state) { + const device = await ctx.auth.getDeviceByUserCode(userCode); + if (!device) { return html(errorPage("Invalid or expired code. Please try again."), 400); } - // Redirect to OAuth provider - const redirectUri = `${ctx.baseUrl}/api/v1/auth/callback/${state.provider}`; - const authUrl = buildAuthUrl(state.provider, state.oauthState, redirectUri); + const redirectUri = `${ctx.baseUrl}/api/v1/auth/callback/${device.provider}`; + const authUrl = buildAuthUrl(device.provider, device.oauthState, redirectUri); + return new Response(null, { status: 302, headers: { Location: authUrl } }); +} - return new Response(null, { - status: 302, - headers: { Location: authUrl }, - }); +/** + * Redeem pending space invitations for a just-verified login email: join the + * user to each invited space (owner@home + the per-invite share level). + * Idempotent, and best-effort — a redemption failure is logged and swallowed so + * it never fails the sign-in (the next login retries). Returns the number of + * spaces joined. The caller MUST have verified the user owns this email first + * (invitations are email-keyed; redeeming for an unverified email would let a + * caller claim invites sent to an address they don't control). + */ +export async function redeemInvitationsForVerifiedLogin( + core: CoreStore, + userId: string, + email: string, +): Promise { + try { + const joined = await core.redeemSpaceInvitations(userId, email); + if (joined.length > 0) { + info("Redeemed space invitations", { email, spaces: joined.length }); + } + return joined.length; + } catch (err) { + reportError( + "Invitation redemption failed (continuing sign-in)", + err as Error, + { email }, + ); + return 0; + } } /** - * GET /api/v1/auth/callback/:provider + * GET /api/v1/auth/callback/:provider — OAuth callback. * - * OAuth callback. Exchange code for tokens, create/link identity. + * Resolves the user (account → verified email → provision), redeems pending + * space invitations for the verified email, binds them to the device (status + * stays 'pending'), and shows the consent page. Authorization only happens when + * the human approves (POST /device/approve). */ export async function oauthCallbackHandler( request: Request, params: RouteParams, ctx: AuthHandlerContext, ): Promise { - const provider = params.provider as OAuthProvider; - if (provider !== "google" && provider !== "github") { + if (!isProvider(params.provider)) { return html(errorPage("Unknown OAuth provider"), 400); } + const provider = params.provider; const url = new URL(request.url); const code = url.searchParams.get("code"); const oauthState = url.searchParams.get("state"); const errorParam = url.searchParams.get("error"); - // Check for OAuth error if (errorParam) { const errorDesc = url.searchParams.get("error_description") || errorParam; - // If we have state, mark device as denied if (oauthState) { - const deviceState = await getDeviceStateByOAuthState(ctx.db, oauthState); - if (deviceState) { - await denyDevice(ctx.db, deviceState.deviceCode); - } + const device = await ctx.auth.getDeviceByOAuthState(oauthState); + if (device) await ctx.auth.denyDevice(device.deviceCode); } return html(errorPage(`OAuth error: ${errorDesc}`), 400); } - if (!code || !oauthState) { return html(errorPage("Missing code or state parameter"), 400); } - // Find device state by OAuth state - const deviceState = await getDeviceStateByOAuthState(ctx.db, oauthState); - if (!deviceState) { + const device = await ctx.auth.getDeviceByOAuthState(oauthState); + if (!device) { return html( - errorPage( - "Invalid or expired session. Please restart the sign-in process.", - ), + errorPage("Invalid or expired session. Please restart the sign-in."), 400, ); } const redirectUri = `${ctx.baseUrl}/api/v1/auth/callback/${provider}`; - try { - // Exchange code for tokens const tokens = await exchangeCode(provider, code, redirectUri); - - // Fetch user info const userInfo = await fetchUserInfo(provider, tokens.accessToken); - // Find or create identity - let identity = await ctx.db.getIdentityByEmail(userInfo.email); - if (!identity) { - // Create new identity and provision personal account - identity = await ctx.db.createIdentity({ - email: userInfo.email, - name: userInfo.name, - }); - await provisionPersonalAccount(ctx, identity); + // Reject unverified emails — the gate that prevents account-takeover via a + // provider asserting someone else's address. + if (!userInfo.emailVerified) { + await ctx.auth.denyDevice(device.deviceCode); + return html( + errorPage( + `Your ${provider} email (${userInfo.email}) is not verified. Verify it with ${provider} and try again.`, + ), + 400, + ); } - // Link OAuth account (upserts if exists) - await ctx.db.linkOAuthAccount({ - identityId: identity.id, + // Resolve the user: existing account → existing verified email → new user. + let userId: string; + const account = await ctx.auth.getAccountByProvider( provider, - providerAccountId: userInfo.providerAccountId, - email: userInfo.email, - accessToken: tokens.accessToken, - refreshToken: tokens.refreshToken ?? undefined, - tokenExpiresAt: tokens.expiresIn - ? new Date(Date.now() + tokens.expiresIn * 1000) - : undefined, - }); - - // Mark device as authorized - await authorizeDevice(ctx.db, deviceState.deviceCode, identity.id); - - // Show success page - return html(successPage()); + userInfo.providerAccountId, + ); + if (account) { + userId = account.userId; + } else { + const byEmail = await ctx.auth.getUserByEmail(userInfo.email); + if (byEmail) { + // verified (gated above) → safe to link this provider to the user + userId = byEmail.id; + await ctx.auth.upsertAccount( + userId, + provider, + userInfo.providerAccountId, + ); + } else { + const result = await provisionUser( + ctx.db, + { auth: ctx.authSchema, core: ctx.coreSchema }, + { + email: userInfo.email, + name: userInfo.name, + provider, + accountId: userInfo.providerAccountId, + emailVerified: true, + }, + ); + userId = result.userId; + info("Provisioned new user", { email: userInfo.email }); + } + } + + // The email is verified (gated above) → proven owned, so redeem any pending + // space invitations sent to it (the user joins each invited space). + await redeemInvitationsForVerifiedLogin( + coreStore(ctx.db, ctx.coreSchema), + userId, + userInfo.email, + ); + + // Bind the user; the device stays 'pending' until the human consents. + await ctx.auth.bindDeviceUser(device.deviceCode, userId); + return html( + consentPage(userInfo.email, device.userCode, provider, oauthState), + 200, + CONSENT_CSP, + ); } catch (err) { reportError("OAuth callback error", err as Error); return html(errorPage("Authentication failed. Please try again."), 500); } } -// ============================================================================= -// Personal Account Provisioning -// ============================================================================= - /** - * Provision a personal account for a newly created identity. - * - * Creates a personal org (with the identity as owner) and a default - * memory engine within that org. This runs during first login so the - * user has an immediate, working environment. + * POST /api/v1/auth/device/approve — the human approves (or denies) the device. + * The browser only ever carries the oauth_state, never the device_code. */ -async function provisionPersonalAccount( +export async function deviceApproveHandler( + request: Request, ctx: AuthHandlerContext, - identity: Identity, -): Promise { - const { db, engineSql, serverVersion } = ctx; - - const org = await db.withTransaction(async (txDb) => { - // Create personal org - const newOrg = await txDb.createOrg({ name: "Personal" }); - - // Add identity as owner - await txDb.addMember(newOrg.id, identity.id, "owner"); - - // Create default engine record - const engine = await txDb.createEngine({ - orgId: newOrg.id, - name: "default", - }); - - // Provision the engine schema in the engine database - const engineConfig: EngineConfig = { - embedding_dimensions: embeddingConstants.dimensions, - bm25_text_config: engine.language, - }; - - try { - await provisionEngine( - engineSql, - engine.slug, - engineConfig, - serverVersion, - engine.shardId, - ); - } catch (err) { - // Clean up partially-created schema - const schema = `me_${engine.slug}`; - try { - await engineSql.begin(async (tx) => { - await tx.unsafe(`set local pgdog.shard to ${engine.shardId}`); - await setLocalEngineTimeouts(tx); - await tx.unsafe(`drop schema if exists ${schema} cascade`); - }); - } catch { - // Log but don't mask original error - } - throw err; - } +): Promise { + const formData = await request.formData(); + const oauthState = formData.get("oauth_state"); + const decision = formData.get("decision"); + if (typeof oauthState !== "string") { + return html(errorPage("Missing state."), 400); + } - return newOrg; - }); + const device = await ctx.auth.getDeviceByOAuthState(oauthState); + if (!device) { + return html( + errorPage("Invalid or expired session. Please restart the sign-in."), + 400, + ); + } - info("Provisioned personal account", { - email: identity.email, - orgId: org.id, - }); + if (decision === "deny") { + await ctx.auth.denyDevice(device.deviceCode); + return html(deniedPage()); + } + + const ok = await ctx.auth.approveDevice(device.deviceCode); + if (!ok) { + return html( + errorPage("This request was already handled or has expired."), + 400, + ); + } + return html(successPage()); } -/** - * Generate error HTML page. - */ -function errorPage(message: string): string { - return ` - - - - - Error - Memory Engine - + .primary { background: #0066cc; } .primary:hover { background: #0052a3; } + .secondary { background: #888; } .secondary:hover { background: #666; } + .checkmark { font-size: 64px; margin-bottom: 16px; } + a { display: inline-block; padding: 12px 24px; background: #0066cc; color: white; text-decoration: none; border-radius: 8px; } + `; + +/** Consent page: shown after OAuth; the user explicitly approves the device. */ +function consentPage( + email: string, + userCode: string, + provider: string, + oauthState: string, +): string { + return ` + + + + + Approve sign-in - Memory Engine + ${PAGE_STYLE}
-

Error

+

Approve this sign-in?

+

A device is requesting access to your memory as + ${escapeHtml(email)} (via ${escapeHtml(provider)}).

+ Only approve if you started this and the code below matches the one your + device shows: ${escapeHtml(userCode)}

+
+ + + +
+
+ +`; +} + +const CONSENT_CSP = + "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'"; + +function errorPage(message: string): string { + return ` + + + + + Error - Memory Engine + ${PAGE_STYLE} + + +
+

Error

${escapeHtml(message)}

Try Again
@@ -492,9 +452,6 @@ function errorPage(message: string): string { `; } -/** - * Generate success HTML page. - */ function successPage(): string { return ` @@ -502,37 +459,11 @@ function successPage(): string { Success - Memory Engine - + ${PAGE_STYLE}
-
✓
+
✓

You're signed in!

You can close this window and return to your CLI.

@@ -540,9 +471,24 @@ function successPage(): string { `; } -/** - * Escape HTML entities. - */ +function deniedPage(): string { + return ` + + + + + Denied - Memory Engine + ${PAGE_STYLE} + + +
+

Request denied

+

No access was granted. You can close this window.

+
+ +`; +} + function escapeHtml(text: string): string { return text .replace(/&/g, "&") diff --git a/packages/server/handlers/health.ts b/packages/server/handlers/health.ts index 1e3c061..c2d3bdf 100644 --- a/packages/server/handlers/health.ts +++ b/packages/server/handlers/health.ts @@ -1,5 +1,5 @@ import { info } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; +import type { Sql } from "postgres"; import { json, text } from "../util/response"; /** @@ -14,32 +14,23 @@ export function healthHandler(_request: Request): Response { /** * Readiness check handler. - * Verifies both database pools are alive via SELECT 1. - * Returns 200 if both succeed, 503 if either fails. + * Verifies the database pool is alive via SELECT 1. + * Returns 200 on success, 503 on failure. */ export function readyHandler( - accountsSql: SQL, - engineSql: SQL, + db: Sql, ): (_request: Request) => Promise { return async (_request: Request) => { const checks: Record = {}; - const [accountsResult, engineResult] = await Promise.allSettled([ - accountsSql`SELECT 1`, - engineSql`SELECT 1`, - ]); + const [dbResult] = await Promise.allSettled([db`SELECT 1`]); - checks.accounts_db = - accountsResult.status === "fulfilled" + checks.db = + dbResult.status === "fulfilled" ? "ok" - : `error: ${accountsResult.reason instanceof Error ? accountsResult.reason.message : String(accountsResult.reason)}`; + : `error: ${dbResult.reason instanceof Error ? dbResult.reason.message : String(dbResult.reason)}`; - checks.engine_db = - engineResult.status === "fulfilled" - ? "ok" - : `error: ${engineResult.reason instanceof Error ? engineResult.reason.message : String(engineResult.reason)}`; - - const allOk = checks.accounts_db === "ok" && checks.engine_db === "ok"; + const allOk = checks.db === "ok"; return json( { diff --git a/packages/server/index.ts b/packages/server/index.ts index c3f07d7..86a5f7c 100644 --- a/packages/server/index.ts +++ b/packages/server/index.ts @@ -1,25 +1,12 @@ // packages/server/index.ts -import { createAccountsDB } from "@memory.build/accounts"; -import { migrate as migrateAccounts } from "@memory.build/accounts/migrate/runner"; -import type { EmbeddingConfig } from "@memory.build/embedding"; -import { - discoverEngineSchemas, - slugToSchema, -} from "@memory.build/engine/migrate"; -import { bootstrap as bootstrapEngine } from "@memory.build/engine/migrate/bootstrap"; -import { migrateAll as migrateEngines } from "@memory.build/engine/migrate/runner"; -import { - DEFAULT_ENGINE_TIMEOUTS, - type EngineTimeouts, -} from "@memory.build/engine/ops/_tx"; -import { WorkerPool } from "@memory.build/worker"; -import { configure, info, reportError, span } from "@pydantic/logfire-node"; -import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; -import { embeddingConstants } from "./config"; -import type { ServerContext } from "./context"; -import { checkSizeLimit } from "./middleware"; -import { createRouter } from "./router"; -import { internalError } from "./util/response"; +// +// Production entrypoint: configure telemetry, boot the server via startServer(), +// then install the process-level signal/error handlers that index.ts owns (and +// startServer deliberately does not). The actual bootstrap lives in start.ts so +// it can be driven in-process by tests. +import { configure, info, reportError } from "@pydantic/logfire-node"; +import { SERVER_VERSION } from "../../version"; +import { startServer } from "./start"; // Resolve git revision for Logfire code source linking. // Locally, use the actual commit hash for precise source-linking. @@ -56,511 +43,10 @@ configure({ }, }); -// ============================================================================= -// Environment Variables -// ============================================================================= -// -// Required: -// ACCOUNTS_DATABASE_URL - PostgreSQL connection string for accounts database -// (stores engines, API keys, users) -// ACCOUNTS_MASTER_KEY - 32-byte hex string for encrypting API keys at rest -// Generate with: openssl rand -hex 32 -// ENGINE_DATABASE_URL - PostgreSQL connection string for engine databases -// (stores memories, each engine in its own schema) -// API_BASE_URL - Public URL for OAuth callbacks -// (e.g., "https://memory.build") -// -// Optional: -// PORT - HTTP server port (default: 3000) -// ACCOUNTS_SCHEMA - Schema name in accounts database (default: "accounts") -// -// Connection Pool - Accounts Database: -// ACCOUNTS_POOL_MAX - Max connections (default: 10) -// ACCOUNTS_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: 300) -// ACCOUNTS_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: 0) -// ACCOUNTS_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: 30) -// ACCOUNTS_STATEMENT_TIMEOUT - Per-accounts-query timeout (default: 25s) -// ACCOUNTS_LOCK_TIMEOUT - Per-accounts-lock wait timeout (default: 5s) -// ACCOUNTS_TRANSACTION_TIMEOUT - Per-accounts-transaction timeout (default: 30s) -// ACCOUNTS_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Idle-in-transaction timeout (default: 30s) -// -// Connection Pool - Engine Database: -// ENGINE_POOL_MAX - Max connections (default: 20) -// ENGINE_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: 300) -// ENGINE_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: 0) -// ENGINE_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: 30) -// ENGINE_STATEMENT_TIMEOUT - Per-engine-query timeout (default: 25s) -// ENGINE_LOCK_TIMEOUT - Per-engine-lock wait timeout (default: 5s) -// ENGINE_TRANSACTION_TIMEOUT - Per-engine-transaction timeout (default: 30s) -// ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Idle-in-transaction timeout (default: 30s) -// -// Embedding Worker Engine Database: -// WORKER_ENGINE_DATABASE_URL - PostgreSQL connection string for worker engine traffic (default: ENGINE_DATABASE_URL) -// WORKER_ENGINE_POOL_MAX - Max worker engine connections (default: WORKER_COUNT) -// WORKER_ENGINE_POOL_IDLE_REAP_SECONDS - Close idle pooled connections after N seconds (default: ENGINE_POOL_IDLE_REAP_SECONDS) -// WORKER_ENGINE_POOL_MAX_LIFETIME - Max lifetime in seconds, 0=forever (default: ENGINE_POOL_MAX_LIFETIME) -// WORKER_ENGINE_POOL_CONNECTION_TIMEOUT - Connection timeout in seconds (default: ENGINE_POOL_CONNECTION_TIMEOUT) -// WORKER_ENGINE_STATEMENT_TIMEOUT - Worker engine query timeout (default: ENGINE_STATEMENT_TIMEOUT) -// WORKER_ENGINE_LOCK_TIMEOUT - Worker engine lock wait timeout (default: ENGINE_LOCK_TIMEOUT) -// WORKER_ENGINE_TRANSACTION_TIMEOUT - Worker engine transaction timeout (default: ENGINE_TRANSACTION_TIMEOUT) -// WORKER_ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT - Worker engine idle-in-transaction timeout (default: ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT) -// -// Cleanup: -// DEVICE_FLOW_CLEANUP_CRON - Cron schedule for cleaning up expired device auths -// (default: "*/15 * * * *" = every 15 minutes, UTC) -// -// Embedding Worker: -// WORKER_COUNT - Number of concurrent embedding workers (default: 2) -// WORKER_BATCH_SIZE - Queue entries to claim per batch (default: 10) -// WORKER_LOCK_DURATION - PostgreSQL interval for claim lock (default: "5 minutes") -// WORKER_IDLE_DELAY_MS - Poll interval when idle in ms (default: 10000) -// WORKER_MAX_BACKOFF_MS - Max error backoff in ms (default: 60000) -// WORKER_REFRESH_INTERVAL_MS - Engine re-discovery interval in ms (default: 60000) -// -// ============================================================================= - -/** - * Parse an integer from an environment variable with NaN guard. - */ -function parseIntEnv( - name: string, - value: string, - defaultValue: string, -): number { - const raw = value || defaultValue; - const parsed = parseInt(raw, 10); - if (Number.isNaN(parsed)) { - throw new Error( - `Invalid value for ${name}: "${raw}" is not a valid integer`, - ); - } - return parsed; -} - -const port = process.env.PORT || 3000; - -const accountsDatabaseUrl = process.env.ACCOUNTS_DATABASE_URL; -if (!accountsDatabaseUrl) { - throw new Error("ACCOUNTS_DATABASE_URL environment variable is required"); -} - -const accountsMasterKey = process.env.ACCOUNTS_MASTER_KEY; -if (!accountsMasterKey) { - throw new Error("ACCOUNTS_MASTER_KEY environment variable is required"); -} - -const engineDatabaseUrl = process.env.ENGINE_DATABASE_URL; -if (!engineDatabaseUrl) { - throw new Error("ENGINE_DATABASE_URL environment variable is required"); -} - -const apiBaseUrl = process.env.API_BASE_URL; -if (!apiBaseUrl) { - throw new Error("API_BASE_URL environment variable is required"); -} - -const deviceFlowCleanupCron = - process.env.DEVICE_FLOW_CLEANUP_CRON || "*/15 * * * *"; - -const accountsSchema = process.env.ACCOUNTS_SCHEMA || "accounts"; - -const workerCount = parseIntEnv( - "WORKER_COUNT", - process.env.WORKER_COUNT || "", - "2", -); - -// Connection pool settings - Accounts database -const accountsPoolMax = parseIntEnv( - "ACCOUNTS_POOL_MAX", - process.env.ACCOUNTS_POOL_MAX || "", - "10", -); -const accountsPoolIdleReapSeconds = parseIntEnv( - "ACCOUNTS_POOL_IDLE_REAP_SECONDS", - process.env.ACCOUNTS_POOL_IDLE_REAP_SECONDS || "", - "300", -); -const accountsPoolMaxLifetime = parseIntEnv( - "ACCOUNTS_POOL_MAX_LIFETIME", - process.env.ACCOUNTS_POOL_MAX_LIFETIME || "", - "0", -); -const accountsPoolConnectionTimeout = parseIntEnv( - "ACCOUNTS_POOL_CONNECTION_TIMEOUT", - process.env.ACCOUNTS_POOL_CONNECTION_TIMEOUT || "", - "30", -); - -// Connection pool settings - Engine database -const enginePoolMax = parseIntEnv( - "ENGINE_POOL_MAX", - process.env.ENGINE_POOL_MAX || "", - "20", -); -const enginePoolIdleReapSeconds = parseIntEnv( - "ENGINE_POOL_IDLE_REAP_SECONDS", - process.env.ENGINE_POOL_IDLE_REAP_SECONDS || "", - "300", -); -const enginePoolMaxLifetime = parseIntEnv( - "ENGINE_POOL_MAX_LIFETIME", - process.env.ENGINE_POOL_MAX_LIFETIME || "", - "0", -); -const enginePoolConnectionTimeout = parseIntEnv( - "ENGINE_POOL_CONNECTION_TIMEOUT", - process.env.ENGINE_POOL_CONNECTION_TIMEOUT || "", - "30", -); - -// Connection pool settings - Embedding worker engine database -const workerEngineDatabaseUrl = - process.env.WORKER_ENGINE_DATABASE_URL || engineDatabaseUrl; -const workerEnginePoolMax = parseIntEnv( - "WORKER_ENGINE_POOL_MAX", - process.env.WORKER_ENGINE_POOL_MAX || "", - String(Math.max(workerCount, 1)), -); -const workerEnginePoolIdleReapSeconds = parseIntEnv( - "WORKER_ENGINE_POOL_IDLE_REAP_SECONDS", - process.env.WORKER_ENGINE_POOL_IDLE_REAP_SECONDS || "", - String(enginePoolIdleReapSeconds), -); -const workerEnginePoolMaxLifetime = parseIntEnv( - "WORKER_ENGINE_POOL_MAX_LIFETIME", - process.env.WORKER_ENGINE_POOL_MAX_LIFETIME || "", - String(enginePoolMaxLifetime), -); -const workerEnginePoolConnectionTimeout = parseIntEnv( - "WORKER_ENGINE_POOL_CONNECTION_TIMEOUT", - process.env.WORKER_ENGINE_POOL_CONNECTION_TIMEOUT || "", - String(enginePoolConnectionTimeout), -); -const workerEngineTimeouts: EngineTimeouts = { - statementTimeout: - process.env.WORKER_ENGINE_STATEMENT_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.statementTimeout, - lockTimeout: - process.env.WORKER_ENGINE_LOCK_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.lockTimeout, - transactionTimeout: - process.env.WORKER_ENGINE_TRANSACTION_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.transactionTimeout, - idleInTransactionSessionTimeout: - process.env.WORKER_ENGINE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? - DEFAULT_ENGINE_TIMEOUTS.idleInTransactionSessionTimeout, -}; - -// ============================================================================= -// Embedding Config -// ============================================================================= -// -// Model and dimensions are hardcoded - all engines use the same embedding model. -// Only the API key is configurable via environment variable. -// -// Required: -// EMBEDDING_API_KEY - OpenAI API key -// -// Optional: -// EMBEDDING_BASE_URL - API base URL (default: OpenAI) -// EMBEDDING_TIMEOUT_MS - Per-call timeout in ms (default: none) -// EMBEDDING_MAX_RETRIES - Retries on transient failures (default: 2, from AI SDK) -// EMBEDDING_MAX_PARALLEL_CALLS - Max concurrent batch chunk requests (default: Infinity) -// -// ============================================================================= - -function buildEmbeddingConfig(): EmbeddingConfig { - const apiKey = process.env.EMBEDDING_API_KEY; - if (!apiKey) { - throw new Error("EMBEDDING_API_KEY is required"); - } - - const options: EmbeddingConfig["options"] = {}; - - if (process.env.EMBEDDING_TIMEOUT_MS) { - options.timeoutMs = parseIntEnv( - "EMBEDDING_TIMEOUT_MS", - process.env.EMBEDDING_TIMEOUT_MS, - "0", - ); - } - if (process.env.EMBEDDING_MAX_RETRIES) { - options.maxRetries = parseIntEnv( - "EMBEDDING_MAX_RETRIES", - process.env.EMBEDDING_MAX_RETRIES, - "0", - ); - } - if (process.env.EMBEDDING_MAX_PARALLEL_CALLS) { - options.maxParallelCalls = parseIntEnv( - "EMBEDDING_MAX_PARALLEL_CALLS", - process.env.EMBEDDING_MAX_PARALLEL_CALLS, - "0", - ); - } - - return { - provider: "openai", - model: embeddingConstants.model, - dimensions: embeddingConstants.dimensions, - apiKey, - baseUrl: process.env.EMBEDDING_BASE_URL, - options, - }; -} - -const embeddingConfig = buildEmbeddingConfig(); - -// ============================================================================= -// OAuth Provider Validation -// ============================================================================= - -// Warn at startup if OAuth providers are not configured, rather than -// failing with a confusing error when someone tries to log in. -const configuredProviders: string[] = []; -if (process.env.GITHUB_CLIENT_ID && process.env.GITHUB_CLIENT_SECRET) { - configuredProviders.push("github"); -} -if (process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) { - configuredProviders.push("google"); -} -if (configuredProviders.length === 0) { - console.warn( - "WARNING: No OAuth providers configured. Set GITHUB_CLIENT_ID/GITHUB_CLIENT_SECRET or GOOGLE_CLIENT_ID/GOOGLE_CLIENT_SECRET.", - ); -} else { - info("OAuth providers configured", { providers: configuredProviders }); -} - -// ============================================================================= -// Database Pools -// ============================================================================= - -// Parse master key from hex string to Buffer -const masterKeyBuffer = Buffer.from(accountsMasterKey, "hex"); -if (masterKeyBuffer.length !== 32) { - throw new Error( - "ACCOUNTS_MASTER_KEY must be a 32-byte (64 character) hex string", - ); -} - -// Create database connection pools -const accountsSql = new Bun.SQL(accountsDatabaseUrl, { - max: accountsPoolMax, - idleTimeout: accountsPoolIdleReapSeconds, - maxLifetime: accountsPoolMaxLifetime, - connectionTimeout: accountsPoolConnectionTimeout, -}); - -const engineSql = new Bun.SQL(engineDatabaseUrl, { - max: enginePoolMax, - idleTimeout: enginePoolIdleReapSeconds, - maxLifetime: enginePoolMaxLifetime, - connectionTimeout: enginePoolConnectionTimeout, -}); - -const workerEngineSql = new Bun.SQL(workerEngineDatabaseUrl, { - max: workerEnginePoolMax, - idleTimeout: workerEnginePoolIdleReapSeconds, - maxLifetime: workerEnginePoolMaxLifetime, - connectionTimeout: workerEnginePoolConnectionTimeout, -}); - -// Create accounts DB with operations layer -const accountsDb = createAccountsDB(accountsSql, accountsSchema, { - masterKey: masterKeyBuffer, -}); - -// ============================================================================= -// Database Bootstrap & Migrations (blocking — server won't serve until current) -// ============================================================================= - -// Bootstrap engine database (extensions + roles, idempotent) -// If the DB user lacks CREATE EXTENSION privileges (e.g., RDS), this will -// throw with a clear error describing what's missing. -await bootstrapEngine(engineSql); -info("Engine database bootstrapped"); - -// Migrate accounts schema (scaffold creates schema if missing) -const accountsMigrateResult = await migrateAccounts( - accountsSql, - { schema: accountsSchema }, - SERVER_VERSION, -); -if (accountsMigrateResult.status === "error") { - throw new Error( - `Accounts migration failed: ${accountsMigrateResult.error?.message}`, - ); -} -if (accountsMigrateResult.applied.length > 0) { - info("Accounts migrations applied", { - applied: accountsMigrateResult.applied, - }); -} else { - info("Accounts schema up to date"); -} - -// Ensure encryption data key exists (idempotent) -try { - const keyId = await accountsDb.createDataKey(); - await accountsDb.activateDataKey(keyId); - info("Encryption data key created", { keyId }); -} catch { - // Key already exists — expected on subsequent startups -} - -// Migrate all engine schemas -const engineSchemas = await discoverEngineSchemas(engineSql); -if (engineSchemas.length > 0) { - const engineMigrateResults = await migrateEngines( - engineSql, - engineSchemas, - { embedding_dimensions: embeddingConstants.dimensions }, - SERVER_VERSION, - ); - - let totalApplied = 0; - let totalErrors = 0; - for (const [schema, result] of engineMigrateResults) { - if (result.status === "error") { - totalErrors++; - reportError( - `Engine migration failed for ${schema}`, - result.error ?? new Error("Unknown migration error"), - ); - } else if (result.applied.length > 0) { - totalApplied += result.applied.length; - } - } - - if (totalErrors > 0) { - throw new Error( - `${totalErrors} engine schema(s) failed to migrate. Check logs for details.`, - ); - } - - if (totalApplied > 0) { - info("Engine migrations applied", { - schemas: engineSchemas.length, - totalApplied, - }); - } else { - info("Engine schemas up to date", { schemas: engineSchemas.length }); - } -} else { - info("No engine schemas to migrate"); -} - -// ============================================================================= -// Router -// ============================================================================= - -const serverContext: ServerContext = { - accountsDb, - accountsSql, - engineSql, - embeddingConfig, - apiBaseUrl, - serverVersion: SERVER_VERSION, - minClientVersion: MIN_CLIENT_VERSION, -}; - -const router = createRouter(serverContext); - -// ============================================================================= -// Embedding Worker Pool -// ============================================================================= - -const workerPool = new WorkerPool(workerEngineSql, { - embedding: embeddingConfig, - discover: async () => { - const engines = await accountsDb.listActiveEngines(); - return engines.map((e) => ({ - schema: slugToSchema(e.slug), - shard: e.shardId, - })); - }, - batchSize: parseIntEnv( - "WORKER_BATCH_SIZE", - process.env.WORKER_BATCH_SIZE || "", - "10", - ), - lockDuration: process.env.WORKER_LOCK_DURATION || "5 minutes", - idleDelayMs: parseIntEnv( - "WORKER_IDLE_DELAY_MS", - process.env.WORKER_IDLE_DELAY_MS || "", - "10000", - ), - maxBackoffMs: parseIntEnv( - "WORKER_MAX_BACKOFF_MS", - process.env.WORKER_MAX_BACKOFF_MS || "", - "60000", - ), - refreshIntervalMs: parseIntEnv( - "WORKER_REFRESH_INTERVAL_MS", - process.env.WORKER_REFRESH_INTERVAL_MS || "", - "60000", - ), - workerEngineTimeouts, -}); - -await workerPool.start(workerCount); -info("Embedding worker pool started", { workers: workerCount }); - -// ============================================================================= -// Cleanup Jobs -// ============================================================================= - -// Cleanup expired device authorizations on a cron schedule (UTC) -const cleanupCron = Bun.cron(deviceFlowCleanupCron, async () => { - try { - const count = await accountsDb.deleteExpired(); - if (count > 0) { - info("Cleaned up expired device authorizations", { count }); - } - } catch (error) { - reportError("Failed to cleanup device authorizations", error as Error); - } -}); - -// ============================================================================= -// Server -// ============================================================================= - -const server = Bun.serve({ - port, - async fetch(request) { - const url = new URL(request.url); - const method = request.method; - const path = url.pathname; - - try { - return await span("http.request", { - attributes: { - "http.method": method, - "http.url": request.url, - "http.path": path, - }, - callback: async () => { - // Check size limit - const sizeError = checkSizeLimit(request); - if (sizeError) { - return sizeError; - } - - // Route and handle request - return await router.handleRequest(request); - }, - }); - } catch { - // Error already recorded on http.request span by the helper - return internalError(); - } - }, -}); - -info("Server started", { port }); +// Boot the full stack. All env parsing / pools / migrate / worker / Bun.serve +// happen inside startServer(); see start.ts for the documented environment +// variables. +const srv = await startServer(); // ============================================================================= // Graceful Shutdown @@ -571,33 +57,11 @@ let shutdownRequested = false; async function shutdown() { if (shutdownRequested) return; shutdownRequested = true; - - info("Shutting down server..."); - - // Stop accepting new connections - server.stop(); - - // Stop embedding workers try { - await workerPool.stop(); - info("Embedding worker pool stopped"); + await srv.stop(); } catch (error) { - reportError("Error stopping embedding workers", error as Error); + reportError("Error during shutdown", error as Error); } - - // Stop background jobs - cleanupCron.stop(); - - // Close database pools - try { - await accountsSql.close(); - await engineSql.close(); - await workerEngineSql.close(); - } catch (error) { - reportError("Error closing database connections", error as Error); - } - - info("Shutdown complete"); process.exit(0); } @@ -616,3 +80,5 @@ process.on("uncaughtException", (error) => { reportError("Uncaught exception", error); process.exit(1); }); + +info("Server ready", { url: srv.url }); diff --git a/packages/server/lib.ts b/packages/server/lib.ts index 87c6d48..09142bf 100644 --- a/packages/server/lib.ts +++ b/packages/server/lib.ts @@ -2,13 +2,11 @@ // Type exports for consumers who need to test or extend the server export type { ServerContext } from "./context"; -export { - type AccountsAuthContext, - type AuthContext, - type AuthResult, - authenticateAccounts, - authenticateEngine, - ENGINE_SCHEMA_PREFIX, - type EngineAuthContext, -} from "./middleware"; +export { extractBearerToken } from "./middleware"; export { createRouter, type Router } from "./router"; +export { + buildEmbeddingConfig, + type RunningServer, + type StartServerOptions, + startServer, +} from "./start"; diff --git a/packages/server/middleware/authenticate-space.integration.test.ts b/packages/server/middleware/authenticate-space.integration.test.ts new file mode 100644 index 0000000..4fb94f9 --- /dev/null +++ b/packages/server/middleware/authenticate-space.integration.test.ts @@ -0,0 +1,263 @@ +// Integration test for space authentication (authenticateSpace). +// +// Stands up auth + core schemas and the space DB in one database, provisions a +// user (auth identity + core principal + space + owner grant), then exercises +// the session and api-key credential modes plus the failure paths. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/middleware/authenticate-space.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + generateSlug, + migrateAuth, + migrateCore, + provisionSpace, +} from "@memory.build/database"; +import * as engineCore from "@memory.build/engine/core"; +import postgres, { type Sql } from "postgres"; +import { addSpaceCreator, provisionUser } from "../provision"; +import { authenticateSpace, SPACE_HEADER } from "./authenticate-space"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; +const email = () => `space_${crypto.randomUUID().slice(0, 8)}@example.com`; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +// The deps authenticateSpace needs; bound to the test schemas. +function deps() { + return { + core: engineCore.coreStore(sql, coreSchema), + auth: authStore(sql, authSchema), + db: sql, + }; +} + +/** Build a request with optional bearer token + X-Me-Space header. */ +function req(opts: { token?: string; space?: string }): Request { + const headers: Record = {}; + if (opts.token) headers.Authorization = `Bearer ${opts.token}`; + if (opts.space) headers[SPACE_HEADER] = opts.space; + return new Request("http://localhost/api/v1/memory/rpc", { + method: "POST", + headers, + }); +} + +// Provision a user + space and return its slug, the user id, and a session token. +async function provision() { + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: email(), + name: "Tester", + provider: "github", + accountId: crypto.randomUUID(), + }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + const { token } = await authStore(sql, authSchema).createSession(r.userId); + return { ...r, token }; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +test("session: member with owner grant resolves space + treeAccess", async () => { + const p = await provision(); + const result = await authenticateSpace( + req({ token: p.token, space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.context.space.id).toBe(p.spaceId); + expect(result.context.principalId).toBe(p.userId); + expect(result.context.apiKeyId).toBeNull(); + // the creator owns the shared root (and its own home), not owner@root + expect(result.context.treeAccess).toContainEqual({ + tree_path: "share", + access: engineCore.ACCESS.owner, + }); + } +}); + +test("api key: agent of the space resolves with apiKeyId set", async () => { + const p = await provision(); + const core = engineCore.coreStore(sql, coreSchema); + + const agentId = await core.createAgent(p.userId, `agent-${rand()}`); + await core.addPrincipalToSpace(p.spaceId, agentId); + // grant within the owner's access (it owns `share`) so the agent's clamped + // effective access is non-empty — the owner is no longer owner@root. + await core.grantTreeAccess( + p.spaceId, + agentId, + "share", + engineCore.ACCESS.read, + ); + const key = await core.createApiKey(agentId, "ci"); + const fullKey = engineCore.formatApiKey(key.lookupId, key.secret); + + const result = await authenticateSpace( + req({ token: fullKey, space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.context.principalId).toBe(agentId); + expect(result.context.apiKeyId).not.toBeNull(); + expect(result.context.treeAccess.length).toBeGreaterThan(0); + } +}); + +test("api key is global: one key authenticates into every space the agent belongs to", async () => { + const p = await provision(); + const core = engineCore.coreStore(sql, coreSchema); + + // A second space also created by p, so p (the agent's owner) has access in + // both — the agent's effective access is clamped to its owner's. + const slug2 = generateSlug(); + const spaceId2 = await core.createSpace(slug2, "second"); + await provisionSpace(sql, { slug: slug2 }); + createdSpaceSchemas.push(`me_${slug2}`); + await addSpaceCreator(core, spaceId2, p.userId); + + const agentId = await core.createAgent(p.userId, `agent-${rand()}`); + for (const sid of [p.spaceId, spaceId2]) { + await core.addPrincipalToSpace(sid, agentId); + await core.grantTreeAccess(sid, agentId, "share", engineCore.ACCESS.read); + } + const key = await core.createApiKey(agentId, "ci"); + const fullKey = engineCore.formatApiKey(key.lookupId, key.secret); + + for (const slug of [p.spaceSlug, slug2]) { + const result = await authenticateSpace( + req({ token: fullKey, space: slug }), + deps(), + ); + expect(result.ok).toBe(true); + if (result.ok) expect(result.context.principalId).toBe(agentId); + } +}); + +test("legacy 4-part api key → 401 with a LEGACY_API_KEY recreate message", async () => { + const p = await provision(); + // A token shaped like the retired me... format. + const legacy = `me.${p.spaceSlug}.${"a".repeat(16)}.${"s".repeat(32)}`; + const result = await authenticateSpace( + req({ token: legacy, space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.status).toBe(401); + const body = (await result.error.json()) as { + error: { code: string; message: string }; + }; + expect(body.error.code).toBe("LEGACY_API_KEY"); + expect(body.error.message).toContain("me apikey create"); + } +}); + +test("missing Authorization → 401", async () => { + const result = await authenticateSpace( + req({ space: "abcdef012345" }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(401); +}); + +test("missing X-Me-Space → 400", async () => { + const p = await provision(); + const result = await authenticateSpace(req({ token: p.token }), deps()); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(400); +}); + +test("unknown space → 401", async () => { + const p = await provision(); + const result = await authenticateSpace( + req({ token: p.token, space: "zzzzzz999999" }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(401); +}); + +test("invalid session token → 401", async () => { + const p = await provision(); + const result = await authenticateSpace( + req({ token: "not-a-real-session-token", space: p.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(401); +}); + +test("api key: agent with no access in the requested space → 403", async () => { + const p = await provision(); + const other = await provision(); + const core = engineCore.coreStore(sql, coreSchema); + const agentId = await core.createAgent(p.userId, `agent-${rand()}`); + await core.addPrincipalToSpace(p.spaceId, agentId); + await core.grantTreeAccess( + p.spaceId, + agentId, + "share", + engineCore.ACCESS.read, + ); + const key = await core.createApiKey(agentId, "ci"); + // A valid global key, but the agent has no access in `other` — the access gate + // (build_tree_access empty) denies it rather than a parse-time rejection. + const fullKey = engineCore.formatApiKey(key.lookupId, key.secret); + const result = await authenticateSpace( + req({ token: fullKey, space: other.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(403); +}); + +test("session: member of another space has no grant here → 403", async () => { + const a = await provision(); + const b = await provision(); + // b's session against a's space — b has no grant in a's space. + const result = await authenticateSpace( + req({ token: b.token, space: a.spaceSlug }), + deps(), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.status).toBe(403); +}); diff --git a/packages/server/middleware/authenticate-space.ts b/packages/server/middleware/authenticate-space.ts new file mode 100644 index 0000000..05291f8 --- /dev/null +++ b/packages/server/middleware/authenticate-space.ts @@ -0,0 +1,185 @@ +/** + * Authentication for the space memory RPC (`/api/v1/memory/rpc`). + * + * Resolves an authenticated principal and a target space into the access set + * (`treeAccess`) that the space SQL functions consume. Two credential modes, + * discriminated by whether the bearer token parses as an api key: + * + * - api key (agent): `me..` — validated against core. + * - session (human): an opaque session token — validated against auth. + * + * The space is always selected by the `X-Me-Space` header (uniform for both + * modes). `core.buildTreeAccess(principalId, space.id)` is the single + * authorization gate: a principal with no grants in the space resolves to an + * empty set and is denied. Api keys are global, so a key whose principal isn't a + * member of the requested space is denied here rather than at parse time. + */ +import type { AuthStore } from "@memory.build/auth"; +import { slugToSchema } from "@memory.build/database"; +import { + type CoreStore, + isLegacyApiKey, + parseApiKey, + type Space, + type TreeAccess, +} from "@memory.build/engine/core"; +import { type SpaceStore, spaceStore } from "@memory.build/engine/space"; +import { SPACE_HEADER } from "@memory.build/protocol/headers"; +import { debug, span } from "@pydantic/logfire-node"; +import type { Sql } from "postgres"; +import { error, forbidden, unauthorized } from "../util/response"; +import { extractBearerToken } from "./authenticate"; + +export { SPACE_HEADER }; + +/** + * The authenticated principal + resolved space for a memory RPC request. + */ +export interface SpaceAuthContext { + type: "space"; + /** Space data-plane store bound to the `me_` schema. */ + store: SpaceStore; + /** Core control-plane store (shared; used by the management methods). */ + core: CoreStore; + /** The resolved space. */ + space: Space; + /** Authenticated principal id (user id for sessions, agent id for api keys). */ + principalId: string; + /** Api key id when authenticated by api key; null for sessions. */ + apiKeyId: string | null; + /** The principal's effective grants in this space — the access gate. */ + treeAccess: TreeAccess; + /** Whether the principal is a space admin (principal_space.admin). */ + admin: boolean; +} + +export type SpaceAuthResult = + | { ok: true; context: SpaceAuthContext } + | { ok: false; error: Response }; + +export interface SpaceAuthDeps { + /** Core control-plane store (on the new-model pool). */ + core: CoreStore; + /** Auth store (auth schema) for session validation. */ + auth: AuthStore; + /** New-model pool — used to bind the per-space data-plane store. */ + db: Sql; +} + +/** + * Authenticate a memory RPC request and resolve its space access set. + */ +export async function authenticateSpace( + request: Request, + deps: SpaceAuthDeps, +): Promise { + return span("auth.space", { + attributes: { "auth.type": "space" }, + callback: () => authenticateSpaceInner(request, deps), + }); +} + +async function authenticateSpaceInner( + request: Request, + deps: SpaceAuthDeps, +): Promise { + const { core, auth, db } = deps; + + // 1. Bearer token (a session token or an api key). + const token = extractBearerToken(request); + if (!token) { + debug("space auth failed: missing Authorization header"); + return { + ok: false, + error: unauthorized("Missing or invalid Authorization header"), + }; + } + + // 2. Space slug — always from the X-Me-Space header (uniform for both modes). + const slug = request.headers.get(SPACE_HEADER); + if (!slug) { + debug("space auth failed: missing X-Me-Space header"); + return { + ok: false, + error: error(`Missing ${SPACE_HEADER} header`, 400, "MISSING_SPACE"), + }; + } + + // 3. Resolve the space (shared step). Generic 401 to avoid space enumeration. + const space = await core.getSpace(slug); + if (!space) { + debug("space auth failed: unknown space", { slug }); + return { ok: false, error: unauthorized("Invalid credentials") }; + } + + // 4. Resolve the principal — the only line that differs between modes. + const parsed = parseApiKey(token); + let principalId: string; + let apiKeyId: string | null; + + if (parsed) { + // Api keys are global; the space comes solely from the header. A key whose + // principal isn't a member of this space falls through to the empty-access + // gate below (403), not a parse-time rejection. + const validated = await core.validateApiKey(parsed.lookupId, parsed.secret); + if (!validated) { + debug("space auth failed: invalid api key"); + return { ok: false, error: unauthorized("Invalid credentials") }; + } + principalId = validated.memberId; + apiKeyId = validated.apiKeyId; + } else if (isLegacyApiKey(token)) { + // A pre-global 4-part key (me...). These no longer + // authenticate; tell the operator to recreate the key rather than failing + // with a confusing generic 401. + debug("space auth failed: legacy 4-part api key"); + return { + ok: false, + error: error( + "This API key uses the old space-scoped format (me...) and no longer works. Recreate it with `me apikey create `, then update ME_API_KEY or your MCP/plugin config.", + 401, + "LEGACY_API_KEY", + ), + }; + } else { + const session = await auth.validateSession(token); + if (!session) { + debug("space auth failed: invalid or expired session"); + return { ok: false, error: unauthorized("Invalid credentials") }; + } + principalId = session.userId; + apiKeyId = null; + } + + // 5. The single membership / authorization gate. An empty set means the + // principal has no grants in this space (incl. a wrong-space api key) — deny. + const treeAccess = await core.buildTreeAccess(principalId, space.id); + if (treeAccess.length === 0) { + debug("space auth failed: no access in space", { slug, principalId }); + return { ok: false, error: forbidden("No access to this space") }; + } + + // 6. Bind the data-plane store to this space's schema, and resolve whether + // the principal is a space admin (membership-level management authority). + const store = spaceStore(db, slugToSchema(space.slug)); + const admin = await core.isSpaceAdmin(principalId, space.id); + + debug("space auth succeeded", { + slug, + principalId, + byApiKey: apiKeyId !== null, + }); + return { + ok: true, + context: { + type: "space", + store, + core, + space, + principalId, + apiKeyId, + treeAccess, + admin, + }, + }; +} diff --git a/packages/server/middleware/authenticate-user.ts b/packages/server/middleware/authenticate-user.ts new file mode 100644 index 0000000..ac665d9 --- /dev/null +++ b/packages/server/middleware/authenticate-user.ts @@ -0,0 +1,48 @@ +/** + * Authentication for the user RPC (`/api/v1/user/rpc`). + * + * User-scoped, session-only: it resolves the calling human (a user principal) + * from a session token. Api keys are agent credentials and do not authenticate + * here (agents can't manage agents), so an api-key token simply fails session + * validation → 401. + */ +import type { AuthStore } from "@memory.build/auth"; +import { debug, span } from "@pydantic/logfire-node"; +import { unauthorized } from "../util/response"; +import { extractBearerToken } from "./authenticate"; + +export interface UserAuthContext { + type: "user"; + /** The authenticated user id (== the core user-principal id). */ + userId: string; +} + +export type UserAuthResult = + | { ok: true; context: UserAuthContext } + | { ok: false; error: Response }; + +export async function authenticateUser( + request: Request, + auth: AuthStore, +): Promise { + return span("auth.user", { + attributes: { "auth.type": "user" }, + callback: async () => { + const token = extractBearerToken(request); + if (!token) { + debug("user auth failed: missing Authorization header"); + return { + ok: false, + error: unauthorized("Missing or invalid Authorization header"), + }; + } + const session = await auth.validateSession(token); + if (!session) { + debug("user auth failed: invalid or expired session"); + return { ok: false, error: unauthorized("Invalid or expired session") }; + } + debug("user auth succeeded", { userId: session.userId }); + return { ok: true, context: { type: "user", userId: session.userId } }; + }, + }); +} diff --git a/packages/server/middleware/authenticate.test.ts b/packages/server/middleware/authenticate.test.ts index 4492193..2041edc 100644 --- a/packages/server/middleware/authenticate.test.ts +++ b/packages/server/middleware/authenticate.test.ts @@ -1,15 +1,5 @@ -import { describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; -import type { EngineDB } from "@memory.build/engine"; -import type { SQL } from "bun"; -import { - authenticateAccounts, - authenticateEngine, - type CreateEngineDBFn, - type EngineInfo, - extractBearerToken, - type Identity, -} from "./authenticate"; +import { describe, expect, test } from "bun:test"; +import { extractBearerToken } from "./authenticate"; describe("extractBearerToken", () => { test("extracts token from valid Authorization header", () => { @@ -38,256 +28,3 @@ describe("extractBearerToken", () => { expect(extractBearerToken(request)).toBeNull(); }); }); - -describe("authenticateAccounts", () => { - const mockIdentity: Identity = { - id: "identity-123", - email: "test@example.com", - name: "Test User", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: null, - }; - - test("returns 401 when no Authorization header", async () => { - const request = new Request("http://localhost/test"); - const mockDb = { - validateSession: mock(() => Promise.resolve(null)), - } as unknown as AccountsDB; - - const result = await authenticateAccounts(request, mockDb); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 401 when session validation fails", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: "Bearer invalid-token" }, - }); - const validateSession = mock(() => Promise.resolve(null)); - const mockDb = { validateSession } as unknown as AccountsDB; - - const result = await authenticateAccounts(request, mockDb); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - expect(validateSession).toHaveBeenCalledWith("invalid-token"); - }); - - test("returns identity when session is valid", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: "Bearer valid-token" }, - }); - const mockDb = { - validateSession: mock(() => - Promise.resolve({ - session: { id: "session-1", identityId: mockIdentity.id }, - identity: mockIdentity, - }), - ), - } as unknown as AccountsDB; - - const result = await authenticateAccounts(request, mockDb); - - expect(result.ok).toBe(true); - if (result.ok && result.context.type === "accounts") { - expect(result.context.identity).toEqual(mockIdentity); - } - }); -}); - -describe("authenticateEngine", () => { - const mockEngine: EngineInfo = { - id: "engine-123", - orgId: "org-456", - slug: "abc123xyz789", - name: "Test Engine", - shardId: 7, - status: "active", - }; - - const createMockAccountsDb = (engine: EngineInfo | null) => - ({ - getEngineBySlug: mock(() => Promise.resolve(engine)), - }) as unknown as AccountsDB; - - const createMockEngineDb = (validationResult: { - valid: boolean; - userId?: string; - apiKeyId?: string; - }) => - ({ - validateApiKey: mock(() => Promise.resolve(validationResult)), - setUser: mock(() => {}), - }) as unknown as EngineDB; - - const mockCreateEngineDB = mock((_sql: SQL, _schema: string) => { - return createMockEngineDb({ - valid: true, - userId: "user-789", - apiKeyId: "apikey-abc", - }); - }) as unknown as CreateEngineDBFn; - - // Valid API key format: me.{slug}.{lookupId}.{secret} - // Secret must be exactly 32 chars (base64url) - const validApiKey = - "me.abc123xyz789.Sh00uLs5rmSHHun3.pREy3xfnbCpgUXiaBcDefghij1234567"; - - test("returns 401 when no Authorization header", async () => { - const request = new Request("http://localhost/test"); - const mockAccountsDb = createMockAccountsDb(mockEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 401 when API key format is invalid", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: "Bearer invalid-format" }, - }); - const mockAccountsDb = createMockAccountsDb(mockEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 401 when engine not found", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const mockAccountsDb = createMockAccountsDb(null); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns 403 when engine is suspended", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const suspendedEngine = { ...mockEngine, status: "suspended" as const }; - const mockAccountsDb = createMockAccountsDb(suspendedEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(403); - } - }); - - test("returns 403 when engine is deleted", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const deletedEngine = { ...mockEngine, status: "deleted" as const }; - const mockAccountsDb = createMockAccountsDb(deletedEngine); - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDB, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(403); - } - }); - - test("returns 401 when API key validation fails", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const mockAccountsDb = createMockAccountsDb(mockEngine); - const mockCreateEngineDBInvalid = mock(() => - createMockEngineDb({ valid: false }), - ) as unknown as CreateEngineDBFn; - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDBInvalid, - ); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error.status).toBe(401); - } - }); - - test("returns engine context when authentication succeeds", async () => { - const request = new Request("http://localhost/test", { - headers: { Authorization: `Bearer ${validApiKey}` }, - }); - const mockAccountsDb = createMockAccountsDb(mockEngine); - const mockEngineDb = createMockEngineDb({ - valid: true, - userId: "user-789", - apiKeyId: "apikey-abc", - }); - const mockCreateEngineDBSuccess = mock( - () => mockEngineDb, - ) as unknown as CreateEngineDBFn; - - const result = await authenticateEngine( - request, - mockAccountsDb, - {} as SQL, - mockCreateEngineDBSuccess, - ); - - expect(result.ok).toBe(true); - if (result.ok && result.context.type === "engine") { - expect(result.context.db).toBe(mockEngineDb); - expect(result.context.userId).toBe("user-789"); - expect(result.context.apiKeyId).toBe("apikey-abc"); - expect(result.context.engine).toEqual(mockEngine); - expect(mockEngineDb.setUser).toHaveBeenCalledWith("user-789"); - expect(mockCreateEngineDBSuccess).toHaveBeenCalledWith( - {} as SQL, - "me_abc123xyz789", - { shard: 7 }, - ); - } - }); -}); diff --git a/packages/server/middleware/authenticate.ts b/packages/server/middleware/authenticate.ts index 0a636fb..6772a55 100644 --- a/packages/server/middleware/authenticate.ts +++ b/packages/server/middleware/authenticate.ts @@ -1,94 +1,11 @@ -import type { AccountsDB } from "@memory.build/accounts"; -import { - createEngineDB, - type EngineDB, - parseApiKey, -} from "@memory.build/engine"; -import { debug, span } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import { forbidden, unauthorized } from "../util/response"; - -// ============================================================================= -// Constants -// ============================================================================= - -/** - * Schema prefix for engine databases. - * Engine schemas are named `{ENGINE_SCHEMA_PREFIX}{engineSlug}`. - */ -export const ENGINE_SCHEMA_PREFIX = "me_"; - -// ============================================================================= -// Types -// ============================================================================= - -/** - * Identity from accounts DB (for accounts RPC). - */ -export interface Identity { - id: string; - email: string; - name: string; - createdAt: Date; - updatedAt: Date | null; -} - -/** - * Engine info from accounts DB. - */ -export interface EngineInfo { - id: string; - orgId: string; - slug: string; - name: string; - shardId: number; - status: "active" | "suspended" | "deleted"; -} - /** - * Auth context for accounts RPC requests. - */ -export interface AccountsAuthContext { - type: "accounts"; - identity: Identity; -} - -/** - * Auth context for engine RPC requests. - */ -export interface EngineAuthContext { - type: "engine"; - db: EngineDB; - userId: string; - apiKeyId: string; - engine: EngineInfo; -} - -/** - * Factory function type for creating EngineDB instances. - * Allows dependency injection for testing. - */ -export type CreateEngineDBFn = ( - sql: SQL, - schema: string, - options?: { shard?: number }, -) => EngineDB; - -/** - * Union type for all auth contexts. - */ -export type AuthContext = AccountsAuthContext | EngineAuthContext; - -/** - * Result of authentication attempt. + * Shared bearer-token extraction for the RPC auth middlewares. + * + * The per-endpoint authenticators live in `authenticate-space.ts` (memory RPC: + * session or api key + X-Me-Space) and `authenticate-user.ts` (user RPC: + * session only). Both resolve the credential from the `Authorization` header + * via `extractBearerToken`. */ -export type AuthResult = - | { ok: true; context: AuthContext } - | { ok: false; error: Response }; - -// ============================================================================= -// Helpers -// ============================================================================= /** * Extract Bearer token from Authorization header. @@ -112,175 +29,3 @@ export function extractBearerToken(request: Request): string | null { return token; } - -// ============================================================================= -// Accounts Authentication -// ============================================================================= - -/** - * Authenticate request for accounts RPC. - * Validates session token and returns identity. - */ -export async function authenticateAccounts( - request: Request, - accountsDb: AccountsDB, -): Promise { - const token = extractBearerToken(request); - if (!token) { - debug("accounts auth failed: missing Authorization header"); - return { - ok: false, - error: unauthorized("Missing or invalid Authorization header"), - }; - } - - const sessionResult = await accountsDb.validateSession(token); - if (!sessionResult) { - debug("accounts auth failed: invalid or expired session"); - return { - ok: false, - error: unauthorized("Invalid or expired session"), - }; - } - - debug("accounts auth succeeded", { identityId: sessionResult.identity.id }); - return { - ok: true, - context: { - type: "accounts", - identity: sessionResult.identity, - }, - }; -} - -// ============================================================================= -// Engine Authentication -// ============================================================================= - -/** - * Authenticate request for engine RPC. - * Parses API key, looks up engine, validates key against engine DB. - * - * Security note: Error messages are intentionally generic to prevent - * enumeration attacks. The specific failure reason is logged for debugging. - */ -export async function authenticateEngine( - request: Request, - accountsDb: AccountsDB, - engineSql: SQL, - createEngineDBFn: CreateEngineDBFn = createEngineDB, -): Promise { - return span("auth.engine", { - attributes: { - "auth.type": "engine", - }, - callback: () => - authenticateEngineInner(request, accountsDb, engineSql, createEngineDBFn), - }); -} - -async function authenticateEngineInner( - request: Request, - accountsDb: AccountsDB, - engineSql: SQL, - createEngineDBFn: CreateEngineDBFn, -): Promise { - // 1. Extract bearer token - const token = extractBearerToken(request); - if (!token) { - debug("engine auth failed: missing Authorization header"); - return { - ok: false, - error: unauthorized("Missing or invalid Authorization header"), - }; - } - - // 2. Parse API key - const parsed = parseApiKey(token); - if (!parsed) { - debug("engine auth failed: invalid API key format"); - return { ok: false, error: unauthorized("Invalid API key") }; - } - - const { engineSlug, lookupId, secret } = parsed; - - // 3. Look up engine in accounts DB - const engine = await span("auth.engine.accounts_lookup", { - attributes: { - "engine.slug": engineSlug, - "api_key.lookup_id": lookupId, - }, - callback: () => accountsDb.getEngineBySlug(engineSlug), - }); - if (!engine) { - // Generic error to prevent engine enumeration - debug("engine auth failed: engine not found", { engineSlug }); - return { ok: false, error: unauthorized("Invalid API key") }; - } - - // 4. Check engine status - if (engine.status !== "active") { - // 403 Forbidden for suspended/deleted engines - the key is valid but access is denied - debug("engine auth failed: engine not active", { - engineSlug, - status: engine.status, - }); - return { - ok: false, - error: forbidden("Access denied"), - }; - } - - // 5. Create EngineDB for this engine's schema - const schema = `${ENGINE_SCHEMA_PREFIX}${engineSlug}`; - const db = createEngineDBFn(engineSql, schema, { shard: engine.shardId }); - - // 6. Validate API key - const validation = await span("auth.engine.validate_api_key", { - attributes: { - "db.schema": schema, - "engine.id": engine.id, - "engine.slug": engineSlug, - "engine.shard": engine.shardId, - "api_key.lookup_id": lookupId, - }, - callback: () => db.validateApiKey(lookupId, secret), - }); - if (!validation.valid || !validation.userId || !validation.apiKeyId) { - debug("engine auth failed: API key validation failed", { - engineSlug, - lookupId, - }); - return { ok: false, error: unauthorized("Invalid API key") }; - } - - // 7. Set user on db for RLS context - db.setUser(validation.userId); - - // 8. Build engine info - const engineInfo: EngineInfo = { - id: engine.id, - orgId: engine.orgId, - slug: engine.slug, - name: engine.name, - shardId: engine.shardId, - status: engine.status, - }; - - debug("engine auth succeeded", { - engineSlug, - userId: validation.userId, - apiKeyId: validation.apiKeyId, - }); - - return { - ok: true, - context: { - type: "engine", - db, - userId: validation.userId, - apiKeyId: validation.apiKeyId, - engine: engineInfo, - }, - }; -} diff --git a/packages/server/middleware/client-version.test.ts b/packages/server/middleware/client-version.test.ts index c9570be..0a24923 100644 --- a/packages/server/middleware/client-version.test.ts +++ b/packages/server/middleware/client-version.test.ts @@ -5,7 +5,7 @@ import { checkClientVersion } from "./client-version"; const MIN = "0.2.0"; function req(headers: Record = {}): Request { - return new Request("http://localhost/api/v1/engine/rpc", { + return new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers, }); diff --git a/packages/server/middleware/index.ts b/packages/server/middleware/index.ts index dafa8e0..78a0d48 100644 --- a/packages/server/middleware/index.ts +++ b/packages/server/middleware/index.ts @@ -1,16 +1,11 @@ +export { extractBearerToken } from "./authenticate"; export { - type AccountsAuthContext, - type AuthContext, - type AuthResult, - authenticateAccounts, - authenticateEngine, - type CreateEngineDBFn, - ENGINE_SCHEMA_PREFIX, - type EngineAuthContext, - type EngineInfo, - extractBearerToken, - type Identity, -} from "./authenticate"; + authenticateSpace, + SPACE_HEADER, + type SpaceAuthContext, + type SpaceAuthDeps, + type SpaceAuthResult, +} from "./authenticate-space"; export { checkClientVersion } from "./client-version"; export { checkSizeLimit, diff --git a/packages/server/package.json b/packages/server/package.json index f747f73..d4a9840 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -2,12 +2,14 @@ "name": "memory-engine-server", "version": "0.2.5", "dependencies": { - "@memory.build/accounts": "workspace:*", + "@memory.build/auth": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", "@memory.build/engine": "workspace:*", "@memory.build/protocol": "workspace:*", "@memory.build/worker": "workspace:*", "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9", "zod": "^4.0.0" }, "exports": { diff --git a/packages/server/provision.integration.test.ts b/packages/server/provision.integration.test.ts new file mode 100644 index 0000000..8abfd59 --- /dev/null +++ b/packages/server/provision.integration.test.ts @@ -0,0 +1,142 @@ +// Integration test for first-login provisioning (provisionUser). +// +// Stands up auth + core schemas and bootstraps the space DB in one database, +// then provisions users through a single connection (the one-pool model the +// server consolidates to in Phase 4). +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 packages/server/provision.integration.test.ts +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import * as engineCore from "@memory.build/engine/core"; +import postgres, { type Sql } from "postgres"; +import { provisionUser } from "./provision"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; +const email = () => `prov_${crypto.randomUUID().slice(0, 8)}@example.com`; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +async function schemaExists(name: string): Promise { + const [r] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${name} + ) as e`; + return Boolean(r?.e); +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); // extensions for me_ + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +test("provisions a new user: identity + principal + space + owner grant", async () => { + const e = email(); + const accountId = crypto.randomUUID(); + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { email: e, name: "Alice", provider: "github", accountId }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + + // auth.users + const user = await authStore(sql, authSchema).getUser(r.userId); + expect(user?.email).toBe(e); + + // oauth account link resolves back to the user + const acct = await authStore(sql, authSchema).getAccountByProvider( + "github", + accountId, + ); + expect(acct?.userId).toBe(r.userId); + + // core principal shares the same id + const principal = await engineCore + .coreStore(sql, coreSchema) + .getPrincipal(r.userId); + expect(principal?.kind).toBe("u"); + expect(principal?.id).toBe(r.userId); + + // space registered in core + its data schema exists + const space = await engineCore + .coreStore(sql, coreSchema) + .getSpace(r.spaceSlug); + expect(space?.id).toBe(r.spaceId); + expect(await schemaExists(`me_${r.spaceSlug}`)).toBe(true); + + // the creator's default grants: owner of its home + the shared root (`share`), + // but NOT owner@root + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(r.userId, r.spaceId); + expect(ta).toContainEqual({ + tree_path: "share", + access: engineCore.ACCESS.owner, + }); + expect(ta).toContainEqual({ + tree_path: `home.${r.userId.replace(/-/g, "")}`, + access: engineCore.ACCESS.owner, + }); + expect(ta).not.toContainEqual({ + tree_path: engineCore.ROOT_PATH, + access: engineCore.ACCESS.owner, + }); +}); + +test("is atomic: a failure rolls everything back", async () => { + const e = email(); + const a1 = crypto.randomUUID(); + const r1 = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { email: e, name: "Bob", provider: "github", accountId: a1 }, + ); + createdSpaceSchemas.push(`me_${r1.spaceSlug}`); + + // re-provisioning the same email fails (users.email is unique) — the whole + // transaction must roll back, leaving no trace of the second attempt. + const a2 = crypto.randomUUID(); + await expect( + provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { email: e, name: "Bob2", provider: "github", accountId: a2 }, + ), + ).rejects.toThrow(); + + // the second account link was rolled back + expect( + await authStore(sql, authSchema).getAccountByProvider("github", a2), + ).toBeNull(); +}); diff --git a/packages/server/provision.ts b/packages/server/provision.ts new file mode 100644 index 0000000..e3fd5c1 --- /dev/null +++ b/packages/server/provision.ts @@ -0,0 +1,97 @@ +import { authStore, type OAuthProvider } from "@memory.build/auth"; +import { + generateSlug, + provisionSpace, + SHARE_NAMESPACE, +} from "@memory.build/database"; +import * as engineCore from "@memory.build/engine/core"; +import type { Sql } from "postgres"; + +/** + * First-login provisioning. + * + * Atomically (one transaction) stands up everything a brand-new user needs: + * - auth.users row (the global identity) + the OAuth account link + * - core.principal (kind 'u') sharing the SAME id as auth.users + * - a default core.space + its me_ data schema (provisionSpace runs the + * schema DDL inside this transaction) + * - the user as space admin + owner of its home and the shared root (`share`), + * not owner@root + * + * Because schema creation is transactional, any failure rolls the whole thing + * back — no orphaned me_ schema, no cleanup code. No API key is minted: + * keys are agent-only; humans reach the engine via their session. + * + * Requires a single connection that can write the auth, core, and me_ + * schemas (the DB must already be bootstrapped with the required extensions). + */ +export interface ProvisionUserParams { + email: string; + /** Display name, stored on auth.users. */ + name: string; + provider: OAuthProvider; + /** The provider's stable account id (the OAuth `sub`). */ + accountId: string; + emailVerified?: boolean; + image?: string; + /** Name for the personal space (default "default"). */ + spaceName?: string; +} + +export interface ProvisionUserResult { + userId: string; + spaceId: string; + spaceSlug: string; +} + +/** + * Grant a new space's creator its default access — shared by first-login + * provisioning and `space.create` so the two stay in lockstep. The creator + * becomes a space admin who owns its home (via add_principal_to_space) and the + * shared root (`share`), but NOT owner@root: it sees `/share` and its own `~`, + * not other members' homes. As an admin it can self-grant owner@root later if it + * wants the whole tree. Call inside the space-creation transaction. + */ +export async function addSpaceCreator( + core: engineCore.CoreStore, + spaceId: string, + userId: string, +): Promise { + await core.addPrincipalToSpace(spaceId, userId, true); // admin + owner@home + await core.grantTreeAccess( + spaceId, + userId, + SHARE_NAMESPACE, + engineCore.ACCESS.owner, + ); +} + +export function provisionUser( + sql: Sql, + schemas: { auth: string; core: string }, + params: ProvisionUserParams, +): Promise { + const slug = generateSlug(); + + return sql.begin(async (tx) => { + const auth = authStore(tx as unknown as Sql, schemas.auth); + const core = engineCore.coreStore(tx as unknown as Sql, schemas.core); + + const userId = await auth.createUser(params.email, params.name, { + emailVerified: params.emailVerified, + image: params.image, + }); + await auth.upsertAccount(userId, params.provider, params.accountId); + + // The core principal shares the auth user id (one identity across schemas). + // Its globally-unique principal name is the email — the natural unique + // handle for a user (display name lives on auth.users.name). + await core.createUser(userId, params.email); + + const spaceId = await core.createSpace(slug, params.spaceName ?? "default"); + await provisionSpace(tx, { slug }); // creates the me_ data schema + await addSpaceCreator(core, spaceId, userId); + + return { userId, spaceId, spaceSlug: slug }; + }) as Promise; +} diff --git a/packages/server/router.test.ts b/packages/server/router.test.ts index 3651abe..fce1db2 100644 --- a/packages/server/router.test.ts +++ b/packages/server/router.test.ts @@ -1,7 +1,8 @@ import { describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { SQL } from "bun"; +import type { CoreStore } from "@memory.build/engine/core"; +import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; import { createRouter } from "./router"; @@ -9,12 +10,13 @@ import { createRouter } from "./router"; // Mock ServerContext for testing function createMockContext(): ServerContext { return { - accountsDb: { + db: {} as Sql, + auth: { validateSession: mock(() => Promise.resolve(null)), - getEngineBySlug: mock(() => Promise.resolve(null)), - } as unknown as AccountsDB, - accountsSql: {} as SQL, - engineSql: {} as SQL, + } as unknown as AuthStore, + core: {} as unknown as CoreStore, + authSchema: "auth", + coreSchema: "core", embeddingConfig: { provider: "openai", model: "text-embedding-3-small", @@ -103,30 +105,30 @@ describe("matchRoute", () => { }); }); - describe("accounts RPC endpoint", () => { - test("matches POST /api/v1/accounts/rpc", () => { - const match = router.matchRoute("POST", "/api/v1/accounts/rpc"); + describe("memory RPC endpoint", () => { + test("matches POST /api/v1/memory/rpc", () => { + const match = router.matchRoute("POST", "/api/v1/memory/rpc"); expect(match).not.toBeNull(); - expect(match?.route.pattern).toBe("/api/v1/accounts/rpc"); + expect(match?.route.pattern).toBe("/api/v1/memory/rpc"); expect(match?.params).toEqual({}); }); - test("does not match GET /api/v1/accounts/rpc", () => { - const match = router.matchRoute("GET", "/api/v1/accounts/rpc"); + test("does not match GET /api/v1/memory/rpc", () => { + const match = router.matchRoute("GET", "/api/v1/memory/rpc"); expect(match).toBeNull(); }); }); - describe("engine RPC endpoint", () => { - test("matches POST /api/v1/engine/rpc", () => { - const match = router.matchRoute("POST", "/api/v1/engine/rpc"); + describe("user RPC endpoint", () => { + test("matches POST /api/v1/user/rpc", () => { + const match = router.matchRoute("POST", "/api/v1/user/rpc"); expect(match).not.toBeNull(); - expect(match?.route.pattern).toBe("/api/v1/engine/rpc"); + expect(match?.route.pattern).toBe("/api/v1/user/rpc"); expect(match?.params).toEqual({}); }); - test("does not match GET /api/v1/engine/rpc", () => { - const match = router.matchRoute("GET", "/api/v1/engine/rpc"); + test("does not match GET /api/v1/user/rpc", () => { + const match = router.matchRoute("GET", "/api/v1/user/rpc"); expect(match).toBeNull(); }); }); @@ -204,7 +206,7 @@ describe("handleRequest", () => { const ctx = createMockContext(); const router = createRouter(ctx); - const request = new Request("http://localhost/api/v1/engine/rpc", { + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers: { "Content-Type": "application/json", @@ -232,7 +234,7 @@ describe("handleRequest", () => { const ctx = createMockContext(); const router = createRouter(ctx); - const request = new Request("http://localhost/api/v1/engine/rpc", { + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ diff --git a/packages/server/router.ts b/packages/server/router.ts index 98bb645..a7230bf 100644 --- a/packages/server/router.ts +++ b/packages/server/router.ts @@ -1,6 +1,7 @@ import type { ServerContext } from "./context"; import { type AuthHandlerContext, + deviceApproveHandler, deviceCodeHandler, deviceTokenHandler, deviceVerifyGetHandler, @@ -9,12 +10,10 @@ import { } from "./handlers/auth"; import { healthHandler, readyHandler } from "./handlers/health"; import { versionHandler } from "./handlers/version"; -import { - authenticateAccounts, - authenticateEngine, -} from "./middleware/authenticate"; +import { authenticateSpace } from "./middleware/authenticate-space"; +import { authenticateUser } from "./middleware/authenticate-user"; import { checkClientVersion } from "./middleware/client-version"; -import { accountsMethods, createRpcHandler, engineMethods } from "./rpc"; +import { createRpcHandler, memoryMethods, userMethods } from "./rpc"; import { notFound } from "./util/response"; /** @@ -120,21 +119,24 @@ export interface Router { */ export function createRouter(ctx: ServerContext): Router { const { - accountsDb, - accountsSql, - engineSql, + db, + auth, + core, + authSchema, + coreSchema, embeddingConfig, apiBaseUrl, serverVersion, minClientVersion, } = ctx; - // Auth handler context for device flow endpoints + // Auth handler context for the device flow + OAuth endpoints const authCtx: AuthHandlerContext = { - db: accountsDb, + auth, + db, + authSchema, + coreSchema, baseUrl: apiBaseUrl, - engineSql, - serverVersion, }; // Wrap an RPC handler with the X-Client-Version check, so requests from @@ -151,49 +153,33 @@ export function createRouter(ctx: ServerContext): Router { }; } - // Engine RPC: authenticate and provide db context - const engineRpcHandler = createRpcHandler(engineMethods, async (request) => { - const auth = await authenticateEngine(request, accountsDb, engineSql); - if (!auth.ok) { - return auth.error; - } - // TypeScript narrows auth.context to AuthContext after ok check - // We know it's EngineAuthContext since we called authenticateEngine - const ctx = auth.context; - if (ctx.type !== "engine") { - throw new Error("Unexpected auth context type"); + // Memory RPC (new model): authenticate principal + space, provide space context + const memoryRpcHandler = createRpcHandler(memoryMethods, async (request) => { + const result = await authenticateSpace(request, { core, auth, db }); + if (!result.ok) { + return result.error; } + const spaceContext = result.context; return { - db: ctx.db, - userId: ctx.userId, - apiKeyId: ctx.apiKeyId, - engine: ctx.engine, + store: spaceContext.store, + core: spaceContext.core, + space: spaceContext.space, + principalId: spaceContext.principalId, + apiKeyId: spaceContext.apiKeyId, + treeAccess: spaceContext.treeAccess, + admin: spaceContext.admin, embeddingConfig, }; }); - // Accounts RPC: authenticate and provide identity context - const accountsRpcHandler = createRpcHandler( - accountsMethods, - async (request) => { - const auth = await authenticateAccounts(request, accountsDb); - if (!auth.ok) { - return auth.error; - } - // TypeScript narrows auth.context to AuthContext after ok check - // We know it's AccountsAuthContext since we called authenticateAccounts - const ctx = auth.context; - if (ctx.type !== "accounts") { - throw new Error("Unexpected auth context type"); - } - return { - db: accountsDb, - identity: ctx.identity, - engineSql, - serverVersion, - }; - }, - ); + // User RPC (new model): session-only, user-scoped (agent lifecycle) + const userRpcHandler = createRpcHandler(userMethods, async (request) => { + const result = await authenticateUser(request, auth); + if (!result.ok) { + return result.error; + } + return { core, auth, userId: result.context.userId, db, coreSchema }; + }); /** * Application routes. @@ -210,7 +196,7 @@ export function createRouter(ctx: ServerContext): Router { { method: "GET", pattern: "/ready", - handler: readyHandler(accountsSql, engineSql), + handler: readyHandler(db), }, // Version compatibility check (unauthenticated) @@ -246,6 +232,13 @@ export function createRouter(ctx: ServerContext): Router { handler: (req) => deviceVerifyPostHandler(req, authCtx), }, + // OAuth Device Flow - User approves/denies after OAuth (consent step) + { + method: "POST", + pattern: "/api/v1/auth/device/approve", + handler: (req) => deviceApproveHandler(req, authCtx), + }, + // OAuth Callback - Provider redirects here after user authorizes { method: "GET", @@ -253,18 +246,18 @@ export function createRouter(ctx: ServerContext): Router { handler: (req, params) => oauthCallbackHandler(req, params, authCtx), }, - // Accounts RPC + // Memory RPC (new model: space data-plane + management) { method: "POST", - pattern: "/api/v1/accounts/rpc", - handler: withClientVersionCheck(accountsRpcHandler), + pattern: "/api/v1/memory/rpc", + handler: withClientVersionCheck(memoryRpcHandler), }, - // Engine RPC + // User RPC (new model: session-only, user-scoped agent lifecycle) { method: "POST", - pattern: "/api/v1/engine/rpc", - handler: withClientVersionCheck(engineRpcHandler), + pattern: "/api/v1/user/rpc", + handler: withClientVersionCheck(userRpcHandler), }, ]; diff --git a/packages/server/rpc/accounts/engine.integration.test.ts b/packages/server/rpc/accounts/engine.integration.test.ts deleted file mode 100644 index ece1bb1..0000000 --- a/packages/server/rpc/accounts/engine.integration.test.ts +++ /dev/null @@ -1,651 +0,0 @@ -/** - * Integration tests for engine provisioning via RPC. - * - * Tests that engine.create properly: - * 1. Creates engine record with correct language - * 2. Provisions schema in the engine database - * 3. Defaults language to "english" when not specified - */ - -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { - type AccountsDB, - createAccountsDB, - type Identity, -} from "@memory.build/accounts"; -import { TestDatabase as AccountsTestDatabase } from "@memory.build/accounts/migrate/test-utils"; -import { createEngineDB } from "@memory.build/engine"; -import { bootstrap } from "@memory.build/engine/migrate/bootstrap"; -import { TestDatabase as EngineTestDatabase } from "@memory.build/engine/migrate/test-utils"; -import { SQL } from "bun"; -import { SERVER_VERSION } from "../../../../version"; -import type { HandlerContext } from "../types"; -import { engineMethods } from "./engine"; -import type { AccountsRpcContext } from "./types"; - -// Test master key (32 bytes for AES-256) -const TEST_MASTER_KEY = Buffer.from( - "0123456789abcdef0123456789abcdef", - "utf-8", -); - -// Test fixtures -let accountsTestDb: AccountsTestDatabase; -let engineTestDb: EngineTestDatabase; -let accountsDb: AccountsDB; -let engineSql: SQL; -let engineConnectionString: string; - -// Test data -let testOrgId: string; -let testIdentity: Identity; - -beforeAll(async () => { - // Set up accounts database - accountsTestDb = await AccountsTestDatabase.create(); - accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema, { - masterKey: TEST_MASTER_KEY, - }); - - // Create and activate encryption key - const keyId = await accountsDb.createDataKey(); - await accountsDb.activateDataKey(keyId); - - // Set up engine database - engineTestDb = new EngineTestDatabase(); - engineConnectionString = await engineTestDb.create(); - engineSql = new SQL(engineConnectionString); - - // Bootstrap the engine database (extensions, roles) - await bootstrap(engineSql); - - // Create test identity and org - testIdentity = await accountsDb.createIdentity({ - email: "engine-test@example.com", - name: "Engine Test User", - }); - - const org = await accountsDb.createOrg({ - name: "Engine Test Org", - }); - testOrgId = org.id; - - // Make identity an owner of the org - await accountsDb.addMember(org.id, testIdentity.id, "owner"); -}); - -afterAll(async () => { - await engineSql.close(); - await engineTestDb.drop(); - await accountsTestDb.dispose(); -}); - -/** - * Helper to create a context for engine methods. - */ -function createContext(identity: Identity): HandlerContext { - return { - request: new Request("http://localhost"), - db: accountsDb, - identity, - engineSql, - serverVersion: SERVER_VERSION, - } as unknown as AccountsRpcContext; -} - -/** - * Helper to check if a schema exists. - */ -async function schemaExists(sql: SQL, name: string): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.schemata - where schema_name = ${name} - ) as exists - `; - return row.exists; -} - -/** - * Helper to check if a table exists in a schema. - */ -async function tableExists( - sql: SQL, - schema: string, - table: string, -): Promise { - const [row] = await sql` - select exists ( - select 1 from information_schema.tables - where table_schema = ${schema} and table_name = ${table} - ) as exists - `; - return row.exists; -} - -/** - * Helper to extract the text_config from the BM25 index definition. - * The config is embedded in the index during migration, not stored in a table. - */ -async function getBm25TextConfig( - sql: SQL, - schema: string, -): Promise { - try { - const [row] = await sql` - select pg_get_indexdef(indexrelid) as def - from pg_index i - join pg_class c on c.oid = i.indexrelid - join pg_namespace n on n.oid = c.relnamespace - where n.nspname = ${schema} - and c.relname = 'memory_content_bm25_idx' - `; - if (!row?.def) return null; - - // Extract text_config from: "... WITH (text_config=english, ..." (no quotes) - const match = row.def.match(/text_config=(\w+)/); - return match?.[1] ?? null; - } catch { - return null; - } -} - -// --------------------------------------------------------------------------- -// Engine Provisioning Tests -// --------------------------------------------------------------------------- - -describe("engine.create integration", () => { - test("creates engine record with default language (english)", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Default Language Engine" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify engine record - expect(result.id).toBeDefined(); - expect(result.slug).toMatch(/^[a-z0-9]{12}$/); - expect(result.language).toBe("english"); - - // Verify schema was provisioned - const schema = `me_${result.slug}`; - expect(await schemaExists(engineSql, schema)).toBe(true); - - // Verify core tables exist - expect(await tableExists(engineSql, schema, "memory")).toBe(true); - expect(await tableExists(engineSql, schema, "user")).toBe(true); - expect(await tableExists(engineSql, schema, "api_key")).toBe(true); - expect(await tableExists(engineSql, schema, "version")).toBe(true); - expect(await tableExists(engineSql, schema, "migration")).toBe(true); - - // Verify config has correct bm25_text_config - expect(await getBm25TextConfig(engineSql, schema)).toBe("english"); - }); - - test("creates engine record with custom language (german)", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "German Engine", language: "german" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify engine record - expect(result.id).toBeDefined(); - expect(result.language).toBe("german"); - - // Verify schema was provisioned - const schema = `me_${result.slug}`; - expect(await schemaExists(engineSql, schema)).toBe(true); - - // Verify config has correct bm25_text_config - expect(await getBm25TextConfig(engineSql, schema)).toBe("german"); - }); - - test("creates engine record with simple language (simple)", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Simple Engine", language: "simple" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify engine record - expect(result.language).toBe("simple"); - - // Verify schema was provisioned with simple text config - const schema = `me_${result.slug}`; - expect(await getBm25TextConfig(engineSql, schema)).toBe("simple"); - }); - - test("provisions schema with embedding_queue table", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Queue Test Engine" }, - context, - )) as { slug: string }; - - const schema = `me_${result.slug}`; - expect(await tableExists(engineSql, schema, "embedding_queue")).toBe(true); - }); - - test("rejects non-member creating engine", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - // Create a new identity that is NOT a member of the org - const outsider = await accountsDb.createIdentity({ - email: "outsider@example.com", - name: "Outsider", - }); - - const context = createContext(outsider); - - await expect( - handler({ orgId: testOrgId, name: "Unauthorized Engine" }, context), - ).rejects.toThrow("Only owners and admins can create engines"); - }); - - test("rejects member (non-admin) creating engine", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - // Create a new identity and add as regular member - const member = await accountsDb.createIdentity({ - email: "member@example.com", - name: "Regular Member", - }); - await accountsDb.addMember(testOrgId, member.id, "member"); - - const context = createContext(member); - - await expect( - handler({ orgId: testOrgId, name: "Member Engine" }, context), - ).rejects.toThrow("Only owners and admins can create engines"); - }); - - test("admin can create engine", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - // Create a new identity and add as admin - const admin = await accountsDb.createIdentity({ - email: "admin@example.com", - name: "Admin User", - }); - await accountsDb.addMember(testOrgId, admin.id, "admin"); - - const context = createContext(admin); - - const result = (await handler( - { orgId: testOrgId, name: "Admin Engine" }, - context, - )) as { id: string; slug: string }; - - expect(result.id).toBeDefined(); - const schema = `me_${result.slug}`; - expect(await schemaExists(engineSql, schema)).toBe(true); - }); - - test("engine record is persisted in accounts database", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Persisted Engine" }, - context, - )) as { id: string; slug: string; language: string }; - - // Verify we can fetch the engine from the accounts DB - const engine = await accountsDb.getEngine(result.id); - expect(engine).not.toBeNull(); - expect(engine?.name).toBe("Persisted Engine"); - expect(engine?.orgId).toBe(testOrgId); - expect(engine?.status).toBe("active"); - expect(engine?.language).toBe("english"); - }); - - test("engine can be retrieved by slug after creation", async () => { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - - const context = createContext(testIdentity); - const result = (await handler( - { orgId: testOrgId, name: "Slug Lookup Engine" }, - context, - )) as { id: string; slug: string }; - - // Verify we can fetch the engine by slug - const engine = await accountsDb.getEngineBySlug(result.slug); - expect(engine).not.toBeNull(); - expect(engine?.id).toBe(result.id); - }); -}); - -// --------------------------------------------------------------------------- -// engine.update Tests -// --------------------------------------------------------------------------- - -describe("engine.update integration", () => { - function getCreateHandler() { - const handler = engineMethods.get("engine.create")?.handler; - if (!handler) throw new Error("engine.create handler not found"); - return handler; - } - - function getUpdateHandler() { - const handler = engineMethods.get("engine.update")?.handler; - if (!handler) throw new Error("engine.update handler not found"); - return handler; - } - - test("owner can rename engine; slug is unchanged", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const context = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Rename Source" }, - context, - )) as { id: string; slug: string; updatedAt: string | null }; - - const result = (await update( - { id: created.id, name: "Renamed Engine" }, - context, - )) as { id: string; name: string; slug: string; updatedAt: string | null }; - - expect(result.id).toBe(created.id); - expect(result.name).toBe("Renamed Engine"); - expect(result.slug).toBe(created.slug); - expect(result.updatedAt).not.toBeNull(); - - // Verify the underlying schema name (which uses slug) is unaffected. - expect(await schemaExists(engineSql, `me_${created.slug}`)).toBe(true); - }); - - test("admin can rename engine", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const ownerCtx = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Admin Rename Source" }, - ownerCtx, - )) as { id: string }; - - const admin = await accountsDb.createIdentity({ - email: "rename-admin@example.com", - name: "Rename Admin", - }); - await accountsDb.addMember(testOrgId, admin.id, "admin"); - - const result = (await update( - { id: created.id, name: "Admin Renamed" }, - createContext(admin), - )) as { name: string }; - - expect(result.name).toBe("Admin Renamed"); - }); - - test("member (non-admin) cannot rename engine", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const ownerCtx = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Member Rename Source" }, - ownerCtx, - )) as { id: string }; - - const member = await accountsDb.createIdentity({ - email: "rename-member@example.com", - name: "Rename Member", - }); - await accountsDb.addMember(testOrgId, member.id, "member"); - - await expect( - update( - { id: created.id, name: "Forbidden Rename" }, - createContext(member), - ), - ).rejects.toThrow("Only owners and admins can update engines"); - }); - - test("non-member cannot rename engine", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const ownerCtx = createContext(testIdentity); - - const created = (await create( - { orgId: testOrgId, name: "Outsider Rename Source" }, - ownerCtx, - )) as { id: string }; - - const outsider = await accountsDb.createIdentity({ - email: "rename-outsider@example.com", - name: "Rename Outsider", - }); - - await expect( - update( - { id: created.id, name: "Outsider Rename" }, - createContext(outsider), - ), - ).rejects.toThrow("Only owners and admins can update engines"); - }); - - test("rename to a sibling's name returns CONFLICT", async () => { - const create = getCreateHandler(); - const update = getUpdateHandler(); - const context = createContext(testIdentity); - - await create({ orgId: testOrgId, name: "Conflict A" }, context); - const second = (await create( - { orgId: testOrgId, name: "Conflict B" }, - context, - )) as { id: string }; - - await expect( - update({ id: second.id, name: "Conflict A" }, context), - ).rejects.toThrow(/already exists in this organization/); - }); - - test("rename a non-existent engine returns NOT_FOUND", async () => { - const update = getUpdateHandler(); - const context = createContext(testIdentity); - - await expect( - update( - { id: "019d694f-79f6-7595-8faf-b70b01c11f98", name: "Nope" }, - context, - ), - ).rejects.toThrow(/Engine not found/); - }); -}); - -// --------------------------------------------------------------------------- -// engine.setupAccess Tests -// --------------------------------------------------------------------------- - -describe("engine.setupAccess integration", () => { - // Create a dedicated engine for setupAccess tests - let setupAccessEngineId: string; - let setupAccessEngineSlug: string; - - beforeAll(async () => { - const createHandler = engineMethods.get("engine.create")?.handler; - if (!createHandler) throw new Error("engine.create handler not found"); - - const result = (await createHandler( - { orgId: testOrgId, name: "SetupAccess Test Engine" }, - createContext(testIdentity), - )) as { id: string; slug: string }; - - setupAccessEngineId = result.id; - setupAccessEngineSlug = result.slug; - }); - - function getHandler() { - const handler = engineMethods.get("engine.setupAccess")?.handler; - if (!handler) throw new Error("engine.setupAccess handler not found"); - return handler; - } - - test("owner gets superuser + createrole user and API key", async () => { - const handler = getHandler(); - const context = createContext(testIdentity); - - const result = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { - rawKey: string; - engineSlug: string; - userId: string; - engineName: string; - orgName: string; - }; - - expect(result.rawKey).toBeDefined(); - expect(result.rawKey.length).toBeGreaterThan(0); - expect(result.engineSlug).toBe(setupAccessEngineSlug); - expect(result.userId).toBeDefined(); - expect(result.engineName).toBe("SetupAccess Test Engine"); - expect(result.orgName).toBe("Engine Test Org"); - - // Verify the engine user has superuser privileges - const engineDb = createEngineDB(engineSql, `me_${setupAccessEngineSlug}`); - const user = await engineDb.getUser(result.userId); - expect(user).not.toBeNull(); - expect(user?.superuser).toBe(true); - expect(user?.createrole).toBe(true); - expect(user?.identityId).toBe(testIdentity.id); - }); - - test("admin gets superuser + createrole user and API key", async () => { - const handler = getHandler(); - - const admin = await accountsDb.createIdentity({ - email: "setup-admin@example.com", - name: "Setup Admin", - }); - await accountsDb.addMember(testOrgId, admin.id, "admin"); - - const context = createContext(admin); - const result = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - expect(result.rawKey).toBeDefined(); - - const engineDb = createEngineDB(engineSql, `me_${setupAccessEngineSlug}`); - const user = await engineDb.getUser(result.userId); - expect(user?.superuser).toBe(true); - expect(user?.createrole).toBe(true); - }); - - test("member gets vanilla user (no superuser) and API key", async () => { - const handler = getHandler(); - - const member = await accountsDb.createIdentity({ - email: "setup-member@example.com", - name: "Setup Member", - }); - await accountsDb.addMember(testOrgId, member.id, "member"); - - const context = createContext(member); - const result = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - expect(result.rawKey).toBeDefined(); - - const engineDb = createEngineDB(engineSql, `me_${setupAccessEngineSlug}`); - const user = await engineDb.getUser(result.userId); - expect(user?.superuser).toBe(false); - expect(user?.createrole).toBe(false); - }); - - test("non-member is forbidden", async () => { - const handler = getHandler(); - - const outsider = await accountsDb.createIdentity({ - email: "setup-outsider@example.com", - name: "Setup Outsider", - }); - - const context = createContext(outsider); - - await expect( - handler({ engineId: setupAccessEngineId }, context), - ).rejects.toThrow("Not a member of the organization"); - }); - - test("engine not found returns error", async () => { - const handler = getHandler(); - const context = createContext(testIdentity); - - await expect( - handler({ engineId: "019d694f-79f6-7595-8faf-b70b01c11f98" }, context), - ).rejects.toThrow("Engine not found"); - }); - - test("idempotent: second call reuses user, creates new API key", async () => { - const handler = getHandler(); - - const idempotentUser = await accountsDb.createIdentity({ - email: "setup-idempotent@example.com", - name: "Idempotent User", - }); - await accountsDb.addMember(testOrgId, idempotentUser.id, "owner"); - - const context = createContext(idempotentUser); - - // First call - const result1 = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - // Second call - const result2 = (await handler( - { engineId: setupAccessEngineId }, - context, - )) as { userId: string; rawKey: string }; - - // Same user, different API keys - expect(result2.userId).toBe(result1.userId); - expect(result2.rawKey).not.toBe(result1.rawKey); - }); - - test("custom API key name is used", async () => { - const handler = getHandler(); - - const namedKeyUser = await accountsDb.createIdentity({ - email: "setup-named@example.com", - name: "Named Key User", - }); - await accountsDb.addMember(testOrgId, namedKeyUser.id, "owner"); - - const context = createContext(namedKeyUser); - const result = (await handler( - { engineId: setupAccessEngineId, apiKeyName: "my-custom-key" }, - context, - )) as { rawKey: string }; - - expect(result.rawKey).toBeDefined(); - }); -}); diff --git a/packages/server/rpc/accounts/engine.ts b/packages/server/rpc/accounts/engine.ts deleted file mode 100644 index 9fe4e29..0000000 --- a/packages/server/rpc/accounts/engine.ts +++ /dev/null @@ -1,432 +0,0 @@ -/** - * Accounts RPC engine methods. - * - * Implements: - * - engine.create: Create a new engine for an organization - * - engine.list: List engines for an organization - * - engine.get: Get engine by ID - * - engine.update: Update engine name/status - * - engine.delete: Delete an engine (mark deleted + drop schema) - * - engine.setupAccess: Bootstrap engine access for a session-authenticated identity - */ -import type { Engine } from "@memory.build/accounts"; -import { - createEngineDB, - type EngineConfig, - provisionEngine, -} from "@memory.build/engine"; -import { setLocalEngineTimeouts } from "@memory.build/engine/ops/_tx"; -import type { - EngineCreateParams, - EngineDeleteParams, - EngineGetParams, - EngineListParams, - EngineResponse, - EngineSetupAccessParams, - EngineSetupAccessResult, - EngineUpdateParams, -} from "@memory.build/protocol/accounts/engine"; -import { - engineCreateParams, - engineDeleteParams, - engineGetParams, - engineListParams, - engineSetupAccessParams, - engineUpdateParams, -} from "@memory.build/protocol/accounts/engine"; -import { SQL } from "bun"; -import { embeddingConstants } from "../../config"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an Engine to a serializable response. - */ -function toEngineResponse(engine: Engine): EngineResponse { - return { - id: engine.id, - orgId: engine.orgId, - slug: engine.slug, - name: engine.name, - shardId: engine.shardId, - status: engine.status, - language: engine.language, - createdAt: engine.createdAt.toISOString(), - updatedAt: engine.updatedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * engine.create - Create a new engine for an organization. - * Requires owner or admin role. - */ -async function engineCreate( - params: EngineCreateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity, engineSql, serverVersion } = - context as AccountsRpcContext; - - // Check if caller has admin or owner role - const member = await db.getMember(params.orgId, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can create engines", - ); - } - - // Create the engine record in accounts DB - const engine = await db.createEngine({ - orgId: params.orgId, - name: params.name, - language: params.language ?? "english", - }); - - // Provision the engine schema in the engine DB - const engineConfig: EngineConfig = { - embedding_dimensions: embeddingConstants.dimensions, - bm25_text_config: engine.language, - }; - - try { - await provisionEngine( - engineSql, - engine.slug, - engineConfig, - serverVersion, - engine.shardId, - ); - } catch (err) { - // Attempt to clean up partially-created schema - const schema = `me_${engine.slug}`; - try { - await engineSql.begin(async (tx) => { - await tx.unsafe(`set local pgdog.shard to ${engine.shardId}`); - await setLocalEngineTimeouts(tx); - await tx.unsafe(`drop schema if exists ${schema} cascade`); - }); - } catch { - // Log but don't mask original error - } - // Mark engine as deleted in accounts DB - await db.updateEngine(engine.id, { status: "deleted" }); - throw new AppError( - "INTERNAL_ERROR", - `Failed to provision engine schema: ${err instanceof Error ? err.message : String(err)}`, - ); - } - - return toEngineResponse(engine); -} - -/** - * engine.list - List engines for an organization. - * Requires membership in the org. - */ -async function engineList( - params: EngineListParams, - context: HandlerContext, -): Promise<{ engines: EngineResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const member = await db.getMember(params.orgId, identity.id); - if (!member) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const engines = await db.listEnginesByOrg(params.orgId); - return { engines: engines.map(toEngineResponse) }; -} - -/** - * engine.get - Get engine by ID. - * Requires membership in the org that owns the engine. - */ -async function engineGet( - params: EngineGetParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - const engine = await db.getEngine(params.id); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - // Check if caller is a member of the org - const member = await db.getMember(engine.orgId, identity.id); - if (!member) { - throw new AppError( - "FORBIDDEN", - "Not a member of the organization that owns this engine", - ); - } - - return toEngineResponse(engine); -} - -/** - * engine.update - Update engine name/status. - * Requires owner or admin role. - */ -async function engineUpdate( - params: EngineUpdateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - const engine = await db.getEngine(params.id); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - // Check if caller has admin or owner role - const member = await db.getMember(engine.orgId, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can update engines", - ); - } - - let updated: boolean; - try { - updated = await db.updateEngine(params.id, { - name: params.name, - status: params.status, - }); - } catch (err) { - // Translate the unique-name violation on (org_id, name) into a - // friendly CONFLICT error so the CLI can show a clean message. - if ( - err instanceof SQL.PostgresError && - err.errno === "23505" && - typeof err.constraint === "string" && - err.constraint.includes("name") - ) { - throw new AppError( - "CONFLICT", - `An engine named '${params.name}' already exists in this organization`, - ); - } - throw err; - } - - if (!updated) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - const updatedEngine = await db.getEngine(params.id); - if (!updatedEngine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - return toEngineResponse(updatedEngine); -} - -/** - * engine.setupAccess - Bootstrap engine access for a session-authenticated identity. - * - * Find-or-creates an engine user for the caller's identity, then creates an API key. - * Any org member can call this. Privilege level maps from org role: - * - owner/admin → superuser + createrole - * - member → vanilla user - */ -async function engineSetupAccess( - params: EngineSetupAccessParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity, engineSql } = context as AccountsRpcContext; - - // Look up the engine - const engine = await db.getEngine(params.engineId); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.engineId}`); - } - if (engine.status !== "active") { - throw new AppError( - "VALIDATION_ERROR", - `Engine is not active: ${engine.status}`, - ); - } - - // Look up the org - const org = await db.getOrg(engine.orgId); - if (!org) { - throw new AppError("NOT_FOUND", `Organization not found: ${engine.orgId}`); - } - - // Check caller's membership - const member = await db.getMember(engine.orgId, identity.id); - if (!member) { - throw new AppError( - "FORBIDDEN", - "Not a member of the organization that owns this engine", - ); - } - - // Create an EngineDB for this engine's schema - const schema = `me_${engine.slug}`; - const engineDb = createEngineDB(engineSql, schema, { shard: engine.shardId }); - - // Find or create a user for this identity - let user = await engineDb.getUserByIdentity(identity.id); - if (!user) { - const isSuperuser = member.role === "owner" || member.role === "admin"; - user = await engineDb.createUser({ - name: slugifyUserName(identity.name || identity.email), - identityId: identity.id, - canLogin: true, - superuser: isSuperuser, - createrole: isSuperuser, - }); - } - - // Create an API key for the user - const apiKeyName = - params.apiKeyName ?? `cli-${new Date().toISOString().slice(0, 10)}`; - const { rawKey } = await engineDb.createApiKey({ - userId: user.id, - name: apiKeyName, - }); - - return { - rawKey, - engineSlug: engine.slug, - userId: user.id, - engineName: engine.name, - orgName: org.name, - }; -} - -/** - * Derive a shell-friendly engine user name from a display name or email. - * Lowercases, replaces whitespace runs with hyphens, strips non-alphanumeric - * characters (except hyphens), and trims leading/trailing hyphens. - */ -function slugifyUserName(name: string): string { - return name - .toLowerCase() - .replace(/\s+/g, "-") - .replace(/[^a-z0-9-]/g, "") - .replace(/-{2,}/g, "-") - .replace(/^-|-$/g, ""); -} - -/** - * engine.delete - Delete an engine permanently. - * - * Three-step sequence (each step is idempotent so a failed prior attempt - * can be retried to completion): - * 1. Mark the engine row status='deleted' so middleware rejects new - * API-key auth immediately, even if subsequent steps stall. - * 2. Drop the engine schema (uses `drop schema if exists`). - * 3. Hard-delete the engine row from the accounts DB so it no longer - * blocks `org.delete` and disappears from listings. - * - * Requires owner role on the org. - */ -async function engineDelete( - params: EngineDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertAccountsRpcContext(context); - const { db, identity, engineSql } = context as AccountsRpcContext; - - const engine = await db.getEngine(params.id); - if (!engine) { - throw new AppError("NOT_FOUND", `Engine not found: ${params.id}`); - } - - // Only org owners can delete engines - const member = await db.getMember(engine.orgId, identity.id); - if (!member || member.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can delete engines"); - } - - // Step 1: Mark as deleted (idempotent — bumping updated_at on a row that - // is already 'deleted' is harmless and lets us proceed to retry steps - // 2 and 3 if a previous attempt failed mid-way). - if (engine.status !== "deleted") { - await db.updateEngine(engine.id, { status: "deleted" }); - } - - // Step 2: Drop the engine schema. `drop schema if exists` is idempotent. - const schema = `me_${engine.slug}`; - await dropSchemaWithRetry(engineSql, schema, { - retries: 3, - delayMs: 2000, - shardId: engine.shardId, - }); - - // Step 3: Hard-delete the row so it no longer blocks org deletion or - // appears in engine listings. Must come after the schema drop — if we - // delete the row first and the schema drop then fails, the schema is - // orphaned with no metadata to find it. - await db.deleteEngine(engine.id); - - return { deleted: true }; -} - -/** - * Drop an engine schema with retry logic. - * Sets a lock_timeout to avoid blocking indefinitely if the embedding worker - * or in-flight requests hold locks, then retries on timeout. - * Sets pgdog.shard for correct shard routing. - */ -async function dropSchemaWithRetry( - sql: import("bun").SQL, - schema: string, - opts: { retries: number; delayMs: number; shardId: number }, -): Promise { - for (let attempt = 1; attempt <= opts.retries; attempt++) { - try { - await sql.begin(async (tx) => { - await tx.unsafe(`set local pgdog.shard to ${opts.shardId}`); - await setLocalEngineTimeouts(tx); - await tx.unsafe(`set local lock_timeout = '5s'`); - await tx.unsafe(`drop schema if exists ${schema} cascade`); - }); - return; - } catch (err) { - const isLockTimeout = - err instanceof Error && err.message?.includes("lock timeout"); - if (!isLockTimeout || attempt === opts.retries) { - throw new AppError( - "INTERNAL_ERROR", - `Failed to drop engine schema ${schema}: ${err instanceof Error ? err.message : String(err)}`, - ); - } - await new Promise((resolve) => setTimeout(resolve, opts.delayMs)); - } - } -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the engine methods registry. - */ -export const engineMethods = buildRegistry() - .register("engine.create", engineCreateParams, engineCreate) - .register("engine.list", engineListParams, engineList) - .register("engine.get", engineGetParams, engineGet) - .register("engine.update", engineUpdateParams, engineUpdate) - .register("engine.delete", engineDeleteParams, engineDelete) - .register("engine.setupAccess", engineSetupAccessParams, engineSetupAccess) - .build(); diff --git a/packages/server/rpc/accounts/index.ts b/packages/server/rpc/accounts/index.ts deleted file mode 100644 index d06aca2..0000000 --- a/packages/server/rpc/accounts/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { buildRegistry } from "../registry"; -import { engineMethods } from "./engine"; -import { invitationMethods } from "./invitation"; -import { meMethods } from "./me"; -import { orgMethods } from "./org"; -import { orgMemberMethods } from "./org-member"; -import { sessionMethods } from "./session"; - -/** - * Accounts RPC method registry. - * - * Identity methods: - * - me.get - * - * Session methods: - * - session.revoke - * - * Organization methods: - * - org.create, org.list, org.get, org.update, org.delete - * - * Organization member methods: - * - org.member.list, org.member.add, org.member.remove, org.member.updateRole - * - * Engine methods: - * - engine.create, engine.list, engine.get, engine.update - * - * Invitation methods: - * - invitation.create, invitation.list, invitation.revoke, invitation.accept - */ -export const accountsMethods = buildRegistry() - .merge(meMethods) - .merge(sessionMethods) - .merge(orgMethods) - .merge(orgMemberMethods) - .merge(engineMethods) - .merge(invitationMethods) - .build(); - -// Re-export types for consumers -export type { AccountsRpcContext } from "./types"; -export { assertAccountsRpcContext, isAccountsRpcContext } from "./types"; diff --git a/packages/server/rpc/accounts/invitation.ts b/packages/server/rpc/accounts/invitation.ts deleted file mode 100644 index 8b5a3e6..0000000 --- a/packages/server/rpc/accounts/invitation.ts +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Accounts RPC invitation methods. - * - * Implements: - * - invitation.create: Create an invitation to join an organization - * - invitation.list: List pending invitations for an organization - * - invitation.revoke: Revoke a pending invitation - * - invitation.accept: Accept an invitation (adds caller to org) - */ -import type { Invitation } from "@memory.build/accounts"; -import type { - InvitationAcceptParams, - InvitationCreateParams, - InvitationCreateResult, - InvitationListParams, - InvitationResponse, - InvitationRevokeParams, -} from "@memory.build/protocol/accounts/invitation"; -import { - invitationAcceptParams, - invitationCreateParams, - invitationListParams, - invitationRevokeParams, -} from "@memory.build/protocol/accounts/invitation"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an Invitation to a serializable response. - */ -function toInvitationResponse(invitation: Invitation): InvitationResponse { - return { - id: invitation.id, - orgId: invitation.orgId, - email: invitation.email, - role: invitation.role, - invitedBy: invitation.invitedBy, - expiresAt: invitation.expiresAt.toISOString(), - acceptedAt: invitation.acceptedAt?.toISOString() ?? null, - createdAt: invitation.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * invitation.create - Create an invitation to join an organization. - * Requires owner or admin role. - * Returns the invitation with the raw token (only shown once). - */ -async function invitationCreate( - params: InvitationCreateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const member = await db.getMember(params.orgId, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can create invitations", - ); - } - - // Only owners can invite other owners - if (params.role === "owner" && member.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can invite other owners"); - } - - const result = await db.createInvitation({ - orgId: params.orgId, - email: params.email, - role: params.role, - invitedBy: identity.id, - expiresInDays: params.expiresInDays, - }); - - return { - ...toInvitationResponse(result.invitation), - token: result.rawToken, - }; -} - -/** - * invitation.list - List pending invitations for an organization. - * Requires membership in the org. - */ -async function invitationList( - params: InvitationListParams, - context: HandlerContext, -): Promise<{ invitations: InvitationResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const member = await db.getMember(params.orgId, identity.id); - if (!member) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const invitations = await db.listPendingInvitations(params.orgId); - return { invitations: invitations.map(toInvitationResponse) }; -} - -/** - * invitation.revoke - Revoke a pending invitation. - * Requires owner or admin role. - */ -async function invitationRevoke( - params: InvitationRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // We need to find the invitation first to check org membership - // Since we don't have getInvitation by ID, we'll need to verify via different means - // For now, we'll revoke and let the database handle the not found case - // The authorization check happens via listing invitations for orgs the user is admin of - - // Get all orgs the caller is admin/owner of - const orgs = await db.listOrgsByIdentity(identity.id); - - // For each org, check if the invitation belongs to it and caller has permission - for (const org of orgs) { - const member = await db.getMember(org.id, identity.id); - if (member && (member.role === "owner" || member.role === "admin")) { - const invitations = await db.listPendingInvitations(org.id); - const invitation = invitations.find((inv) => inv.id === params.id); - if (invitation) { - const revoked = await db.revokeInvitation(params.id); - return { revoked }; - } - } - } - - throw new AppError("NOT_FOUND", `Invitation not found: ${params.id}`); -} - -/** - * invitation.accept - Accept an invitation (adds caller to org). - * The caller's email must match the invitation email. - */ -async function invitationAccept( - params: InvitationAcceptParams, - context: HandlerContext, -): Promise<{ accepted: boolean; orgId: string }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Find the invitation by token - const invitation = await db.getInvitationByToken(params.token); - if (!invitation) { - throw new AppError("NOT_FOUND", "Invalid or expired invitation token"); - } - - // Check email matches (identity already available from auth - no DB lookup needed) - if (invitation.email.toLowerCase() !== identity.email.toLowerCase()) { - throw new AppError( - "FORBIDDEN", - "Invitation is for a different email address", - ); - } - - // Accept the invitation and add member in a transaction - await db.withTransaction(async (txDb) => { - const accepted = await txDb.acceptInvitation(invitation.id); - if (!accepted) { - throw new AppError("CONFLICT", "Invitation has already been accepted"); - } - - // Add the user as a member with the invited role - await txDb.addMember(invitation.orgId, identity.id, invitation.role); - }); - - return { accepted: true, orgId: invitation.orgId }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the invitation methods registry. - */ -export const invitationMethods = buildRegistry() - .register("invitation.create", invitationCreateParams, invitationCreate) - .register("invitation.list", invitationListParams, invitationList) - .register("invitation.revoke", invitationRevokeParams, invitationRevoke) - .register("invitation.accept", invitationAcceptParams, invitationAccept) - .build(); diff --git a/packages/server/rpc/accounts/me.test.ts b/packages/server/rpc/accounts/me.test.ts deleted file mode 100644 index d2d52a1..0000000 --- a/packages/server/rpc/accounts/me.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Unit tests for identity/me RPC handlers. - * - * Uses mocked AccountsDB to test handler logic in isolation. - */ -import { describe, expect, mock, test } from "bun:test"; -import type { HandlerContext } from "../types"; -import { meMethods } from "./me"; - -function createMockContext( - dbOverrides: Record = {}, -): HandlerContext { - return { - request: new Request("http://localhost"), - db: { - getIdentityByEmail: mock(() => Promise.resolve(null)), - ...dbOverrides, - }, - identity: { - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "alice@example.com", - name: "Alice", - createdAt: new Date("2026-01-15T00:00:00.000Z"), - updatedAt: null, - }, - engineSql: mock(() => {}) as unknown, - serverVersion: "0.1.1", - } as unknown as HandlerContext; -} - -// ============================================================================= -// me.get -// ============================================================================= - -describe("me.get", () => { - test("returns the authenticated identity", async () => { - const handler = meMethods.get("me.get")?.handler; - if (!handler) throw new Error("me.get handler not found"); - - const context = createMockContext(); - const result = (await handler({}, context)) as { - id: string; - email: string; - name: string; - }; - - expect(result.id).toBe("019d694f-79f6-7595-8faf-b70b01c11f98"); - expect(result.email).toBe("alice@example.com"); - expect(result.name).toBe("Alice"); - }); -}); - -// ============================================================================= -// identity.getByEmail -// ============================================================================= - -describe("identity.getByEmail", () => { - test("returns identity when found", async () => { - const handler = meMethods.get("identity.getByEmail")?.handler; - if (!handler) throw new Error("identity.getByEmail handler not found"); - - const identity = { - id: "019d694f-79f6-7595-8faf-b70b01c11f99", - email: "bob@example.com", - name: "Bob", - createdAt: new Date("2026-01-15T00:00:00.000Z"), - updatedAt: null, - }; - - const context = createMockContext({ - getIdentityByEmail: mock(() => Promise.resolve(identity)), - }); - - const result = (await handler({ email: "bob@example.com" }, context)) as { - identity: { id: string; email: string; name: string } | null; - }; - - expect(result.identity).not.toBeNull(); - expect(result.identity!.id).toBe("019d694f-79f6-7595-8faf-b70b01c11f99"); - expect(result.identity!.email).toBe("bob@example.com"); - expect(result.identity!.name).toBe("Bob"); - }); - - test("returns null identity when not found", async () => { - const handler = meMethods.get("identity.getByEmail")?.handler; - if (!handler) throw new Error("identity.getByEmail handler not found"); - - const context = createMockContext({ - getIdentityByEmail: mock(() => Promise.resolve(null)), - }); - - const result = (await handler( - { email: "nobody@example.com" }, - context, - )) as { identity: null }; - - expect(result.identity).toBeNull(); - }); - - test("passes email to db lookup", async () => { - const handler = meMethods.get("identity.getByEmail")?.handler; - if (!handler) throw new Error("identity.getByEmail handler not found"); - - const getIdentityByEmail = mock(() => Promise.resolve(null)); - const context = createMockContext({ getIdentityByEmail }); - - await handler({ email: "test@example.com" }, context); - - expect(getIdentityByEmail).toHaveBeenCalledWith("test@example.com"); - }); -}); diff --git a/packages/server/rpc/accounts/me.ts b/packages/server/rpc/accounts/me.ts deleted file mode 100644 index 01f3142..0000000 --- a/packages/server/rpc/accounts/me.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Accounts RPC me methods. - * - * Implements: - * - me.get: Get the current authenticated identity - */ -import type { Identity } from "@memory.build/accounts"; -import type { - IdentityGetByEmailParams, - IdentityGetByEmailResult, - IdentityResponse, - MeGetParams, -} from "@memory.build/protocol/accounts/identity"; -import { - identityGetByEmailParams, - meGetParams, -} from "@memory.build/protocol/accounts/identity"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertAccountsRpcContext } from "./types"; - -/** - * Convert an Identity to a serializable response. - */ -function toIdentityResponse(identity: Identity): IdentityResponse { - return { - id: identity.id, - email: identity.email, - name: identity.name, - createdAt: identity.createdAt.toISOString(), - updatedAt: identity.updatedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * me.get - Get the current authenticated identity. - */ -async function meGet( - _params: MeGetParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - // Identity is already available from authentication - no DB lookup needed - return toIdentityResponse(context.identity); -} - -/** - * identity.getByEmail - Look up an identity by email address. - */ -async function identityGetByEmail( - params: IdentityGetByEmailParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db } = context as import("./types").AccountsRpcContext; - - const identity = await db.getIdentityByEmail(params.email); - return { identity: identity ? toIdentityResponse(identity) : null }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the me/identity methods registry. - */ -export const meMethods = buildRegistry() - .register("me.get", meGetParams, meGet) - .register("identity.getByEmail", identityGetByEmailParams, identityGetByEmail) - .build(); diff --git a/packages/server/rpc/accounts/org-member.ts b/packages/server/rpc/accounts/org-member.ts deleted file mode 100644 index e397d2a..0000000 --- a/packages/server/rpc/accounts/org-member.ts +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Accounts RPC org member methods. - * - * Implements: - * - org.member.list: List members of an organization - * - org.member.add: Add a member to an organization - * - org.member.remove: Remove a member from an organization - * - org.member.updateRole: Update a member's role - */ -import { AccountsError, type OrgMember } from "@memory.build/accounts"; -import type { - OrgMemberAddParams, - OrgMemberListParams, - OrgMemberRemoveParams, - OrgMemberResponse, - OrgMemberUpdateRoleParams, -} from "@memory.build/protocol/accounts/org-member"; -import { - orgMemberAddParams, - orgMemberListParams, - orgMemberRemoveParams, - orgMemberUpdateRoleParams, -} from "@memory.build/protocol/accounts/org-member"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an OrgMember to a serializable response. - */ -function toOrgMemberResponse(member: OrgMember): OrgMemberResponse { - return { - orgId: member.orgId, - identityId: member.identityId, - role: member.role, - name: member.name, - email: member.email, - createdAt: member.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * org.member.list - List members of an organization. - * Requires membership in the org. - */ -async function orgMemberList( - params: OrgMemberListParams, - context: HandlerContext, -): Promise<{ members: OrgMemberResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const callerMember = await db.getMember(params.orgId, identity.id); - if (!callerMember) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const members = await db.listMembers(params.orgId); - return { members: members.map(toOrgMemberResponse) }; -} - -/** - * org.member.add - Add a member to an organization. - * Requires owner or admin role. - */ -async function orgMemberAdd( - params: OrgMemberAddParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const callerMember = await db.getMember(params.orgId, identity.id); - if ( - !callerMember || - (callerMember.role !== "owner" && callerMember.role !== "admin") - ) { - throw new AppError("FORBIDDEN", "Only owners and admins can add members"); - } - - // Only owners can add other owners - if (params.role === "owner" && callerMember.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can add other owners"); - } - - const member = await db.addMember( - params.orgId, - params.identityId, - params.role, - ); - return toOrgMemberResponse(member); -} - -/** - * org.member.remove - Remove a member from an organization. - * Requires owner or admin role (admins cannot remove owners). - */ -async function orgMemberRemove( - params: OrgMemberRemoveParams, - context: HandlerContext, -): Promise<{ removed: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const callerMember = await db.getMember(params.orgId, identity.id); - if ( - !callerMember || - (callerMember.role !== "owner" && callerMember.role !== "admin") - ) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can remove members", - ); - } - - // Check target member's role - const targetMember = await db.getMember(params.orgId, params.identityId); - if (!targetMember) { - throw new AppError("NOT_FOUND", "Member not found"); - } - - // Admins cannot remove owners - if (targetMember.role === "owner" && callerMember.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can remove other owners"); - } - - try { - const removed = await db.removeMember(params.orgId, params.identityId); - return { removed }; - } catch (err) { - if (err instanceof AccountsError && err.code === "ORG_MUST_HAVE_OWNER") { - throw new AppError( - "CONFLICT", - "Cannot remove the last owner from an organization", - ); - } - throw err; - } -} - -/** - * org.member.updateRole - Update a member's role. - * Requires owner or admin role (admins cannot promote to owner or demote owners). - */ -async function orgMemberUpdateRole( - params: OrgMemberUpdateRoleParams, - context: HandlerContext, -): Promise<{ updated: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const callerMember = await db.getMember(params.orgId, identity.id); - if ( - !callerMember || - (callerMember.role !== "owner" && callerMember.role !== "admin") - ) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can update member roles", - ); - } - - // Check target member's current role - const targetMember = await db.getMember(params.orgId, params.identityId); - if (!targetMember) { - throw new AppError("NOT_FOUND", "Member not found"); - } - - // Only owners can: - // - Promote to owner - // - Demote from owner - if ( - (params.role === "owner" || targetMember.role === "owner") && - callerMember.role !== "owner" - ) { - throw new AppError( - "FORBIDDEN", - "Only owners can promote to or demote from owner role", - ); - } - - try { - const updated = await db.updateRole( - params.orgId, - params.identityId, - params.role, - ); - return { updated }; - } catch (err) { - if (err instanceof AccountsError && err.code === "ORG_MUST_HAVE_OWNER") { - throw new AppError( - "CONFLICT", - "Cannot remove the last owner from an organization", - ); - } - throw err; - } -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the org member methods registry. - */ -export const orgMemberMethods = buildRegistry() - .register("org.member.list", orgMemberListParams, orgMemberList) - .register("org.member.add", orgMemberAddParams, orgMemberAdd) - .register("org.member.remove", orgMemberRemoveParams, orgMemberRemove) - .register( - "org.member.updateRole", - orgMemberUpdateRoleParams, - orgMemberUpdateRole, - ) - .build(); diff --git a/packages/server/rpc/accounts/org.integration.test.ts b/packages/server/rpc/accounts/org.integration.test.ts deleted file mode 100644 index dd7b71f..0000000 --- a/packages/server/rpc/accounts/org.integration.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Integration tests for org RPC handlers. - * - * Currently focused on org.update (rename). Other org methods are exercised - * indirectly via engine.integration.test.ts and CLI smoke tests. - */ - -import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; -import { - type AccountsDB, - createAccountsDB, - type Identity, -} from "@memory.build/accounts"; -import { TestDatabase as AccountsTestDatabase } from "@memory.build/accounts/migrate/test-utils"; -import { SERVER_VERSION } from "../../../../version"; -import type { HandlerContext } from "../types"; -import { orgMethods } from "./org"; -import type { AccountsRpcContext } from "./types"; - -const TEST_MASTER_KEY = Buffer.from( - "0123456789abcdef0123456789abcdef", - "utf-8", -); - -let accountsTestDb: AccountsTestDatabase; -let accountsDb: AccountsDB; -let testIdentity: Identity; - -beforeAll(async () => { - accountsTestDb = await AccountsTestDatabase.create(); - accountsDb = createAccountsDB(accountsTestDb.sql, accountsTestDb.schema, { - masterKey: TEST_MASTER_KEY, - }); - - const keyId = await accountsDb.createDataKey(); - await accountsDb.activateDataKey(keyId); - - testIdentity = await accountsDb.createIdentity({ - email: "org-rpc-test@example.com", - name: "Org RPC Test User", - }); -}); - -afterAll(async () => { - await accountsTestDb.dispose(); -}); - -function createContext(identity: Identity): HandlerContext { - return { - request: new Request("http://localhost"), - db: accountsDb, - identity, - // org.update never touches engineSql; a stub that satisfies the - // assertAccountsRpcContext type guard (typeof === "function") is enough. - engineSql: mock(() => {}) as unknown, - serverVersion: SERVER_VERSION, - } as unknown as AccountsRpcContext; -} - -describe("org.update integration", () => { - function getUpdateHandler() { - const handler = orgMethods.get("org.update")?.handler; - if (!handler) throw new Error("org.update handler not found"); - return handler; - } - - test("owner can rename org; slug is unchanged", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Original Org Name" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const result = (await update( - { id: org.id, name: "Renamed Org" }, - createContext(testIdentity), - )) as { id: string; name: string; slug: string; updatedAt: string | null }; - - expect(result.id).toBe(org.id); - expect(result.name).toBe("Renamed Org"); - expect(result.slug).toBe(org.slug); - expect(result.updatedAt).not.toBeNull(); - }); - - test("admin can rename org", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Admin Rename" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const admin = await accountsDb.createIdentity({ - email: "org-admin@example.com", - name: "Org Admin", - }); - await accountsDb.addMember(org.id, admin.id, "admin"); - - const result = (await update( - { id: org.id, name: "Renamed By Admin" }, - createContext(admin), - )) as { name: string }; - - expect(result.name).toBe("Renamed By Admin"); - }); - - test("member (non-admin) cannot rename org", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Member Rename" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const member = await accountsDb.createIdentity({ - email: "org-member@example.com", - name: "Org Member", - }); - await accountsDb.addMember(org.id, member.id, "member"); - - await expect( - update({ id: org.id, name: "Forbidden" }, createContext(member)), - ).rejects.toThrow("Only owners and admins can update the organization"); - }); - - test("non-member cannot rename org", async () => { - const update = getUpdateHandler(); - const org = await accountsDb.createOrg({ name: "Outsider Rename" }); - await accountsDb.addMember(org.id, testIdentity.id, "owner"); - - const outsider = await accountsDb.createIdentity({ - email: "org-outsider@example.com", - name: "Org Outsider", - }); - - await expect( - update({ id: org.id, name: "Forbidden" }, createContext(outsider)), - ).rejects.toThrow("Only owners and admins can update the organization"); - }); - - test("two orgs can share the same name (no unique constraint)", async () => { - const update = getUpdateHandler(); - - const orgA = await accountsDb.createOrg({ name: "Shared Name Source" }); - await accountsDb.addMember(orgA.id, testIdentity.id, "owner"); - const orgB = await accountsDb.createOrg({ name: "Other Org" }); - await accountsDb.addMember(orgB.id, testIdentity.id, "owner"); - - const result = (await update( - { id: orgB.id, name: "Shared Name Source" }, - createContext(testIdentity), - )) as { name: string }; - - expect(result.name).toBe("Shared Name Source"); - const fetched = await accountsDb.getOrg(orgA.id); - expect(fetched?.name).toBe("Shared Name Source"); - }); - - test("renaming a non-existent org returns FORBIDDEN (no membership)", async () => { - // Membership check runs before the org lookup, so a missing org id - // surfaces as a FORBIDDEN rather than NOT_FOUND for callers who are - // not members. This matches the rest of the codebase's defense-in-depth. - const update = getUpdateHandler(); - await expect( - update( - { id: "019d694f-79f6-7595-8faf-b70b01c11f98", name: "Nope" }, - createContext(testIdentity), - ), - ).rejects.toThrow("Only owners and admins can update the organization"); - }); -}); diff --git a/packages/server/rpc/accounts/org.ts b/packages/server/rpc/accounts/org.ts deleted file mode 100644 index 2a93a78..0000000 --- a/packages/server/rpc/accounts/org.ts +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Accounts RPC org methods. - * - * Implements: - * - org.create: Create a new organization (caller becomes owner) - * - org.list: List organizations for the current identity - * - org.get: Get organization by ID - * - org.update: Update organization name - * - org.delete: Delete an organization - */ -import type { Org } from "@memory.build/accounts"; -import type { - OrgCreateParams, - OrgDeleteParams, - OrgGetParams, - OrgListParams, - OrgResponse, - OrgUpdateParams, -} from "@memory.build/protocol/accounts/org"; -import { - orgCreateParams, - orgDeleteParams, - orgGetParams, - orgListParams, - orgUpdateParams, -} from "@memory.build/protocol/accounts/org"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -/** - * Convert an Org to a serializable response. - */ -function toOrgResponse(org: Org): OrgResponse { - return { - id: org.id, - slug: org.slug, - name: org.name, - createdAt: org.createdAt.toISOString(), - updatedAt: org.updatedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * org.create - Create a new organization. - * The authenticated identity automatically becomes the owner. - */ -async function orgCreate( - params: OrgCreateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Create org and add creator as owner in a transaction - const org = await db.withTransaction(async (txDb) => { - const newOrg = await txDb.createOrg({ - name: params.name, - }); - - // Add creator as owner - await txDb.addMember(newOrg.id, identity.id, "owner"); - - return newOrg; - }); - - return toOrgResponse(org); -} - -/** - * org.list - List organizations for the current identity. - */ -async function orgList( - _params: OrgListParams, - context: HandlerContext, -): Promise<{ orgs: OrgResponse[] }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - const orgs = await db.listOrgsByIdentity(identity.id); - return { orgs: orgs.map(toOrgResponse) }; -} - -/** - * org.get - Get organization by ID. - */ -async function orgGet( - params: OrgGetParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is a member of the org - const member = await db.getMember(params.id, identity.id); - if (!member) { - throw new AppError("FORBIDDEN", "Not a member of this organization"); - } - - const org = await db.getOrg(params.id); - if (!org) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - return toOrgResponse(org); -} - -/** - * org.update - Update organization name. - * Requires owner or admin role. - */ -async function orgUpdate( - params: OrgUpdateParams, - context: HandlerContext, -): Promise { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller has admin or owner role - const member = await db.getMember(params.id, identity.id); - if (!member || (member.role !== "owner" && member.role !== "admin")) { - throw new AppError( - "FORBIDDEN", - "Only owners and admins can update the organization", - ); - } - - const updated = await db.updateOrg(params.id, { - name: params.name, - }); - - if (!updated) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - const org = await db.getOrg(params.id); - if (!org) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - return toOrgResponse(org); -} - -/** - * org.delete - Delete an organization. - * Requires owner role. Refuses if: - * - The org has any engines (delete engines first) - * - It's the caller's only owned org - */ -async function orgDelete( - params: OrgDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - // Check if caller is an owner - const member = await db.getMember(params.id, identity.id); - if (!member || member.role !== "owner") { - throw new AppError("FORBIDDEN", "Only owners can delete the organization"); - } - - // Refuse if the org still has engines - const engines = await db.listEnginesByOrg(params.id); - if (engines.length > 0) { - throw new AppError( - "CONFLICT", - "Cannot delete organization with engines. Delete all engines first.", - ); - } - - // Refuse if this is the caller's only owned org - const ownedCount = await db.countOwnedOrgs(identity.id); - if (ownedCount <= 1) { - throw new AppError("CONFLICT", "Cannot delete your only organization."); - } - - const deleted = await db.deleteOrg(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `Organization not found: ${params.id}`); - } - - return { deleted }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the org methods registry. - */ -export const orgMethods = buildRegistry() - .register("org.create", orgCreateParams, orgCreate) - .register("org.list", orgListParams, orgList) - .register("org.get", orgGetParams, orgGet) - .register("org.update", orgUpdateParams, orgUpdate) - .register("org.delete", orgDeleteParams, orgDelete) - .build(); diff --git a/packages/server/rpc/accounts/schemas.test.ts b/packages/server/rpc/accounts/schemas.test.ts deleted file mode 100644 index 04f8287..0000000 --- a/packages/server/rpc/accounts/schemas.test.ts +++ /dev/null @@ -1,513 +0,0 @@ -/** - * Tests for Accounts RPC schemas. - */ -import { describe, expect, test } from "bun:test"; -import { - emailSchema, - engineCreateSchema, - engineGetSchema, - engineListSchema, - engineSetupAccessSchema, - engineStatusSchema, - engineUpdateSchema, - identityGetByEmailSchema, - invitationAcceptSchema, - invitationCreateSchema, - invitationListSchema, - invitationRevokeSchema, - meGetSchema, - nameSchema, - orgCreateSchema, - orgDeleteSchema, - orgGetSchema, - orgListSchema, - orgMemberAddSchema, - orgMemberListSchema, - orgMemberRemoveSchema, - orgMemberUpdateRoleSchema, - orgRoleSchema, - orgUpdateSchema, - uuidv7Schema, -} from "./schemas"; - -// ============================================================================= -// Common Schema Tests -// ============================================================================= - -describe("uuidv7Schema", () => { - test("accepts valid UUIDv7", () => { - const validUuids = [ - "019d694f-79f6-7595-8faf-b70b01c11f98", - "019d694f-79f6-7595-9faf-b70b01c11f98", - "019d694f-79f6-7595-afaf-b70b01c11f98", - "019d694f-79f6-7595-bfaf-b70b01c11f98", - ]; - for (const uuid of validUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(true); - } - }); - - test("rejects invalid UUIDs", () => { - const invalidUuids = [ - "not-a-uuid", - "019d694f-79f6-4595-8faf-b70b01c11f98", // v4 not v7 - "019d694f-79f6-7595-0faf-b70b01c11f98", // invalid variant - "019d694f79f675958fafb70b01c11f98", // no dashes - "", - ]; - for (const uuid of invalidUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(false); - } - }); -}); - -describe("emailSchema", () => { - test("accepts valid emails", () => { - const validEmails = [ - "user@example.com", - "user.name@example.com", - "user+tag@example.com", - "user@subdomain.example.com", - ]; - for (const email of validEmails) { - expect(emailSchema.safeParse(email).success).toBe(true); - } - }); - - test("rejects invalid emails", () => { - const invalidEmails = ["not-an-email", "user@", "@example.com", ""]; - for (const email of invalidEmails) { - expect(emailSchema.safeParse(email).success).toBe(false); - } - }); -}); - -describe("nameSchema", () => { - test("accepts valid names", () => { - const validNames = ["a", "John Doe", "My Organization", "a".repeat(100)]; - for (const name of validNames) { - expect(nameSchema.safeParse(name).success).toBe(true); - } - }); - - test("rejects invalid names", () => { - const invalidNames = ["", "a".repeat(101)]; - for (const name of invalidNames) { - expect(nameSchema.safeParse(name).success).toBe(false); - } - }); -}); - -describe("orgRoleSchema", () => { - test("accepts valid roles", () => { - const validRoles = ["owner", "admin", "member"]; - for (const role of validRoles) { - expect(orgRoleSchema.safeParse(role).success).toBe(true); - } - }); - - test("rejects invalid roles", () => { - const invalidRoles = ["superuser", "guest", "", "OWNER"]; - for (const role of invalidRoles) { - expect(orgRoleSchema.safeParse(role).success).toBe(false); - } - }); -}); - -describe("engineStatusSchema", () => { - test("accepts valid statuses", () => { - const validStatuses = ["active", "suspended", "deleted"]; - for (const status of validStatuses) { - expect(engineStatusSchema.safeParse(status).success).toBe(true); - } - }); - - test("rejects invalid statuses", () => { - const invalidStatuses = ["pending", "inactive", "", "ACTIVE"]; - for (const status of invalidStatuses) { - expect(engineStatusSchema.safeParse(status).success).toBe(false); - } - }); -}); - -// ============================================================================= -// Me Schema Tests -// ============================================================================= - -describe("identityGetByEmailSchema", () => { - test("accepts valid email", () => { - const result = identityGetByEmailSchema.safeParse({ - email: "user@example.com", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid email", () => { - const result = identityGetByEmailSchema.safeParse({ - email: "not-an-email", - }); - expect(result.success).toBe(false); - }); - - test("rejects missing email", () => { - const result = identityGetByEmailSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -describe("meGetSchema", () => { - test("accepts empty params", () => { - const result = meGetSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("ignores extra params", () => { - const result = meGetSchema.safeParse({ extra: "ignored" }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Org Schema Tests -// ============================================================================= - -describe("orgCreateSchema", () => { - test("accepts valid params", () => { - const result = orgCreateSchema.safeParse({ - name: "My Organization", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = orgCreateSchema.safeParse({ - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("orgListSchema", () => { - test("accepts empty params", () => { - const result = orgListSchema.safeParse({}); - expect(result.success).toBe(true); - }); -}); - -describe("orgGetSchema", () => { - test("accepts valid UUID", () => { - const result = orgGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid UUID", () => { - const result = orgGetSchema.safeParse({ - id: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); -}); - -describe("orgUpdateSchema", () => { - test("accepts id with name update", () => { - const result = orgUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "New Name", - }); - expect(result.success).toBe(true); - }); - - test("accepts id only (no-op)", () => { - const result = orgUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("orgDeleteSchema", () => { - test("accepts valid UUID", () => { - const result = orgDeleteSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Org Member Schema Tests -// ============================================================================= - -describe("orgMemberListSchema", () => { - test("accepts valid orgId", () => { - const result = orgMemberListSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("orgMemberAddSchema", () => { - test("accepts valid params", () => { - const result = orgMemberAddSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "member", - }); - expect(result.success).toBe(true); - }); - - test("accepts all roles", () => { - for (const role of ["owner", "admin", "member"]) { - const result = orgMemberAddSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role, - }); - expect(result.success).toBe(true); - } - }); - - test("rejects invalid role", () => { - const result = orgMemberAddSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "superuser", - }); - expect(result.success).toBe(false); - }); -}); - -describe("orgMemberRemoveSchema", () => { - test("accepts valid params", () => { - const result = orgMemberRemoveSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - }); - expect(result.success).toBe(true); - }); -}); - -describe("orgMemberUpdateRoleSchema", () => { - test("accepts valid params", () => { - const result = orgMemberUpdateRoleSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - role: "admin", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Engine Schema Tests -// ============================================================================= - -describe("engineCreateSchema", () => { - test("accepts valid params", () => { - const result = engineCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "My Engine", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = engineCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("engineListSchema", () => { - test("accepts valid orgId", () => { - const result = engineListSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("engineGetSchema", () => { - test("accepts valid UUID", () => { - const result = engineGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("engineUpdateSchema", () => { - test("accepts name update", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "New Engine Name", - }); - expect(result.success).toBe(true); - }); - - test("accepts status update", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - status: "suspended", - }); - expect(result.success).toBe(true); - }); - - test("accepts id only (no-op)", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid status", () => { - const result = engineUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - status: "invalid", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Invitation Schema Tests -// ============================================================================= - -describe("invitationCreateSchema", () => { - test("accepts minimal params", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "member", - }); - expect(result.success).toBe(true); - }); - - test("accepts with expiresInDays", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "admin", - expiresInDays: 14, - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid email", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "not-an-email", - role: "member", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid role", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "superuser", - }); - expect(result.success).toBe(false); - }); - - test("rejects expiresInDays < 1", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "member", - expiresInDays: 0, - }); - expect(result.success).toBe(false); - }); - - test("rejects expiresInDays > 30", () => { - const result = invitationCreateSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - email: "user@example.com", - role: "member", - expiresInDays: 31, - }); - expect(result.success).toBe(false); - }); -}); - -describe("invitationListSchema", () => { - test("accepts valid orgId", () => { - const result = invitationListSchema.safeParse({ - orgId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("invitationRevokeSchema", () => { - test("accepts valid UUID", () => { - const result = invitationRevokeSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("invitationAcceptSchema", () => { - test("accepts valid token", () => { - const result = invitationAcceptSchema.safeParse({ - token: "some-invitation-token-here", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty token", () => { - const result = invitationAcceptSchema.safeParse({ - token: "", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Engine SetupAccess Schema Tests -// ============================================================================= - -describe("engineSetupAccessSchema", () => { - test("accepts valid engineId only", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("accepts engineId with apiKeyName", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "019d694f-79f6-7595-8faf-b70b01c11f98", - apiKeyName: "my-cli-key", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing engineId", () => { - const result = engineSetupAccessSchema.safeParse({}); - expect(result.success).toBe(false); - }); - - test("rejects invalid UUID", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); - - test("rejects empty apiKeyName", () => { - const result = engineSetupAccessSchema.safeParse({ - engineId: "019d694f-79f6-7595-8faf-b70b01c11f98", - apiKeyName: "", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/server/rpc/accounts/schemas.ts b/packages/server/rpc/accounts/schemas.ts deleted file mode 100644 index 5c3b7b1..0000000 --- a/packages/server/rpc/accounts/schemas.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Re-export accounts schemas from @memory.build/protocol. - * - * @deprecated Import directly from @memory.build/protocol/accounts instead. - */ - -export { - type EngineCreateParams, - type EngineGetParams, - type EngineListParams, - type EngineSetupAccessParams, - type EngineUpdateParams, - // Engine params - engineCreateParams as engineCreateSchema, - engineGetParams as engineGetSchema, - engineListParams as engineListSchema, - engineSetupAccessParams as engineSetupAccessSchema, - engineUpdateParams as engineUpdateSchema, -} from "@memory.build/protocol/accounts/engine"; - -export { - type IdentityGetByEmailParams, - identityGetByEmailParams as identityGetByEmailSchema, - type MeGetParams, - // Identity params - meGetParams as meGetSchema, -} from "@memory.build/protocol/accounts/identity"; -export { - type InvitationAcceptParams, - type InvitationCreateParams, - type InvitationListParams, - type InvitationRevokeParams, - invitationAcceptParams as invitationAcceptSchema, - // Invitation params - invitationCreateParams as invitationCreateSchema, - invitationListParams as invitationListSchema, - invitationRevokeParams as invitationRevokeSchema, -} from "@memory.build/protocol/accounts/invitation"; - -export { - type OrgCreateParams, - type OrgDeleteParams, - type OrgGetParams, - type OrgListParams, - type OrgUpdateParams, - // Org params - orgCreateParams as orgCreateSchema, - orgDeleteParams as orgDeleteSchema, - orgGetParams as orgGetSchema, - orgListParams as orgListSchema, - orgUpdateParams as orgUpdateSchema, -} from "@memory.build/protocol/accounts/org"; - -export { - type OrgMemberAddParams, - type OrgMemberListParams, - type OrgMemberRemoveParams, - type OrgMemberUpdateRoleParams, - orgMemberAddParams as orgMemberAddSchema, - // Org member params - orgMemberListParams as orgMemberListSchema, - orgMemberRemoveParams as orgMemberRemoveSchema, - orgMemberUpdateRoleParams as orgMemberUpdateRoleSchema, -} from "@memory.build/protocol/accounts/org-member"; -export { - type SessionRevokeParams, - // Session params - sessionRevokeParams as sessionRevokeSchema, -} from "@memory.build/protocol/accounts/session"; -export { - // Fields - emailSchema, - engineStatusSchema, - nameSchema, - orgRoleSchema, - uuidv7Schema, -} from "@memory.build/protocol/fields"; diff --git a/packages/server/rpc/accounts/session.ts b/packages/server/rpc/accounts/session.ts deleted file mode 100644 index 63dbc0c..0000000 --- a/packages/server/rpc/accounts/session.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Accounts RPC session methods. - * - * Implements: - * - session.revoke: Revoke the current session (logout) - */ -import type { SessionRevokeParams } from "@memory.build/protocol/accounts/session"; -import { sessionRevokeParams } from "@memory.build/protocol/accounts/session"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { type AccountsRpcContext, assertAccountsRpcContext } from "./types"; - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * session.revoke - Revoke the current session (logout). - * - * This deletes all sessions for the authenticated identity, - * effectively logging the user out of all devices. - * - * TODO: If we need per-session revocation, pass sessionId through context. - */ -async function sessionRevoke( - _params: SessionRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertAccountsRpcContext(context); - const { db, identity } = context as AccountsRpcContext; - - const count = await db.deleteSessionsByIdentity(identity.id); - return { revoked: count > 0 }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the session methods registry. - */ -export const sessionMethods = buildRegistry() - .register("session.revoke", sessionRevokeParams, sessionRevoke) - .build(); diff --git a/packages/server/rpc/accounts/types.ts b/packages/server/rpc/accounts/types.ts deleted file mode 100644 index ab6f494..0000000 --- a/packages/server/rpc/accounts/types.ts +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Accounts RPC context types. - * - * Extends the base HandlerContext with accounts-specific fields. - */ -import type { AccountsDB } from "@memory.build/accounts"; -import type { SQL } from "bun"; -import type { Identity } from "../../middleware/authenticate"; -import type { HandlerContext } from "../types"; - -/** - * Accounts RPC handler context. - * - * Provides access to: - * - `db`: AccountsDB instance for accounts operations - * - `identity`: The authenticated identity (from OAuth session) - * - `engineSql`: SQL connection to the engine database (for schema provisioning) - * - `serverVersion`: Application version string (for migration tracking) - * - * Authentication middleware populates these fields via OAuth session validation. - */ -export interface AccountsRpcContext extends HandlerContext { - /** AccountsDB instance */ - db: AccountsDB; - /** Authenticated identity */ - identity: Identity; - /** SQL connection to the engine database */ - engineSql: SQL; - /** Application version string */ - serverVersion: string; -} - -/** - * Type guard to check if context has accounts fields. - */ -export function isAccountsRpcContext( - ctx: HandlerContext, -): ctx is AccountsRpcContext { - return ( - "db" in ctx && - typeof ctx.db === "object" && - ctx.db !== null && - "identity" in ctx && - typeof ctx.identity === "object" && - ctx.identity !== null && - "id" in ctx.identity && - "email" in ctx.identity && - "engineSql" in ctx && - typeof ctx.engineSql === "function" && - "serverVersion" in ctx && - typeof ctx.serverVersion === "string" - ); -} - -/** - * Assert that context is an AccountsRpcContext, throwing if not. - */ -export function assertAccountsRpcContext( - ctx: HandlerContext, -): asserts ctx is AccountsRpcContext { - if (!isAccountsRpcContext(ctx)) { - throw new Error( - "Accounts context not initialized (authentication required)", - ); - } -} diff --git a/packages/server/rpc/core-error.ts b/packages/server/rpc/core-error.ts new file mode 100644 index 0000000..89a6fca --- /dev/null +++ b/packages/server/rpc/core-error.ts @@ -0,0 +1,39 @@ +/** + * Map core control-plane SQL constraint violations to AppErrors, shared by the + * space management and user RPC handlers. + */ +import { AppError } from "./errors"; + +/** + * Unique → CONFLICT (e.g. duplicate name); the last-admin guard (ME001) → + * LAST_ADMIN; foreign-key / check / bad-input → VALIDATION_ERROR; everything + * else propagates. + */ +function mapCoreError(e: unknown): never { + const code = (e as { code?: string }).code; + if (code === "23505") { + throw new AppError("CONFLICT", "A record with that name already exists"); + } + if (code === "ME001") { + throw new AppError( + "LAST_ADMIN", + "This would leave the space without an admin — promote another principal to admin first.", + ); + } + if (code === "23503" || code === "23514" || code === "22P02") { + throw new AppError( + "VALIDATION_ERROR", + e instanceof Error ? e.message : "Invalid parameter", + ); + } + throw e instanceof Error ? e : new Error(String(e)); +} + +/** Run a coreStore call, mapping constraint violations to AppErrors. */ +export async function guardCore(fn: () => Promise): Promise { + try { + return await fn(); + } catch (e) { + return mapCoreError(e); + } +} diff --git a/packages/server/rpc/engine/api-key.ts b/packages/server/rpc/engine/api-key.ts deleted file mode 100644 index 7bc946e..0000000 --- a/packages/server/rpc/engine/api-key.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Engine RPC API key methods. - * - * Implements: - * - apiKey.create: Create a new API key (returns raw key once) - * - apiKey.get: Get API key metadata by ID - * - apiKey.list: List API keys for a user - * - apiKey.revoke: Revoke an API key - * - apiKey.delete: Permanently delete an API key - */ -import type { ApiKey } from "@memory.build/engine"; -import type { - ApiKeyCreateParams, - ApiKeyCreateResult, - ApiKeyDeleteParams, - ApiKeyGetParams, - ApiKeyListParams, - ApiKeyResponse, - ApiKeyRevokeParams, -} from "@memory.build/protocol/engine/api-key"; -import { - apiKeyCreateParams, - apiKeyDeleteParams, - apiKeyGetParams, - apiKeyListParams, - apiKeyRevokeParams, -} from "@memory.build/protocol/engine/api-key"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert an ApiKey to a serializable response. - */ -function toApiKeyResponse(apiKey: ApiKey): ApiKeyResponse { - return { - id: apiKey.id, - userId: apiKey.userId, - lookupId: apiKey.lookupId, - name: apiKey.name, - expiresAt: apiKey.expiresAt?.toISOString() ?? null, - createdAt: apiKey.createdAt.toISOString(), - revokedAt: apiKey.revokedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * apiKey.create - Create a new API key. - * Returns the raw key once - it cannot be retrieved again. - */ -async function apiKeyCreate( - params: ApiKeyCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const result = await db.createApiKey({ - userId: params.userId, - name: params.name, - expiresAt: params.expiresAt ? new Date(params.expiresAt) : undefined, - }); - - return { - apiKey: toApiKeyResponse(result.apiKey), - rawKey: result.rawKey, - }; -} - -/** - * apiKey.get - Get API key metadata by ID. - */ -async function apiKeyGet( - params: ApiKeyGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const apiKey = await db.getApiKey(params.id); - if (!apiKey) { - throw new AppError("NOT_FOUND", `API key not found: ${params.id}`); - } - - return toApiKeyResponse(apiKey); -} - -/** - * apiKey.list - List API keys for a user. - */ -async function apiKeyList( - params: ApiKeyListParams, - context: HandlerContext, -): Promise<{ apiKeys: ApiKeyResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const apiKeys = await db.listApiKeys(params.userId); - return { apiKeys: apiKeys.map(toApiKeyResponse) }; -} - -/** - * apiKey.revoke - Revoke an API key (soft delete). - */ -async function apiKeyRevoke( - params: ApiKeyRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const revoked = await db.revokeApiKey(params.id); - if (!revoked) { - throw new AppError( - "NOT_FOUND", - `API key not found or already revoked: ${params.id}`, - ); - } - - return { revoked }; -} - -/** - * apiKey.delete - Permanently delete an API key. - */ -async function apiKeyDelete( - params: ApiKeyDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const deleted = await db.deleteApiKey(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `API key not found: ${params.id}`); - } - - return { deleted }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the API key methods registry. - */ -export const apiKeyMethods = buildRegistry() - .register("apiKey.create", apiKeyCreateParams, apiKeyCreate) - .register("apiKey.get", apiKeyGetParams, apiKeyGet) - .register("apiKey.list", apiKeyListParams, apiKeyList) - .register("apiKey.revoke", apiKeyRevokeParams, apiKeyRevoke) - .register("apiKey.delete", apiKeyDeleteParams, apiKeyDelete) - .build(); diff --git a/packages/server/rpc/engine/grant.test.ts b/packages/server/rpc/engine/grant.test.ts deleted file mode 100644 index c2335f8..0000000 --- a/packages/server/rpc/engine/grant.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -/** - * Unit tests for grant RPC handlers. - * - * Uses mocked EngineDB to test handler logic in isolation. - * Verifies that responses include userName from JOINed user data. - */ -import { describe, expect, mock, test } from "bun:test"; -import type { HandlerContext } from "../types"; -import { grantMethods } from "./grant"; - -const TEST_UUID = "019d694f-79f6-7595-8faf-b70b01c11f98"; -const TEST_UUID_2 = "019d694f-79f6-7595-8faf-b70b01c11f99"; - -function createMockContext( - dbOverrides: Record = {}, -): HandlerContext { - return { - request: new Request("http://localhost"), - db: { - grantTreeAccess: mock(() => Promise.resolve()), - revokeTreeAccess: mock(() => Promise.resolve(false)), - listTreeGrants: mock(() => Promise.resolve([])), - getTreeGrant: mock(() => Promise.resolve(null)), - checkTreeAccess: mock(() => Promise.resolve(false)), - ...dbOverrides, - }, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; -} - -// ============================================================================= -// grant.create -// ============================================================================= - -describe("grant.create", () => { - test("calls grantTreeAccess and returns { created: true }", async () => { - const handler = grantMethods.get("grant.create")?.handler; - if (!handler) throw new Error("grant.create handler not found"); - - const grantTreeAccess = mock(() => Promise.resolve()); - const context = createMockContext({ grantTreeAccess }); - - const result = await handler( - { - userId: TEST_UUID, - treePath: "work.projects", - actions: ["read", "create"], - withGrantOption: false, - }, - context, - ); - - expect(result).toEqual({ created: true }); - expect(grantTreeAccess).toHaveBeenCalledTimes(1); - }); -}); - -// ============================================================================= -// grant.list -// ============================================================================= - -describe("grant.list", () => { - test("returns grants with userName", async () => { - const handler = grantMethods.get("grant.list")?.handler; - if (!handler) throw new Error("grant.list handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const listTreeGrants = mock(() => - Promise.resolve([ - { - id: TEST_UUID_2, - userId: TEST_UUID, - userName: "alice", - treePath: "work.projects", - actions: ["read"], - grantedBy: null, - withGrantOption: false, - createdAt: now, - }, - ]), - ); - const context = createMockContext({ listTreeGrants }); - - const result = (await handler({}, context)) as { - grants: Array<{ - userId: string; - userName: string; - treePath: string; - }>; - }; - - expect(result.grants).toHaveLength(1); - expect(result.grants[0]?.userName).toBe("alice"); - expect(result.grants[0]?.userId).toBe(TEST_UUID); - expect(result.grants[0]?.treePath).toBe("work.projects"); - }); - - test("returns empty list when no grants", async () => { - const handler = grantMethods.get("grant.list")?.handler; - if (!handler) throw new Error("grant.list handler not found"); - - const context = createMockContext(); - - const result = (await handler({}, context)) as { - grants: unknown[]; - }; - - expect(result.grants).toHaveLength(0); - }); - - test("passes userId filter when provided", async () => { - const handler = grantMethods.get("grant.list")?.handler; - if (!handler) throw new Error("grant.list handler not found"); - - const listTreeGrants = mock(() => Promise.resolve([])); - const context = createMockContext({ listTreeGrants }); - - await handler({ userId: TEST_UUID }, context); - - expect(listTreeGrants).toHaveBeenCalledWith(TEST_UUID); - }); -}); - -// ============================================================================= -// grant.get -// ============================================================================= - -describe("grant.get", () => { - test("returns grant with userName when found", async () => { - const handler = grantMethods.get("grant.get")?.handler; - if (!handler) throw new Error("grant.get handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const getTreeGrant = mock(() => - Promise.resolve({ - id: TEST_UUID_2, - userId: TEST_UUID, - userName: "alice", - treePath: "work", - actions: ["read", "create"], - grantedBy: null, - withGrantOption: true, - createdAt: now, - }), - ); - const context = createMockContext({ getTreeGrant }); - - const result = (await handler( - { userId: TEST_UUID, treePath: "work" }, - context, - )) as { userName: string; withGrantOption: boolean }; - - expect(result.userName).toBe("alice"); - expect(result.withGrantOption).toBe(true); - }); - - test("throws NOT_FOUND when grant does not exist", async () => { - const handler = grantMethods.get("grant.get")?.handler; - if (!handler) throw new Error("grant.get handler not found"); - - const context = createMockContext({ - getTreeGrant: mock(() => Promise.resolve(null)), - }); - - try { - await handler({ userId: TEST_UUID, treePath: "work" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -// ============================================================================= -// grant.revoke -// ============================================================================= - -describe("grant.revoke", () => { - test("returns { revoked: true } when found", async () => { - const handler = grantMethods.get("grant.revoke")?.handler; - if (!handler) throw new Error("grant.revoke handler not found"); - - const context = createMockContext({ - revokeTreeAccess: mock(() => Promise.resolve(true)), - }); - - const result = await handler( - { userId: TEST_UUID, treePath: "work" }, - context, - ); - expect(result).toEqual({ revoked: true }); - }); - - test("throws NOT_FOUND when grant does not exist", async () => { - const handler = grantMethods.get("grant.revoke")?.handler; - if (!handler) throw new Error("grant.revoke handler not found"); - - const context = createMockContext({ - revokeTreeAccess: mock(() => Promise.resolve(false)), - }); - - try { - await handler({ userId: TEST_UUID, treePath: "work" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -// ============================================================================= -// grant.check -// ============================================================================= - -describe("grant.check", () => { - test("returns { allowed: true } when access granted", async () => { - const handler = grantMethods.get("grant.check")?.handler; - if (!handler) throw new Error("grant.check handler not found"); - - const context = createMockContext({ - checkTreeAccess: mock(() => Promise.resolve(true)), - }); - - const result = await handler( - { userId: TEST_UUID, treePath: "work", action: "read" }, - context, - ); - expect(result).toEqual({ allowed: true }); - }); - - test("returns { allowed: false } when access denied", async () => { - const handler = grantMethods.get("grant.check")?.handler; - if (!handler) throw new Error("grant.check handler not found"); - - const context = createMockContext({ - checkTreeAccess: mock(() => Promise.resolve(false)), - }); - - const result = await handler( - { userId: TEST_UUID, treePath: "work", action: "update" }, - context, - ); - expect(result).toEqual({ allowed: false }); - }); -}); diff --git a/packages/server/rpc/engine/grant.ts b/packages/server/rpc/engine/grant.ts deleted file mode 100644 index ba44036..0000000 --- a/packages/server/rpc/engine/grant.ts +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Engine RPC grant methods. - * - * Implements: - * - grant.create: Grant tree access to a user - * - grant.list: List grants (optionally filter by user) - * - grant.get: Get a specific grant - * - grant.revoke: Revoke tree access - * - grant.check: Check if user has access to a tree path for an action - */ -import type { TreeGrant } from "@memory.build/engine"; -import type { - GrantCheckParams, - GrantCreateParams, - GrantGetParams, - GrantListParams, - GrantResponse, - GrantRevokeParams, -} from "@memory.build/protocol/engine/grant"; -import { - grantCheckParams, - grantCreateParams, - grantGetParams, - grantListParams, - grantRevokeParams, -} from "@memory.build/protocol/engine/grant"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a TreeGrant to a serializable response. - */ -function toGrantResponse(grant: TreeGrant): GrantResponse { - return { - id: grant.id, - userId: grant.userId, - userName: grant.userName, - treePath: grant.treePath, - actions: grant.actions, - grantedBy: grant.grantedBy, - withGrantOption: grant.withGrantOption, - createdAt: grant.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * grant.create - Grant tree access to a user. - */ -async function grantCreate( - params: GrantCreateParams, - context: HandlerContext, -): Promise<{ created: boolean }> { - assertEngineContext(context); - const { db, userId } = context as EngineContext; - - await db.grantTreeAccess({ - userId: params.userId, - treePath: params.treePath, - actions: params.actions, - grantedBy: userId, - withGrantOption: params.withGrantOption, - }); - - return { created: true }; -} - -/** - * grant.list - List grants. - */ -async function grantList( - params: GrantListParams, - context: HandlerContext, -): Promise<{ grants: GrantResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const grants = await db.listTreeGrants(params.userId); - return { grants: grants.map(toGrantResponse) }; -} - -/** - * grant.get - Get a specific grant. - */ -async function grantGet( - params: GrantGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const grant = await db.getTreeGrant(params.userId, params.treePath); - if (!grant) { - throw new AppError( - "NOT_FOUND", - `Grant not found for user ${params.userId} at path ${params.treePath}`, - ); - } - - return toGrantResponse(grant); -} - -/** - * grant.revoke - Revoke tree access. - */ -async function grantRevoke( - params: GrantRevokeParams, - context: HandlerContext, -): Promise<{ revoked: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const revoked = await db.revokeTreeAccess(params.userId, params.treePath); - if (!revoked) { - throw new AppError( - "NOT_FOUND", - `Grant not found for user ${params.userId} at path ${params.treePath}`, - ); - } - - return { revoked }; -} - -/** - * grant.check - Check if user has access to a tree path for an action. - */ -async function grantCheck( - params: GrantCheckParams, - context: HandlerContext, -): Promise<{ allowed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const allowed = await db.checkTreeAccess( - params.userId, - params.treePath, - params.action, - ); - - return { allowed }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the grant methods registry. - */ -export const grantMethods = buildRegistry() - .register("grant.create", grantCreateParams, grantCreate) - .register("grant.list", grantListParams, grantList) - .register("grant.get", grantGetParams, grantGet) - .register("grant.revoke", grantRevokeParams, grantRevoke) - .register("grant.check", grantCheckParams, grantCheck) - .build(); diff --git a/packages/server/rpc/engine/index.ts b/packages/server/rpc/engine/index.ts deleted file mode 100644 index f4b2b28..0000000 --- a/packages/server/rpc/engine/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { buildRegistry } from "../registry"; -import { apiKeyMethods } from "./api-key"; -import { grantMethods } from "./grant"; -import { memoryMethods } from "./memory"; -import { ownerMethods } from "./owner"; -import { roleMethods } from "./role"; -import { userMethods } from "./user"; - -/** - * Engine RPC method registry. - * - * Memory methods (chunk 3): - * - memory.create, memory.batchCreate, memory.get, memory.update, memory.delete - * - memory.search, memory.tree, memory.move, memory.deleteTree, memory.countTree - * - * User, grant, role methods (chunk 4): - * - user.create, user.get, user.getByName, user.list, user.rename, user.delete - * - grant.create, grant.list, grant.get, grant.revoke, grant.check - * - role.create, role.addMember, role.removeMember, role.listMembers, role.listForUser - * - * API key methods (chunk 5): - * - apiKey.create, apiKey.get, apiKey.list, apiKey.revoke, apiKey.delete - */ -export const engineMethods = buildRegistry() - .merge(memoryMethods) - .merge(userMethods) - .merge(grantMethods) - .merge(ownerMethods) - .merge(roleMethods) - .merge(apiKeyMethods) - .build(); - -// Re-export types for consumers -export type { EngineContext } from "./types"; -export { assertEngineContext, isEngineContext } from "./types"; diff --git a/packages/server/rpc/engine/memory.test.ts b/packages/server/rpc/engine/memory.test.ts deleted file mode 100644 index c4887b3..0000000 --- a/packages/server/rpc/engine/memory.test.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { describe, expect, mock, test } from "bun:test"; -import type { HandlerContext } from "../types"; - -describe("memory.search embedding", () => { - test("throws EMBEDDING_NOT_CONFIGURED when semantic provided without config", async () => { - // Import the handler module to test - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockDb = { - searchMemories: mock(() => - Promise.resolve({ results: [], total: 0, limit: 10 }), - ), - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - // embeddingConfig intentionally omitted - } as unknown as HandlerContext; - - const params = { - semantic: "test query", - }; - - try { - await handler(params, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("EMBEDDING_NOT_CONFIGURED"); - } - }); - - test("throws EMBEDDING_FAILED when embedding generation fails", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockDb = { - searchMemories: mock(() => - Promise.resolve({ results: [], total: 0, limit: 10 }), - ), - }; - - const embeddingConfig = { - provider: "openai" as const, - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - embeddingConfig, - } as unknown as HandlerContext; - - const params = { - semantic: "test query", - }; - - // The actual embedding call will fail because we're using a fake API key - // This tests that errors are properly caught and wrapped - try { - await handler(params, context); - // If embedding somehow succeeds (unlikely with fake key), that's fine too - } catch (error) { - // Should be wrapped in AppError with EMBEDDING_FAILED code - expect((error as { code: string }).code).toBe("EMBEDDING_FAILED"); - } - }); - - test("calls searchMemories without embedding when semantic not provided", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockSearchMemories = mock(() => - Promise.resolve({ - results: [ - { - id: "mem-1", - content: "test", - score: 1.0, - meta: {}, - tree: "", - temporal: null, - hasEmbedding: false, - createdAt: new Date(), - createdBy: null, - updatedAt: null, - }, - ], - total: 1, - limit: 10, - }), - ); - - const mockDb = { - searchMemories: mockSearchMemories, - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - // No embeddingConfig needed when not using semantic - } as unknown as HandlerContext; - - const params = { - fulltext: "test query", - }; - - await handler(params, context); - - // Verify searchMemories was called without embedding - expect(mockSearchMemories).toHaveBeenCalled(); - const calls = mockSearchMemories.mock.calls as unknown as Array< - [{ fulltext?: string; embedding?: number[] }] - >; - expect(calls.length).toBeGreaterThan(0); - const callArgs = calls[0]![0]!; - expect(callArgs.fulltext).toBe("test query"); - expect(callArgs.embedding).toBeUndefined(); - }); - - test("passes grep parameter through to searchMemories", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockSearchMemories = mock(() => - Promise.resolve({ - results: [], - total: 0, - limit: 10, - }), - ); - - const mockDb = { - searchMemories: mockSearchMemories, - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; - - const params = { - fulltext: "database", - grep: "version \\d+", - }; - - await handler(params, context); - - expect(mockSearchMemories).toHaveBeenCalled(); - const calls = mockSearchMemories.mock.calls as unknown as Array< - [{ fulltext?: string; grep?: string }] - >; - expect(calls.length).toBeGreaterThan(0); - const callArgs = calls[0]![0]!; - expect(callArgs.fulltext).toBe("database"); - expect(callArgs.grep).toBe("version \\d+"); - }); - - test("throws VALIDATION_ERROR when grep is used alone", async () => { - const { memoryMethods } = await import("./memory"); - const handler = memoryMethods.get("memory.search")?.handler; - - if (!handler) { - throw new Error("memory.search handler not found"); - } - - const mockDb = { - searchMemories: mock(() => - Promise.resolve({ results: [], total: 0, limit: 10 }), - ), - }; - - const context = { - request: new Request("http://localhost"), - db: mockDb, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; - - try { - await handler({ grep: "ERR-\\d+" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("VALIDATION_ERROR"); - } - }); -}); diff --git a/packages/server/rpc/engine/memory.ts b/packages/server/rpc/engine/memory.ts deleted file mode 100644 index c877b34..0000000 --- a/packages/server/rpc/engine/memory.ts +++ /dev/null @@ -1,414 +0,0 @@ -/** - * Engine RPC memory methods. - * - * Implements: - * - memory.create: Create a single memory - * - memory.batchCreate: Create multiple memories - * - memory.get: Get memory by ID - * - memory.update: Update memory content/meta/tree/temporal - * - memory.delete: Delete memory by ID - * - memory.search: Hybrid semantic + fulltext search - * - memory.tree: Get tree structure with counts - * - memory.move: Move memories from one tree path to another - * - memory.deleteTree: Delete all memories under a tree path - */ -import { generateEmbedding } from "@memory.build/embedding"; -import type { Memory, SearchResult, TreeNode } from "@memory.build/engine"; -import type { - MemoryBatchCreateParams, - MemoryCountTreeParams, - MemoryCreateParams, - MemoryDeleteParams, - MemoryDeleteTreeParams, - MemoryGetParams, - MemoryMoveParams, - MemoryResponse, - MemorySearchParams, - MemorySearchResult, - MemoryTreeParams, - MemoryUpdateParams, -} from "@memory.build/protocol/engine/memory"; -import { - memoryBatchCreateParams, - memoryCountTreeParams, - memoryCreateParams, - memoryDeleteParams, - memoryDeleteTreeParams, - memoryGetParams, - memoryMoveParams, - memorySearchParams, - memoryTreeParams, - memoryUpdateParams, -} from "@memory.build/protocol/engine/memory"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a Memory to a serializable response. - */ -function toMemoryResponse(memory: Memory): MemoryResponse { - return { - id: memory.id, - content: memory.content, - meta: memory.meta, - tree: memory.tree, - temporal: memory.temporal - ? { - start: memory.temporal.start.toISOString(), - end: memory.temporal.end.toISOString(), - } - : null, - hasEmbedding: memory.hasEmbedding, - createdAt: memory.createdAt.toISOString(), - createdBy: memory.createdBy, - updatedAt: memory.updatedAt?.toISOString() ?? null, - }; -} - -/** - * Convert SearchResult to serializable response. - */ -function toSearchResultResponse(result: SearchResult): MemorySearchResult { - return { - results: result.results.map((item) => ({ - ...toMemoryResponse(item), - score: item.score, - })), - total: result.total, - limit: result.limit, - }; -} - -// ============================================================================= -// Helper Functions -// ============================================================================= - -/** - * Parse temporal params into Date objects. - */ -function parseTemporal( - temporal: { start: string; end?: string | null } | null | undefined, -): { start: Date; end?: Date } | undefined { - if (!temporal) return undefined; - return { - start: new Date(temporal.start), - end: temporal.end ? new Date(temporal.end) : undefined, - }; -} - -/** - * Parse temporal filter params into the format expected by engine ops. - */ -function parseTemporalFilter( - temporal: - | { - contains?: string; - overlaps?: { start: string; end: string }; - within?: { start: string; end: string }; - } - | null - | undefined, -): - | { - contains?: Date; - overlaps?: [Date, Date]; - within?: [Date, Date]; - } - | undefined { - if (!temporal) return undefined; - - const result: { - contains?: Date; - overlaps?: [Date, Date]; - within?: [Date, Date]; - } = {}; - - if (temporal.contains) { - result.contains = new Date(temporal.contains); - } - if (temporal.overlaps) { - result.overlaps = [ - new Date(temporal.overlaps.start), - new Date(temporal.overlaps.end), - ]; - } - if (temporal.within) { - result.within = [ - new Date(temporal.within.start), - new Date(temporal.within.end), - ]; - } - - return result; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * memory.create - Create a single memory. - */ -async function memoryCreate( - params: MemoryCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db, userId } = context as EngineContext; - - const memory = await db.createMemory({ - id: params.id ?? undefined, - content: params.content, - meta: params.meta ?? undefined, - tree: params.tree ?? undefined, - temporal: parseTemporal(params.temporal), - createdBy: userId, - }); - - return toMemoryResponse(memory); -} - -/** - * memory.batchCreate - Create multiple memories. - */ -async function memoryBatchCreate( - params: MemoryBatchCreateParams, - context: HandlerContext, -): Promise<{ ids: string[] }> { - assertEngineContext(context); - const { db, userId } = context as EngineContext; - - const ids = await db.batchCreateMemories( - params.memories.map( - (m: { - id?: string | null; - content: string; - meta?: Record | null; - tree?: string | null; - temporal?: { start: string; end?: string | null } | null; - }) => ({ - id: m.id ?? undefined, - content: m.content, - meta: m.meta ?? undefined, - tree: m.tree ?? undefined, - temporal: parseTemporal(m.temporal), - createdBy: userId, - }), - ), - ); - - return { ids }; -} - -/** - * memory.get - Get memory by ID. - */ -async function memoryGet( - params: MemoryGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const memory = await db.getMemory(params.id); - if (!memory) { - throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); - } - - return toMemoryResponse(memory); -} - -/** - * memory.update - Update memory content/meta/tree/temporal. - */ -async function memoryUpdate( - params: MemoryUpdateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const memory = await db.updateMemory(params.id, { - content: params.content ?? undefined, - meta: params.meta ?? undefined, - tree: params.tree ?? undefined, - temporal: parseTemporal(params.temporal), - }); - - if (!memory) { - throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); - } - - return toMemoryResponse(memory); -} - -/** - * memory.delete - Delete memory by ID. - */ -async function memoryDelete( - params: MemoryDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const deleted = await db.deleteMemory(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); - } - - return { deleted }; -} - -/** - * memory.search - Hybrid semantic + fulltext search. - */ -async function memorySearch( - params: MemorySearchParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db, embeddingConfig } = context as EngineContext; - - let embedding: number[] | undefined; - - // Generate embedding for semantic search - if (params.semantic) { - if (!embeddingConfig) { - throw new AppError( - "EMBEDDING_NOT_CONFIGURED", - "Semantic search requires embedding configuration. Set EMBEDDING_API_KEY environment variable.", - ); - } - - try { - const result = await generateEmbedding(params.semantic, embeddingConfig); - embedding = result.embedding; - } catch (error) { - throw new AppError( - "EMBEDDING_FAILED", - `Failed to generate embedding: ${error instanceof Error ? error.message : "Unknown error"}`, - ); - } - } - - // grep alone would cause a full table scan — require at least one indexed criterion - if ( - params.grep && - !params.fulltext && - !params.semantic && - !params.tree && - !params.meta && - !params.temporal - ) { - throw new AppError( - "VALIDATION_ERROR", - "grep cannot be used alone (full table scan). Combine with semantic, fulltext, tree, meta, or temporal.", - ); - } - - const result = await db.searchMemories({ - fulltext: params.fulltext ?? undefined, - embedding, - grep: params.grep ?? undefined, - tree: params.tree ?? undefined, - meta: params.meta ?? undefined, - temporal: parseTemporalFilter(params.temporal), - limit: params.limit, - candidateLimit: params.candidateLimit, - semanticThreshold: params.semanticThreshold ?? undefined, - weights: params.weights ?? undefined, - orderBy: params.orderBy, - }); - - return toSearchResultResponse(result); -} - -/** - * memory.tree - Get tree structure with counts. - */ -async function memoryTree( - params: MemoryTreeParams, - context: HandlerContext, -): Promise<{ nodes: TreeNode[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const nodes = await db.getTree({ - tree: params.tree ?? undefined, - levels: params.levels, - }); - - return { nodes }; -} - -/** - * memory.move - Move memories from one tree path to another. - */ -async function memoryMove( - params: MemoryMoveParams, - context: HandlerContext, -): Promise<{ count: number }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - if (params.dryRun) { - return db.countTree(params.source); - } - - const result = await db.moveTree(params.source, params.destination); - return result; -} - -/** - * memory.deleteTree - Delete all memories under a tree path. - */ -async function memoryDeleteTree( - params: MemoryDeleteTreeParams, - context: HandlerContext, -): Promise<{ count: number }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - if (params.dryRun) { - return db.countTree(params.tree); - } - - const result = await db.deleteTree(params.tree); - return result; -} - -/** - * memory.countTree - Count memories under a tree path. - */ -async function memoryCountTree( - params: MemoryCountTreeParams, - context: HandlerContext, -): Promise<{ count: number }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - return db.countTree(params.tree); -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the memory methods registry. - */ -export const memoryMethods = buildRegistry() - .register("memory.create", memoryCreateParams, memoryCreate) - .register("memory.batchCreate", memoryBatchCreateParams, memoryBatchCreate) - .register("memory.get", memoryGetParams, memoryGet) - .register("memory.update", memoryUpdateParams, memoryUpdate) - .register("memory.delete", memoryDeleteParams, memoryDelete) - .register("memory.search", memorySearchParams, memorySearch) - .register("memory.tree", memoryTreeParams, memoryTree) - .register("memory.move", memoryMoveParams, memoryMove) - .register("memory.deleteTree", memoryDeleteTreeParams, memoryDeleteTree) - .register("memory.countTree", memoryCountTreeParams, memoryCountTree) - .build(); diff --git a/packages/server/rpc/engine/owner.test.ts b/packages/server/rpc/engine/owner.test.ts deleted file mode 100644 index cc8c68a..0000000 --- a/packages/server/rpc/engine/owner.test.ts +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Unit tests for owner RPC handlers. - * - * Uses mocked EngineDB to test handler logic in isolation. - */ -import { describe, expect, mock, test } from "bun:test"; -import type { HandlerContext } from "../types"; -import { ownerMethods } from "./owner"; - -function createMockContext( - dbOverrides: Record = {}, -): HandlerContext { - return { - request: new Request("http://localhost"), - db: { - getUserId: mock(() => "user-123"), - setTreeOwner: mock(() => Promise.resolve()), - getTreeOwner: mock(() => Promise.resolve(null)), - removeTreeOwner: mock(() => Promise.resolve(false)), - listTreeOwners: mock(() => Promise.resolve([])), - ...dbOverrides, - }, - userId: "user-123", - apiKeyId: "key-456", - engine: { - id: "eng-1", - orgId: "org-1", - slug: "test", - name: "Test", - status: "active" as const, - }, - } as unknown as HandlerContext; -} - -describe("owner.set", () => { - test("calls setTreeOwner and returns { set: true }", async () => { - const handler = ownerMethods.get("owner.set")?.handler; - if (!handler) throw new Error("owner.set handler not found"); - - const setTreeOwner = mock(() => Promise.resolve()); - const context = createMockContext({ setTreeOwner }); - - const result = await handler( - { - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - }, - context, - ); - - expect(result).toEqual({ set: true }); - expect(setTreeOwner).toHaveBeenCalledTimes(1); - }); -}); - -describe("owner.get", () => { - test("returns owner when found", async () => { - const handler = ownerMethods.get("owner.get")?.handler; - if (!handler) throw new Error("owner.get handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const getTreeOwner = mock(() => - Promise.resolve({ - treePath: "work.projects", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: "user-123", - createdByName: "admin", - createdAt: now, - }), - ); - const context = createMockContext({ getTreeOwner }); - - const result = (await handler({ treePath: "work.projects" }, context)) as { - treePath: string; - userId: string; - userName: string; - createdBy: string; - createdByName: string; - createdAt: string; - }; - - expect(result.treePath).toBe("work.projects"); - expect(result.userId).toBe("019d694f-79f6-7595-8faf-b70b01c11f98"); - expect(result.userName).toBe("alice"); - expect(result.createdBy).toBe("user-123"); - expect(result.createdByName).toBe("admin"); - expect(result.createdAt).toBe("2026-01-15T00:00:00.000Z"); - }); - - test("throws NOT_FOUND when no owner", async () => { - const handler = ownerMethods.get("owner.get")?.handler; - if (!handler) throw new Error("owner.get handler not found"); - - const context = createMockContext({ - getTreeOwner: mock(() => Promise.resolve(null)), - }); - - try { - await handler({ treePath: "work.projects" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -describe("owner.remove", () => { - test("returns { removed: true } when found", async () => { - const handler = ownerMethods.get("owner.remove")?.handler; - if (!handler) throw new Error("owner.remove handler not found"); - - const context = createMockContext({ - removeTreeOwner: mock(() => Promise.resolve(true)), - }); - - const result = await handler({ treePath: "work.projects" }, context); - expect(result).toEqual({ removed: true }); - }); - - test("throws NOT_FOUND when no owner to remove", async () => { - const handler = ownerMethods.get("owner.remove")?.handler; - if (!handler) throw new Error("owner.remove handler not found"); - - const context = createMockContext({ - removeTreeOwner: mock(() => Promise.resolve(false)), - }); - - try { - await handler({ treePath: "work.projects" }, context); - throw new Error("Expected handler to throw"); - } catch (error) { - expect((error as { code: string }).code).toBe("NOT_FOUND"); - } - }); -}); - -describe("owner.list", () => { - test("returns owners list", async () => { - const handler = ownerMethods.get("owner.list")?.handler; - if (!handler) throw new Error("owner.list handler not found"); - - const now = new Date("2026-01-15T00:00:00.000Z"); - const listTreeOwners = mock(() => - Promise.resolve([ - { - treePath: "work.projects", - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - userName: "alice", - createdBy: "user-123", - createdByName: "admin", - createdAt: now, - }, - ]), - ); - const context = createMockContext({ listTreeOwners }); - - const result = (await handler({}, context)) as { - owners: Array<{ treePath: string }>; - }; - - expect(result.owners).toHaveLength(1); - expect(result.owners[0]?.treePath).toBe("work.projects"); - }); - - test("returns empty list when no owners", async () => { - const handler = ownerMethods.get("owner.list")?.handler; - if (!handler) throw new Error("owner.list handler not found"); - - const context = createMockContext({ - listTreeOwners: mock(() => Promise.resolve([])), - }); - - const result = (await handler({}, context)) as { - owners: Array<{ treePath: string }>; - }; - - expect(result.owners).toHaveLength(0); - }); - - test("passes userId filter when provided", async () => { - const handler = ownerMethods.get("owner.list")?.handler; - if (!handler) throw new Error("owner.list handler not found"); - - const listTreeOwners = mock(() => Promise.resolve([])); - const context = createMockContext({ listTreeOwners }); - - await handler({ userId: "019d694f-79f6-7595-8faf-b70b01c11f98" }, context); - - expect(listTreeOwners).toHaveBeenCalledWith( - "019d694f-79f6-7595-8faf-b70b01c11f98", - ); - }); -}); diff --git a/packages/server/rpc/engine/owner.ts b/packages/server/rpc/engine/owner.ts deleted file mode 100644 index 82ef6c1..0000000 --- a/packages/server/rpc/engine/owner.ts +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Engine RPC owner methods. - * - * Implements: - * - owner.set: Set tree path owner - * - owner.get: Get tree path owner - * - owner.remove: Remove tree path owner - * - owner.list: List tree owners - */ -import type { TreeOwner } from "@memory.build/engine"; -import type { - OwnerGetParams, - OwnerListParams, - OwnerRemoveParams, - OwnerResponse, - OwnerSetParams, -} from "@memory.build/protocol/engine/owner"; -import { - ownerGetParams, - ownerListParams, - ownerRemoveParams, - ownerSetParams, -} from "@memory.build/protocol/engine/owner"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a TreeOwner to a serializable response. - */ -function toOwnerResponse(owner: TreeOwner): OwnerResponse { - return { - treePath: owner.treePath, - userId: owner.userId, - userName: owner.userName, - createdBy: owner.createdBy, - createdByName: owner.createdByName, - createdAt: owner.createdAt.toISOString(), - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * owner.set - Set tree path owner (upserts). - */ -async function ownerSet( - params: OwnerSetParams, - context: HandlerContext, -): Promise<{ set: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - await db.setTreeOwner( - params.userId, - params.treePath, - db.getUserId() ?? undefined, - ); - return { set: true }; -} - -/** - * owner.get - Get tree path owner. - */ -async function ownerGet( - params: OwnerGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const owner = await db.getTreeOwner(params.treePath); - if (!owner) { - throw new AppError("NOT_FOUND", `No owner for path: ${params.treePath}`); - } - - return toOwnerResponse(owner); -} - -/** - * owner.remove - Remove tree path owner. - */ -async function ownerRemove( - params: OwnerRemoveParams, - context: HandlerContext, -): Promise<{ removed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const removed = await db.removeTreeOwner(params.treePath); - if (!removed) { - throw new AppError("NOT_FOUND", `No owner for path: ${params.treePath}`); - } - - return { removed }; -} - -/** - * owner.list - List tree owners. - */ -async function ownerList( - params: OwnerListParams, - context: HandlerContext, -): Promise<{ owners: OwnerResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const owners = await db.listTreeOwners(params.userId); - return { owners: owners.map(toOwnerResponse) }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the owner methods registry. - */ -export const ownerMethods = buildRegistry() - .register("owner.set", ownerSetParams, ownerSet) - .register("owner.get", ownerGetParams, ownerGet) - .register("owner.remove", ownerRemoveParams, ownerRemove) - .register("owner.list", ownerListParams, ownerList) - .build(); diff --git a/packages/server/rpc/engine/role.ts b/packages/server/rpc/engine/role.ts deleted file mode 100644 index 1cdbb0f..0000000 --- a/packages/server/rpc/engine/role.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Engine RPC role methods. - * - * Implements: - * - role.create: Create a role (user with canLogin=false) - * - role.addMember: Add a member to a role - * - role.removeMember: Remove a member from a role - * - role.listMembers: List members of a role - * - role.listForUser: List roles a user belongs to - */ -import type { RoleInfo, RoleMember, User } from "@memory.build/engine"; -import type { - RoleAddMemberParams, - RoleCreateParams, - RoleInfoResponse, - RoleListForUserParams, - RoleListMembersParams, - RoleMemberResponse, - RoleRemoveMemberParams, - RoleResponse, -} from "@memory.build/protocol/engine/role"; -import { - roleAddMemberParams, - roleCreateParams, - roleListForUserParams, - roleListMembersParams, - roleRemoveMemberParams, -} from "@memory.build/protocol/engine/role"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a User (role) to a serializable response. - */ -function toRoleResponse(user: User): RoleResponse { - return { - id: user.id, - name: user.name, - identityId: user.identityId, - createdAt: user.createdAt.toISOString(), - updatedAt: user.updatedAt?.toISOString() ?? null, - }; -} - -/** - * Convert a RoleMember to a serializable response. - */ -function toRoleMemberResponse(member: RoleMember): RoleMemberResponse { - return { - roleId: member.roleId, - memberId: member.memberId, - memberName: member.memberName, - withAdminOption: member.withAdminOption, - createdAt: member.createdAt.toISOString(), - }; -} - -/** - * Convert a RoleInfo to a serializable response. - */ -function toRoleInfoResponse(info: RoleInfo): RoleInfoResponse { - return { - id: info.id, - name: info.name, - withAdminOption: info.withAdminOption, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * role.create - Create a role (user with canLogin=false). - */ -async function roleCreate( - params: RoleCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const role = await db.createRole(params.name, params.identityId); - return toRoleResponse(role); -} - -/** - * role.addMember - Add a member to a role. - */ -async function roleAddMember( - params: RoleAddMemberParams, - context: HandlerContext, -): Promise<{ added: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - try { - await db.addRoleMember( - params.roleId, - params.memberId, - params.withAdminOption, - ); - return { added: true }; - } catch (error) { - // Check for cycle error - if ( - error instanceof Error && - error.message.includes("would create a cycle") - ) { - throw new AppError("VALIDATION_ERROR", error.message); - } - throw error; - } -} - -/** - * role.removeMember - Remove a member from a role. - */ -async function roleRemoveMember( - params: RoleRemoveMemberParams, - context: HandlerContext, -): Promise<{ removed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const removed = await db.removeRoleMember(params.roleId, params.memberId); - if (!removed) { - throw new AppError( - "NOT_FOUND", - `Membership not found for role ${params.roleId} and member ${params.memberId}`, - ); - } - - return { removed }; -} - -/** - * role.listMembers - List members of a role. - */ -async function roleListMembers( - params: RoleListMembersParams, - context: HandlerContext, -): Promise<{ members: RoleMemberResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const members = await db.listRoleMembers(params.roleId); - return { members: members.map(toRoleMemberResponse) }; -} - -/** - * role.listForUser - List roles a user belongs to. - */ -async function roleListForUser( - params: RoleListForUserParams, - context: HandlerContext, -): Promise<{ roles: RoleInfoResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const roles = await db.listRolesForUser(params.userId); - return { roles: roles.map(toRoleInfoResponse) }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the role methods registry. - */ -export const roleMethods = buildRegistry() - .register("role.create", roleCreateParams, roleCreate) - .register("role.addMember", roleAddMemberParams, roleAddMember) - .register("role.removeMember", roleRemoveMemberParams, roleRemoveMember) - .register("role.listMembers", roleListMembersParams, roleListMembers) - .register("role.listForUser", roleListForUserParams, roleListForUser) - .build(); diff --git a/packages/server/rpc/engine/schemas.test.ts b/packages/server/rpc/engine/schemas.test.ts deleted file mode 100644 index 5eff0d5..0000000 --- a/packages/server/rpc/engine/schemas.test.ts +++ /dev/null @@ -1,842 +0,0 @@ -/** - * Tests for Engine RPC schemas. - */ -import { describe, expect, test } from "bun:test"; -import { - apiKeyCreateSchema, - apiKeyDeleteSchema, - apiKeyGetSchema, - apiKeyListSchema, - apiKeyRevokeSchema, - grantCheckSchema, - grantCreateSchema, - grantListSchema, - grantRevokeSchema, - memoryBatchCreateSchema, - memoryCountTreeSchema, - memoryCreateSchema, - memoryDeleteSchema, - memoryDeleteTreeSchema, - memoryGetSchema, - memoryMoveSchema, - memorySearchSchema, - memoryTreeSchema, - memoryUpdateSchema, - ownerGetSchema, - ownerListSchema, - ownerRemoveSchema, - ownerSetSchema, - roleAddMemberSchema, - roleCreateSchema, - roleListForUserSchema, - roleListMembersSchema, - roleRemoveMemberSchema, - treePathSchema, - userCreateSchema, - userGetSchema, - userListSchema, - userRenameSchema, - uuidv7Schema, -} from "./schemas"; - -describe("uuidv7Schema", () => { - test("accepts valid UUIDv7", () => { - const validUuids = [ - "019d694f-79f6-7595-8faf-b70b01c11f98", - "019d694f-79f6-7595-9faf-b70b01c11f98", - "019d694f-79f6-7595-afaf-b70b01c11f98", - "019d694f-79f6-7595-bfaf-b70b01c11f98", - ]; - for (const uuid of validUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(true); - } - }); - - test("rejects invalid UUIDs", () => { - const invalidUuids = [ - "not-a-uuid", - "019d694f-79f6-4595-8faf-b70b01c11f98", // v4 not v7 - "019d694f-79f6-7595-0faf-b70b01c11f98", // invalid variant - "019d694f79f675958fafb70b01c11f98", // no dashes - "", - ]; - for (const uuid of invalidUuids) { - expect(uuidv7Schema.safeParse(uuid).success).toBe(false); - } - }); -}); - -describe("treePathSchema", () => { - test("accepts valid tree paths", () => { - const validPaths = [ - "", // root - "work", - "work.projects", - "work.projects.api", - "me_design", - "A1_B2_C3", - ]; - for (const path of validPaths) { - expect(treePathSchema.safeParse(path).success).toBe(true); - } - }); - - test("rejects invalid tree paths", () => { - const invalidPaths = [ - "work.projects.", // trailing dot - ".work.projects", // leading dot - "work..projects", // double dot - "work-projects", // hyphen not allowed - "work projects", // space not allowed - ]; - for (const path of invalidPaths) { - expect(treePathSchema.safeParse(path).success).toBe(false); - } - }); -}); - -describe("memoryCreateSchema", () => { - test("accepts minimal params", () => { - const result = memoryCreateSchema.safeParse({ - content: "Hello world", - }); - expect(result.success).toBe(true); - }); - - test("accepts full params", () => { - const result = memoryCreateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - content: "Hello world", - meta: { type: "note", tags: ["test"] }, - tree: "work.notes", - temporal: { - start: "2024-01-01T00:00:00Z", - end: "2024-01-02T00:00:00Z", - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts point-in-time temporal", () => { - const result = memoryCreateSchema.safeParse({ - content: "Hello world", - temporal: { - start: "2024-01-01T00:00:00Z", - }, - }); - expect(result.success).toBe(true); - }); - - test("rejects empty content", () => { - const result = memoryCreateSchema.safeParse({ - content: "", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid tree path", () => { - const result = memoryCreateSchema.safeParse({ - content: "Hello", - tree: "invalid-path", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryBatchCreateSchema", () => { - test("accepts array of memories", () => { - const result = memoryBatchCreateSchema.safeParse({ - memories: [ - { content: "Memory 1" }, - { content: "Memory 2", tree: "work" }, - ], - }); - expect(result.success).toBe(true); - }); - - test("rejects empty array", () => { - const result = memoryBatchCreateSchema.safeParse({ - memories: [], - }); - expect(result.success).toBe(false); - }); - - test("rejects more than 1000 memories", () => { - const memories = Array(1001) - .fill(null) - .map((_, i) => ({ content: `Memory ${i}` })); - const result = memoryBatchCreateSchema.safeParse({ memories }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryGetSchema", () => { - test("accepts valid UUID", () => { - const result = memoryGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid UUID", () => { - const result = memoryGetSchema.safeParse({ - id: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryUpdateSchema", () => { - test("accepts id with no updates (no-op)", () => { - const result = memoryUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("accepts partial updates", () => { - const result = memoryUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - content: "Updated content", - }); - expect(result.success).toBe(true); - }); - - test("accepts null to clear optional fields", () => { - const result = memoryUpdateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - temporal: null, - }); - expect(result.success).toBe(true); - }); -}); - -describe("memoryDeleteSchema", () => { - test("accepts valid UUID", () => { - const result = memoryDeleteSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("memorySearchSchema", () => { - test("accepts empty params (filter-only)", () => { - const result = memorySearchSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts fulltext search", () => { - const result = memorySearchSchema.safeParse({ - fulltext: "hello world", - }); - expect(result.success).toBe(true); - }); - - test("accepts semantic search", () => { - const result = memorySearchSchema.safeParse({ - semantic: "What is the meaning of life?", - }); - expect(result.success).toBe(true); - }); - - test("accepts hybrid search", () => { - const result = memorySearchSchema.safeParse({ - semantic: "meaning of life", - fulltext: "philosophy", - }); - expect(result.success).toBe(true); - }); - - test("accepts tree filter with lquery pattern", () => { - const result = memorySearchSchema.safeParse({ - tree: "work.*", - }); - expect(result.success).toBe(true); - }); - - test("accepts tree filter with ltxtquery", () => { - const result = memorySearchSchema.safeParse({ - tree: "api & v2", - }); - expect(result.success).toBe(true); - }); - - test("accepts temporal contains filter", () => { - const result = memorySearchSchema.safeParse({ - temporal: { - contains: "2024-01-15T12:00:00Z", - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts temporal overlaps filter", () => { - const result = memorySearchSchema.safeParse({ - temporal: { - overlaps: { - start: "2024-01-01T00:00:00Z", - end: "2024-01-31T23:59:59Z", - }, - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts temporal within filter", () => { - const result = memorySearchSchema.safeParse({ - temporal: { - within: { - start: "2024-01-01T00:00:00Z", - end: "2024-12-31T23:59:59Z", - }, - }, - }); - expect(result.success).toBe(true); - }); - - test("accepts search weights", () => { - const result = memorySearchSchema.safeParse({ - semantic: "test", - fulltext: "test", - weights: { - semantic: 0.7, - fulltext: 0.3, - }, - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid limit", () => { - const result = memorySearchSchema.safeParse({ - limit: 0, - }); - expect(result.success).toBe(false); - }); - - test("rejects limit over 1000", () => { - const result = memorySearchSchema.safeParse({ - limit: 1001, - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid weights", () => { - const result = memorySearchSchema.safeParse({ - weights: { - semantic: 1.5, // > 1 - }, - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryTreeSchema", () => { - test("accepts empty params", () => { - const result = memoryTreeSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts tree path", () => { - const result = memoryTreeSchema.safeParse({ - tree: "work.projects", - }); - expect(result.success).toBe(true); - }); - - test("accepts levels", () => { - const result = memoryTreeSchema.safeParse({ - levels: 3, - }); - expect(result.success).toBe(true); - }); - - test("rejects levels over 100", () => { - const result = memoryTreeSchema.safeParse({ - levels: 101, - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryMoveSchema", () => { - test("accepts valid source and destination", () => { - const result = memoryMoveSchema.safeParse({ - source: "old.path", - destination: "new.path", - }); - expect(result.success).toBe(true); - }); - - test("accepts moving to root", () => { - const result = memoryMoveSchema.safeParse({ - source: "old.path", - destination: "", - }); - expect(result.success).toBe(true); - }); - - test("accepts dryRun flag", () => { - const result = memoryMoveSchema.safeParse({ - source: "old.path", - destination: "new.path", - dryRun: true, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.dryRun).toBe(true); - } - }); - - test("rejects empty source", () => { - const result = memoryMoveSchema.safeParse({ - source: "", - destination: "new.path", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryDeleteTreeSchema", () => { - test("accepts valid tree path", () => { - const result = memoryDeleteTreeSchema.safeParse({ - tree: "old.stuff", - }); - expect(result.success).toBe(true); - }); - - test("accepts dryRun flag", () => { - const result = memoryDeleteTreeSchema.safeParse({ - tree: "old.stuff", - dryRun: true, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.dryRun).toBe(true); - } - }); - - test("rejects empty tree path", () => { - const result = memoryDeleteTreeSchema.safeParse({ - tree: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("memoryCountTreeSchema", () => { - test("accepts valid tree path", () => { - const result = memoryCountTreeSchema.safeParse({ - tree: "old.stuff", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty tree path", () => { - const result = memoryCountTreeSchema.safeParse({ - tree: "", - }); - expect(result.success).toBe(false); - }); - - test("rejects missing tree path", () => { - const result = memoryCountTreeSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// User Schema Tests -// ============================================================================= - -describe("userCreateSchema", () => { - test("accepts minimal params", () => { - const result = userCreateSchema.safeParse({ - name: "alice", - }); - expect(result.success).toBe(true); - }); - - test("accepts full params", () => { - const result = userCreateSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "alice", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f99", - canLogin: true, - superuser: false, - createrole: true, - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = userCreateSchema.safeParse({ - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("userGetSchema", () => { - test("accepts valid UUID", () => { - const result = userGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("userListSchema", () => { - test("accepts empty params", () => { - const result = userListSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts canLogin filter", () => { - const result = userListSchema.safeParse({ - canLogin: false, - }); - expect(result.success).toBe(true); - }); -}); - -describe("userRenameSchema", () => { - test("accepts valid params", () => { - const result = userRenameSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "new-name", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = userRenameSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Grant Schema Tests -// ============================================================================= - -describe("grantCreateSchema", () => { - test("accepts valid params", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - actions: ["read", "create"], - }); - expect(result.success).toBe(true); - }); - - test("accepts with grant option", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - actions: ["update"], - withGrantOption: true, - }); - expect(result.success).toBe(true); - }); - - test("rejects empty actions", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - actions: [], - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid action", () => { - const result = grantCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - actions: ["read", "invalid"], - }); - expect(result.success).toBe(false); - }); -}); - -describe("grantListSchema", () => { - test("accepts empty params", () => { - const result = grantListSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("accepts userId filter", () => { - const result = grantListSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("grantRevokeSchema", () => { - test("accepts valid params", () => { - const result = grantRevokeSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - }); - expect(result.success).toBe(true); - }); -}); - -describe("grantCheckSchema", () => { - test("accepts valid params", () => { - const result = grantCheckSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects.api", - action: "read", - }); - expect(result.success).toBe(true); - }); - - test("rejects invalid action", () => { - const result = grantCheckSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work", - action: "execute", - }); - expect(result.success).toBe(false); - }); -}); - -// ============================================================================= -// Role Schema Tests -// ============================================================================= - -describe("roleCreateSchema", () => { - test("accepts minimal params", () => { - const result = roleCreateSchema.safeParse({ - name: "editors", - }); - expect(result.success).toBe(true); - }); - - test("accepts with identityId", () => { - const result = roleCreateSchema.safeParse({ - name: "editors", - identityId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = roleCreateSchema.safeParse({ - name: "", - }); - expect(result.success).toBe(false); - }); -}); - -describe("roleAddMemberSchema", () => { - test("accepts valid params", () => { - const result = roleAddMemberSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - memberId: "019d694f-79f6-7595-8faf-b70b01c11f99", - }); - expect(result.success).toBe(true); - }); - - test("accepts with admin option", () => { - const result = roleAddMemberSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - memberId: "019d694f-79f6-7595-8faf-b70b01c11f99", - withAdminOption: true, - }); - expect(result.success).toBe(true); - }); -}); - -describe("roleRemoveMemberSchema", () => { - test("accepts valid params", () => { - const result = roleRemoveMemberSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - memberId: "019d694f-79f6-7595-8faf-b70b01c11f99", - }); - expect(result.success).toBe(true); - }); -}); - -describe("roleListMembersSchema", () => { - test("accepts valid params", () => { - const result = roleListMembersSchema.safeParse({ - roleId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("roleListForUserSchema", () => { - test("accepts valid params", () => { - const result = roleListForUserSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// API Key Schema Tests -// ============================================================================= - -describe("apiKeyCreateSchema", () => { - test("accepts minimal params", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "my-api-key", - }); - expect(result.success).toBe(true); - }); - - test("accepts with expiration", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "my-api-key", - expiresAt: "2025-12-31T23:59:59Z", - }); - expect(result.success).toBe(true); - }); - - test("rejects empty name", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid expiration timestamp", () => { - const result = apiKeyCreateSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - name: "my-api-key", - expiresAt: "not-a-timestamp", - }); - expect(result.success).toBe(false); - }); -}); - -describe("apiKeyGetSchema", () => { - test("accepts valid UUID", () => { - const result = apiKeyGetSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("apiKeyListSchema", () => { - test("accepts valid userId", () => { - const result = apiKeyListSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("apiKeyRevokeSchema", () => { - test("accepts valid UUID", () => { - const result = apiKeyRevokeSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -describe("apiKeyDeleteSchema", () => { - test("accepts valid UUID", () => { - const result = apiKeyDeleteSchema.safeParse({ - id: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); -}); - -// ============================================================================= -// Owner Schema Tests -// ============================================================================= - -describe("ownerSetSchema", () => { - test("accepts valid params", () => { - const result = ownerSetSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - treePath: "work.projects", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing userId", () => { - const result = ownerSetSchema.safeParse({ - treePath: "work.projects", - }); - expect(result.success).toBe(false); - }); - - test("rejects missing treePath", () => { - const result = ownerSetSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(false); - }); - - test("rejects invalid UUID", () => { - const result = ownerSetSchema.safeParse({ - userId: "not-a-uuid", - treePath: "work.projects", - }); - expect(result.success).toBe(false); - }); -}); - -describe("ownerGetSchema", () => { - test("accepts valid treePath", () => { - const result = ownerGetSchema.safeParse({ - treePath: "work.projects.alpha", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing treePath", () => { - const result = ownerGetSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -describe("ownerRemoveSchema", () => { - test("accepts valid treePath", () => { - const result = ownerRemoveSchema.safeParse({ - treePath: "work.projects", - }); - expect(result.success).toBe(true); - }); - - test("rejects missing treePath", () => { - const result = ownerRemoveSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); - -describe("ownerListSchema", () => { - test("accepts with userId", () => { - const result = ownerListSchema.safeParse({ - userId: "019d694f-79f6-7595-8faf-b70b01c11f98", - }); - expect(result.success).toBe(true); - }); - - test("accepts without userId", () => { - const result = ownerListSchema.safeParse({}); - expect(result.success).toBe(true); - }); - - test("rejects invalid userId", () => { - const result = ownerListSchema.safeParse({ - userId: "not-a-uuid", - }); - expect(result.success).toBe(false); - }); -}); diff --git a/packages/server/rpc/engine/schemas.ts b/packages/server/rpc/engine/schemas.ts deleted file mode 100644 index acf5dbd..0000000 --- a/packages/server/rpc/engine/schemas.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Re-export engine schemas from @memory.build/protocol. - * - * @deprecated Import directly from @memory.build/protocol/engine instead. - */ - -export { - type ApiKeyCreateParams, - type ApiKeyDeleteParams, - type ApiKeyGetParams, - type ApiKeyListParams, - type ApiKeyRevokeParams, - // API Key params - apiKeyCreateParams as apiKeyCreateSchema, - apiKeyDeleteParams as apiKeyDeleteSchema, - apiKeyGetParams as apiKeyGetSchema, - apiKeyListParams as apiKeyListSchema, - apiKeyRevokeParams as apiKeyRevokeSchema, -} from "@memory.build/protocol/engine/api-key"; -export { - type GrantCheckParams, - type GrantCreateParams, - type GrantGetParams, - type GrantListParams, - type GrantRevokeParams, - grantCheckParams as grantCheckSchema, - // Grant params - grantCreateParams as grantCreateSchema, - grantGetParams as grantGetSchema, - grantListParams as grantListSchema, - grantRevokeParams as grantRevokeSchema, -} from "@memory.build/protocol/engine/grant"; -export { - type MemoryBatchCreateParams, - type MemoryCountTreeParams, - type MemoryCreateParams, - type MemoryDeleteParams, - type MemoryDeleteTreeParams, - type MemoryGetParams, - type MemoryMoveParams, - type MemorySearchParams, - type MemoryTreeParams, - type MemoryUpdateParams, - memoryBatchCreateParams as memoryBatchCreateSchema, - memoryCountTreeParams as memoryCountTreeSchema, - // Memory params - memoryCreateParams as memoryCreateSchema, - memoryDeleteParams as memoryDeleteSchema, - memoryDeleteTreeParams as memoryDeleteTreeSchema, - memoryGetParams as memoryGetSchema, - memoryMoveParams as memoryMoveSchema, - memorySearchParams as memorySearchSchema, - memoryTreeParams as memoryTreeSchema, - memoryUpdateParams as memoryUpdateSchema, -} from "@memory.build/protocol/engine/memory"; -export { - type OwnerGetParams, - type OwnerListParams, - type OwnerRemoveParams, - type OwnerSetParams, - ownerGetParams as ownerGetSchema, - // Owner params - ownerListParams as ownerListSchema, - ownerRemoveParams as ownerRemoveSchema, - ownerSetParams as ownerSetSchema, -} from "@memory.build/protocol/engine/owner"; -export { - type RoleAddMemberParams, - type RoleCreateParams, - type RoleListForUserParams, - type RoleListMembersParams, - type RoleRemoveMemberParams, - roleAddMemberParams as roleAddMemberSchema, - // Role params - roleCreateParams as roleCreateSchema, - roleListForUserParams as roleListForUserSchema, - roleListMembersParams as roleListMembersSchema, - roleRemoveMemberParams as roleRemoveMemberSchema, -} from "@memory.build/protocol/engine/role"; -export { - type UserCreateParams, - type UserDeleteParams, - type UserGetByNameParams, - type UserGetParams, - type UserListParams, - type UserRenameParams, - // User params - userCreateParams as userCreateSchema, - userDeleteParams as userDeleteSchema, - userGetByNameParams as userGetByNameSchema, - userGetParams as userGetSchema, - userListParams as userListSchema, - userRenameParams as userRenameSchema, -} from "@memory.build/protocol/engine/user"; -export { - // Fields - grantActionSchema, - metaSchema, - searchWeightsSchema, - temporalFilterSchema, - temporalSchema, - timestampSchema, - treeFilterSchema, - treePathSchema, - uuidv7Schema, -} from "@memory.build/protocol/fields"; diff --git a/packages/server/rpc/engine/types.ts b/packages/server/rpc/engine/types.ts deleted file mode 100644 index e316384..0000000 --- a/packages/server/rpc/engine/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Engine RPC context types. - * - * Extends the base HandlerContext with engine-specific fields. - */ -import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { EngineDB } from "@memory.build/engine"; -import type { EngineInfo } from "../../middleware/authenticate"; -import type { HandlerContext } from "../types"; - -/** - * Engine handler context. - * - * Provides access to: - * - `db`: EngineDB instance for the authenticated engine - * - `userId`: The authenticated user's ID (from API key) - * - `apiKeyId`: The API key ID used for authentication - * - `engine`: Engine metadata from accounts DB - * - `embeddingConfig`: Optional config for semantic search - */ -export interface EngineContext extends HandlerContext { - /** EngineDB instance for this engine */ - db: EngineDB; - /** Authenticated user ID */ - userId: string; - /** API key ID used for authentication */ - apiKeyId: string; - /** Engine metadata from accounts DB */ - engine: EngineInfo; - /** Embedding config for semantic search (optional) */ - embeddingConfig?: EmbeddingConfig; -} - -/** - * Type guard to check if context has engine fields. - */ -export function isEngineContext(ctx: HandlerContext): ctx is EngineContext { - return ( - "db" in ctx && - typeof ctx.db === "object" && - ctx.db !== null && - "userId" in ctx && - typeof ctx.userId === "string" && - "apiKeyId" in ctx && - typeof ctx.apiKeyId === "string" && - "engine" in ctx && - typeof ctx.engine === "object" && - ctx.engine !== null - // embeddingConfig is optional, don't check - ); -} - -/** - * Assert that context is an EngineContext, throwing if not. - */ -export function assertEngineContext( - ctx: HandlerContext, -): asserts ctx is EngineContext { - if (!isEngineContext(ctx)) { - throw new Error("Engine context not initialized (authentication required)"); - } -} diff --git a/packages/server/rpc/engine/user.ts b/packages/server/rpc/engine/user.ts deleted file mode 100644 index 5579245..0000000 --- a/packages/server/rpc/engine/user.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Engine RPC user methods. - * - * Implements: - * - user.create: Create a new user - * - user.get: Get user by ID - * - user.getByName: Get user by name - * - user.list: List users (optionally filter by canLogin) - * - user.rename: Rename a user - * - user.delete: Delete a user - */ -import type { User } from "@memory.build/engine"; -import type { - UserCreateParams, - UserDeleteParams, - UserGetByNameParams, - UserGetParams, - UserListParams, - UserRenameParams, - UserResponse, -} from "@memory.build/protocol/engine/user"; -import { - userCreateParams, - userDeleteParams, - userGetByNameParams, - userGetParams, - userListParams, - userRenameParams, -} from "@memory.build/protocol/engine/user"; -import { AppError } from "../errors"; -import { buildRegistry } from "../registry"; -import type { HandlerContext } from "../types"; -import { assertEngineContext, type EngineContext } from "./types"; - -/** - * Convert a User to a serializable response. - */ -function toUserResponse(user: User): UserResponse { - return { - id: user.id, - name: user.name, - identityId: user.identityId, - canLogin: user.canLogin, - superuser: user.superuser, - createrole: user.createrole, - createdAt: user.createdAt.toISOString(), - updatedAt: user.updatedAt?.toISOString() ?? null, - }; -} - -// ============================================================================= -// Method Handlers -// ============================================================================= - -/** - * user.create - Create a new user. - */ -async function userCreate( - params: UserCreateParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const user = await db.createUser({ - id: params.id ?? undefined, - name: params.name, - identityId: params.identityId ?? undefined, - canLogin: params.canLogin, - superuser: params.superuser, - createrole: params.createrole, - }); - - return toUserResponse(user); -} - -/** - * user.get - Get user by ID. - */ -async function userGet( - params: UserGetParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const user = await db.getUser(params.id); - if (!user) { - throw new AppError("NOT_FOUND", `User not found: ${params.id}`); - } - - return toUserResponse(user); -} - -/** - * user.getByName - Get user by name. - */ -async function userGetByName( - params: UserGetByNameParams, - context: HandlerContext, -): Promise { - assertEngineContext(context); - const { db } = context as EngineContext; - - const user = await db.getUserByName(params.name); - if (!user) { - throw new AppError("NOT_FOUND", `User not found: ${params.name}`); - } - - return toUserResponse(user); -} - -/** - * user.list - List users. - */ -async function userList( - params: UserListParams, - context: HandlerContext, -): Promise<{ users: UserResponse[] }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const users = await db.listUsers(params.canLogin); - return { users: users.map(toUserResponse) }; -} - -/** - * user.rename - Rename a user. - */ -async function userRename( - params: UserRenameParams, - context: HandlerContext, -): Promise<{ renamed: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const renamed = await db.renameUser(params.id, params.name); - if (!renamed) { - throw new AppError("NOT_FOUND", `User not found: ${params.id}`); - } - - return { renamed }; -} - -/** - * user.delete - Delete a user. - */ -async function userDelete( - params: UserDeleteParams, - context: HandlerContext, -): Promise<{ deleted: boolean }> { - assertEngineContext(context); - const { db } = context as EngineContext; - - const deleted = await db.deleteUser(params.id); - if (!deleted) { - throw new AppError("NOT_FOUND", `User not found: ${params.id}`); - } - - return { deleted }; -} - -// ============================================================================= -// Registry -// ============================================================================= - -/** - * Build the user methods registry. - */ -export const userMethods = buildRegistry() - .register("user.create", userCreateParams, userCreate) - .register("user.get", userGetParams, userGet) - .register("user.getByName", userGetByNameParams, userGetByName) - .register("user.list", userListParams, userList) - .register("user.rename", userRenameParams, userRename) - .register("user.delete", userDeleteParams, userDelete) - .build(); diff --git a/packages/server/rpc/index.ts b/packages/server/rpc/index.ts index 787b087..bc7e876 100644 --- a/packages/server/rpc/index.ts +++ b/packages/server/rpc/index.ts @@ -1,12 +1,3 @@ -// Method registries -export { accountsMethods } from "./accounts"; -export { - assertEngineContext, - type EngineContext, - engineMethods, - isEngineContext, -} from "./engine"; - // Errors export { APP_ERROR_CODES, @@ -24,9 +15,14 @@ export { parseError, RPC_ERROR_CODES, } from "./errors"; - // Handler export { createRpcHandler, handleRpcRequest } from "./handler"; +export { + assertSpaceRpcContext, + isSpaceRpcContext, + memoryMethods, + type SpaceRpcContext, +} from "./memory"; // Registry export { buildRegistry, @@ -48,3 +44,9 @@ export type { MethodRegistry, RegisteredMethod, } from "./types"; +export { + assertUserRpcContext, + isUserRpcContext, + type UserRpcContext, + userMethods, +} from "./user"; diff --git a/packages/server/rpc/memory/grant.ts b/packages/server/rpc/memory/grant.ts new file mode 100644 index 0000000..8f9af81 --- /dev/null +++ b/packages/server/rpc/memory/grant.ts @@ -0,0 +1,113 @@ +/** + * Tree-access grant handlers (grant.*). Three additive levels + * (1 = read, 2 = write, 3 = owner); owner listing is grant.list filtered to 3. + */ +import { ROOT_PATH } from "@memory.build/engine/core"; +import type { + GrantListParams, + GrantListResult, + GrantRemoveParams, + GrantRemoveResult, + GrantSetParams, + GrantSetResult, +} from "@memory.build/protocol/space"; +import { + grantListParams, + grantRemoveParams, + grantSetParams, +} from "@memory.build/protocol/space"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + callerOwnsAgent, + guardCore, + inputTreePath, + ownsTreePath, + requireSpaceAdmin, + requireTreeOwner, + toTreeGrantResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +/** + * Authority to grant/remove access at a path. Allowed when any of: + * - the target is the caller's OWN agent (self-service — capped anyway); + * - the caller is a space admin (admins manage all access); + * - the caller owns the path or an ancestor (owner@root owns the whole tree). + */ +async function requireGrantAuthority( + ctx: SpaceRpcContext, + principalId: string, + treePath: string, +): Promise { + if (await callerOwnsAgent(ctx, principalId)) return; + if (ctx.admin) return; + requireTreeOwner(ctx, treePath); +} + +async function grantSet( + params: GrantSetParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const treePath = inputTreePath(ctx, params.treePath); + await requireGrantAuthority(ctx, params.principalId, treePath); + await guardCore(() => + ctx.core.grantTreeAccess( + ctx.space.id, + params.principalId, + treePath, + params.access, + ), + ); + return { granted: true }; +} + +async function grantRemove( + params: GrantRemoveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const treePath = inputTreePath(ctx, params.treePath); + await requireGrantAuthority(ctx, params.principalId, treePath); + const removed = await guardCore(() => + ctx.core.removeTreeAccessGrant(ctx.space.id, params.principalId, treePath), + ); + return { removed }; +} + +async function grantList( + params: GrantListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // No path filter means the whole space, i.e. the root path. Listing grants + // under a path requires owning that path (root → owning the whole space), + // else space-admin. Listing your OWN agent's grants is always self-service. + const under = + params.treePath !== undefined && params.treePath !== null + ? inputTreePath(ctx, params.treePath) + : ROOT_PATH; + const ownAgent = + params.principalId !== undefined && + params.principalId !== null && + (await callerOwnsAgent(ctx, params.principalId)); + if (!ownAgent && !ownsTreePath(ctx, under)) { + requireSpaceAdmin(ctx); + } + const grants = await ctx.core.listTreeAccessGrants( + ctx.space.id, + params.principalId ?? undefined, + under, + ); + return { grants: grants.map((g) => toTreeGrantResponse(g, ctx)) }; +} + +export const grantMethods = buildRegistry() + .register("grant.set", grantSetParams, grantSet) + .register("grant.remove", grantRemoveParams, grantRemove) + .register("grant.list", grantListParams, grantList) + .build(); diff --git a/packages/server/rpc/memory/group.ts b/packages/server/rpc/memory/group.ts new file mode 100644 index 0000000..c0b59fc --- /dev/null +++ b/packages/server/rpc/memory/group.ts @@ -0,0 +1,186 @@ +/** + * Group handlers (group.*). Groups are space-scoped principals that bundle + * members for tree-access grants; group membership confers space access. + */ +import type { + GroupAddMemberParams, + GroupAddMemberResult, + GroupCreateParams, + GroupCreateResult, + GroupDeleteParams, + GroupDeleteResult, + GroupListForMemberParams, + GroupListForMemberResult, + GroupListMembersParams, + GroupListMembersResult, + GroupListParams, + GroupListResult, + GroupRemoveMemberParams, + GroupRemoveMemberResult, + GroupRenameParams, + GroupRenameResult, +} from "@memory.build/protocol/space"; +import { + groupAddMemberParams, + groupCreateParams, + groupDeleteParams, + groupListForMemberParams, + groupListMembersParams, + groupListParams, + groupRemoveMemberParams, + groupRenameParams, +} from "@memory.build/protocol/space"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + callerOwnsAgent, + guardCore, + requireGroupAdmin, + requireSpaceAdmin, + toGroupMemberResponse, + toGroupMembershipResponse, + toGroupResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +/** Guard that the group exists in this space. */ +async function assertGroupInSpace( + ctx: SpaceRpcContext, + groupId: string, +): Promise { + const groups = await ctx.core.listSpaceGroups(ctx.space.id); + if (!groups.some((g) => g.id === groupId)) { + throw new AppError( + "NOT_FOUND", + `Group not found in this space: ${groupId}`, + ); + } +} + +async function groupCreate( + params: GroupCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const id = await guardCore(() => + ctx.core.createGroup(ctx.space.id, params.name), + ); + return { id }; +} + +async function groupList( + _params: GroupListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const groups = await ctx.core.listSpaceGroups(ctx.space.id); + return { groups: groups.map(toGroupResponse) }; +} + +async function groupRename( + params: GroupRenameParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + await assertGroupInSpace(ctx, params.id); + const renamed = await guardCore(() => + ctx.core.renamePrincipal(params.id, params.name), + ); + return { renamed }; +} + +async function groupDelete( + params: GroupDeleteParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + await assertGroupInSpace(ctx, params.id); + const deleted = await guardCore(() => ctx.core.deletePrincipal(params.id)); + return { deleted }; +} + +async function groupAddMember( + params: GroupAddMemberParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGroupAdmin(ctx, params.groupId); + await assertGroupInSpace(ctx, params.groupId); + await guardCore(() => + ctx.core.addGroupMember( + ctx.space.id, + params.groupId, + params.memberId, + params.admin ?? false, + ), + ); + return { added: true }; +} + +async function groupRemoveMember( + params: GroupRemoveMemberParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGroupAdmin(ctx, params.groupId); + await assertGroupInSpace(ctx, params.groupId); + const removed = await guardCore(() => + ctx.core.removeGroupMember(ctx.space.id, params.groupId, params.memberId), + ); + return { removed }; +} + +async function groupListMembers( + params: GroupListMembersParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + await requireGroupAdmin(ctx, params.groupId); + await assertGroupInSpace(ctx, params.groupId); + const members = await ctx.core.listGroupMembers(ctx.space.id, params.groupId); + return { members: members.map(toGroupMemberResponse) }; +} + +async function groupListForMember( + params: GroupListForMemberParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // You may see your OWN memberships, or those of an agent you own (so + // `me agent group list` works); seeing anyone else's requires space-admin. + if ( + params.memberId !== ctx.principalId && + !(await callerOwnsAgent(ctx, params.memberId)) + ) { + requireSpaceAdmin(ctx); + } + const groups = await ctx.core.listGroupsForMember( + ctx.space.id, + params.memberId, + ); + return { groups: groups.map(toGroupMembershipResponse) }; +} + +export const groupMethods = buildRegistry() + .register("group.create", groupCreateParams, groupCreate) + .register("group.list", groupListParams, groupList) + .register("group.rename", groupRenameParams, groupRename) + .register("group.delete", groupDeleteParams, groupDelete) + .register("group.addMember", groupAddMemberParams, groupAddMember) + .register("group.removeMember", groupRemoveMemberParams, groupRemoveMember) + .register("group.listMembers", groupListMembersParams, groupListMembers) + .register("group.listForMember", groupListForMemberParams, groupListForMember) + .build(); diff --git a/packages/server/rpc/memory/index.ts b/packages/server/rpc/memory/index.ts new file mode 100644 index 0000000..1b6bef4 --- /dev/null +++ b/packages/server/rpc/memory/index.ts @@ -0,0 +1,31 @@ +/** + * Memory RPC method registry — served at `/api/v1/memory/rpc`. + * + * The new-model replacement for the engine RPC, combining the memory data-plane + * methods (spaceStore) with the space management methods (coreStore): membership, + * groups, tree-access grants, and invitations. + */ +import type { MethodRegistry } from "../types"; +import { grantMethods } from "./grant"; +import { groupMethods } from "./group"; +import { invitationMethods } from "./invitation"; +import { memoryDataMethods } from "./memory"; +import { principalMethods } from "./principal"; + +export { + assertSpaceRpcContext, + isSpaceRpcContext, + type SpaceRpcContext, +} from "./types"; + +/** + * The full memory-endpoint registry: data-plane + space management methods. + * (Agent lifecycle and api keys live on the user endpoint — see rpc/user.) + */ +export const memoryMethods: MethodRegistry = new Map([ + ...memoryDataMethods, + ...principalMethods, + ...groupMethods, + ...grantMethods, + ...invitationMethods, +]); diff --git a/packages/server/rpc/memory/invitation.ts b/packages/server/rpc/memory/invitation.ts new file mode 100644 index 0000000..b117a01 --- /dev/null +++ b/packages/server/rpc/memory/invitation.ts @@ -0,0 +1,104 @@ +/** + * Space invitation handlers (invite.*). + * + * Inviting an *already-registered* user adds them to the space immediately — + * access is recomputed per request (build_tree_access), so it takes effect on + * their existing session without a re-login. Inviting a not-yet-registered email + * records a pending invitation, redeemed at their first verified login (see + * redeemInvitationsForVerifiedLogin). Both paths grant owner@home and, when a + * share level is set, that level at the shared root. + * + * Authority: all three methods require space-admin (structural authority over + * the roster, like group management — owner@root alone is not enough). Inviting + * people, optionally as admins, is a deliberate structural act. + */ +import { SHARE_NAMESPACE } from "@memory.build/database"; +import type { + InviteCreateParams, + InviteCreateResult, + InviteListParams, + InviteListResult, + InviteRevokeParams, + InviteRevokeResult, +} from "@memory.build/protocol/space"; +import { + inviteCreateParams, + inviteListParams, + inviteRevokeParams, +} from "@memory.build/protocol/space"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + guardCore, + requireSpaceAdmin, + toSpaceInvitationResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +async function inviteCreate( + params: InviteCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const admin = params.admin ?? false; + const shareAccess = params.shareAccess ?? null; + + // Already-registered user → add them now (instant access on their existing + // session). Not-yet-registered → a pending invite, redeemed at first login. + const existing = await ctx.core.getUserByName(params.email); + if (existing) { + await guardCore(async () => { + await ctx.core.addPrincipalToSpace(ctx.space.id, existing.id, admin); + if (shareAccess !== null) { + await ctx.core.grantTreeAccess( + ctx.space.id, + existing.id, + SHARE_NAMESPACE, + shareAccess, + ); + } + }); + return { applied: true, invitationId: null, principalId: existing.id }; + } + + const invitationId = await guardCore(() => + ctx.core.createSpaceInvitation(ctx.space.id, params.email, { + admin, + shareAccess, + invitedBy: ctx.principalId, + }), + ); + return { applied: false, invitationId, principalId: null }; +} + +async function inviteList( + _params: InviteListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const invitations = await ctx.core.listSpaceInvitations(ctx.space.id); + return { invitations: invitations.map(toSpaceInvitationResponse) }; +} + +async function inviteRevoke( + params: InviteRevokeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + requireSpaceAdmin(ctx); + const revoked = await guardCore(() => + ctx.core.revokeSpaceInvitation(ctx.space.id, params.email), + ); + return { revoked }; +} + +export const invitationMethods = buildRegistry() + .register("invite.create", inviteCreateParams, inviteCreate) + .register("invite.list", inviteListParams, inviteList) + .register("invite.revoke", inviteRevokeParams, inviteRevoke) + .build(); diff --git a/packages/server/rpc/memory/management.integration.test.ts b/packages/server/rpc/memory/management.integration.test.ts new file mode 100644 index 0000000..450426b --- /dev/null +++ b/packages/server/rpc/memory/management.integration.test.ts @@ -0,0 +1,671 @@ +// Integration test for the space management handlers (4C-2b): member / agent / +// group / grant / invite, driven through the merged memory registry against a +// provisioned space. The provisioned owner has owner@root, satisfying the +// management authorization gate. (Api keys are user-endpoint — see +// rpc/user/api-key.integration.test.ts.) +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/memory/management.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import type { TreeAccess } from "@memory.build/engine/core"; +import * as engineCore from "@memory.build/engine/core"; +import * as engineSpace from "@memory.build/engine/space"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import { provisionUser } from "../../provision"; +import type { HandlerContext } from "../types"; +import { memoryMethods } from "./index"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +let ownerTreeAccess: TreeAccess; +let space: { id: string; slug: string }; +let ownerId: string; +let ownerEmail: string; + +function call( + method: string, + params: unknown, + as: { principalId?: string; treeAccess?: TreeAccess; admin?: boolean } = {}, +): Promise { + const registered = memoryMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/memory/rpc"), + store: engineSpace.spaceStore(sql, `me_${space.slug}`), + core: engineCore.coreStore(sql, coreSchema), + space, + principalId: as.principalId ?? ownerId, + apiKeyId: null, + treeAccess: as.treeAccess ?? ownerTreeAccess, + // the provisioned owner is also a space admin; non-owner callers default false + admin: as.admin ?? as.principalId === undefined, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +/** Create a standalone global user (no auth), returning its id. */ +async function makeUser(): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await engineCore + .coreStore(sql, coreSchema) + .createUser(id, `u_${rand(8)}@example.com`); + return id; +} + +/** + * Create a global agent owned by `owner` (the user-endpoint operation), returning + * its id. Not yet a member of any space — principal.add brings it in. + */ +function makeAgent(owner: string): Promise { + return engineCore + .coreStore(sql, coreSchema) + .createAgent(owner, `agent_${rand(6)}`); +} + +/** Create a registered user with a known email (the invite key), returning its id. */ +async function makeUserWithEmail(email: string): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await engineCore.coreStore(sql, coreSchema).createUser(id, email); + return id; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand(8)}`; + coreSchema = `core_test_${rand(8)}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + ownerEmail = `owner_${crypto.randomUUID().slice(0, 8)}@example.com`; + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: ownerEmail, + name: "Owner", + provider: "github", + accountId: crypto.randomUUID(), + }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + space = { id: r.spaceId, slug: r.spaceSlug }; + ownerId = r.userId; + ownerTreeAccess = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(r.userId, r.spaceId); +}); + +test("principal: list / add / remove", async () => { + const listed = await call<{ principals: { id: string; admin: boolean }[] }>( + "principal.list", + {}, + ); + expect(listed.principals.some((m) => m.id === ownerId && m.admin)).toBe(true); + + const other = await makeUser(); + expect( + (await call<{ added: boolean }>("principal.add", { principalId: other })) + .added, + ).toBe(true); + expect( + ( + await call<{ principals: { id: string }[] }>("principal.list", {}) + ).principals.some((m) => m.id === other), + ).toBe(true); + expect( + ( + await call<{ removed: boolean }>("principal.remove", { + principalId: other, + }) + ).removed, + ).toBe(true); +}); + +test("last-admin safeguard: removing/demoting the sole admin → LAST_ADMIN", async () => { + // the provisioned owner is the space's only admin + await expectAppError( + call("principal.remove", { principalId: ownerId }), + "LAST_ADMIN", + ); + await expectAppError( + call("principal.add", { principalId: ownerId, admin: false }), + "LAST_ADMIN", + ); + + // promote a second admin, then the owner can be removed + const other = await makeUser(); + await call("principal.add", { principalId: other, admin: true }); + expect( + ( + await call<{ removed: boolean }>("principal.remove", { + principalId: ownerId, + }) + ).removed, + ).toBe(true); +}); + +test("principal.resolve / lookup are available to non-admin members (list is admin-only)", async () => { + const email = `target_${rand(8)}@example.com`; + const targetId = await makeUserWithEmail(email); + await call("principal.add", { principalId: targetId }); // added by the admin owner + + // a non-admin caller: resolve/lookup have no authority gate beyond being in the + // space, so they work; principal.list (full enumeration) does not. + const asMember = { + principalId: targetId, + treeAccess: [{ tree_path: "x", access: 1 }] as TreeAccess, + admin: false, + }; + + const resolved = await call<{ principals: { id: string; name: string }[] }>( + "principal.resolve", + { name: email.toUpperCase() }, // case-insensitive + asMember, + ); + expect(resolved.principals).toHaveLength(1); + expect(resolved.principals[0]?.id).toBe(targetId); + + const looked = await call<{ principals: { id: string; name: string }[] }>( + "principal.lookup", + { ids: [targetId] }, + asMember, + ); + expect(looked.principals[0]?.name).toBe(email); + + // a name that isn't in the space resolves to nothing + expect( + ( + await call<{ principals: unknown[] }>( + "principal.resolve", + { name: `nobody_${rand(8)}@example.com` }, + asMember, + ) + ).principals, + ).toHaveLength(0); + + // full enumeration stays admin-only + await expectAppError(call("principal.list", {}, asMember), "FORBIDDEN"); +}); + +test("group: create / list / members / rename / delete", async () => { + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "eng", + }); + expect( + (await call<{ groups: { id: string }[] }>("group.list", {})).groups.some( + (g) => g.id === groupId, + ), + ).toBe(true); + + await call("group.addMember", { groupId, memberId: ownerId, admin: true }); + const members = await call<{ + members: { memberId: string; admin: boolean }[]; + }>("group.listMembers", { groupId }); + expect(members.members[0]?.memberId).toBe(ownerId); + expect(members.members[0]?.admin).toBe(true); + + const forMember = await call<{ groups: { groupId: string }[] }>( + "group.listForMember", + { memberId: ownerId }, + ); + expect(forMember.groups.some((g) => g.groupId === groupId)).toBe(true); + + expect( + ( + await call<{ renamed: boolean }>("group.rename", { + id: groupId, + name: "platform", + }) + ).renamed, + ).toBe(true); + expect( + ( + await call<{ removed: boolean }>("group.removeMember", { + groupId, + memberId: ownerId, + }) + ).removed, + ).toBe(true); + expect( + (await call<{ deleted: boolean }>("group.delete", { id: groupId })).deleted, + ).toBe(true); +}); + +test("grant: set / list / remove", async () => { + const other = await makeUser(); + await call("grant.set", { principalId: other, treePath: "docs", access: 1 }); + const grants = await call<{ + grants: { principalId: string; treePath: string; access: number }[]; + }>("grant.list", { principalId: other }); + expect(grants.grants).toHaveLength(1); + expect(grants.grants[0]?.treePath).toBe("docs"); + expect(grants.grants[0]?.access).toBe(1); + + expect( + ( + await call<{ removed: boolean }>("grant.remove", { + principalId: other, + treePath: "docs", + }) + ).removed, + ).toBe(true); +}); + +test("invite.create: a not-yet-registered email creates a pending invite; list + revoke", async () => { + const email = `newcomer_${rand(8)}@example.com`; + const res = await call<{ + applied: boolean; + invitationId: string | null; + principalId: string | null; + }>("invite.create", { email, admin: false, shareAccess: 1 }); + expect(res.applied).toBe(false); + expect(res.invitationId).toBeTruthy(); + expect(res.principalId).toBeNull(); + + const { invitations } = await call<{ + invitations: { + email: string; + shareAccess: number | null; + invitedByName: string | null; + }[]; + }>("invite.list", {}); + expect(invitations).toHaveLength(1); + expect(invitations[0]?.email).toBe(email); + expect(invitations[0]?.shareAccess).toBe(1); + expect(invitations[0]?.invitedByName).toBe(ownerEmail); // the owner invited + + expect( + (await call<{ revoked: boolean }>("invite.revoke", { email })).revoked, + ).toBe(true); + expect( + (await call<{ invitations: unknown[] }>("invite.list", {})).invitations, + ).toHaveLength(0); +}); + +test("invite.create: an already-registered user is added immediately (no pending invite)", async () => { + const email = `existing_${rand(8)}@example.com`; + const existingId = await makeUserWithEmail(email); + + const res = await call<{ + applied: boolean; + invitationId: string | null; + principalId: string | null; + }>("invite.create", { email, admin: true, shareAccess: 2 }); + expect(res.applied).toBe(true); + expect(res.principalId).toBe(existingId); + expect(res.invitationId).toBeNull(); + + // they are a space admin now, with owner@home + write@share + const core = engineCore.coreStore(sql, coreSchema); + const principals = await core.listSpacePrincipals(space.id); + expect(principals.find((p) => p.id === existingId)?.admin).toBe(true); + const ta = await core.buildTreeAccess(existingId, space.id); + expect(ta).toContainEqual({ + tree_path: `home.${existingId.replace(/-/g, "")}`, + access: 3, + }); + expect(ta).toContainEqual({ tree_path: "share", access: 2 }); + + // joined → not shown as a pending invitation + expect( + (await call<{ invitations: unknown[] }>("invite.list", {})).invitations, + ).toHaveLength(0); +}); + +test("invite.* require space-admin authority (owner@root is not enough)", async () => { + // a plain member with no authority + const plain = await makeUser(); + const asPlain = { + principalId: plain, + treeAccess: [] as TreeAccess, + admin: false, + }; + await expectAppError(call("invite.list", {}, asPlain), "FORBIDDEN"); + + // a member who owns the whole data tree (owner@root) but is NOT a space admin + // is still forbidden — inviting is structural, like group management + const rootOwner = await makeUser(); + await call("principal.add", { principalId: rootOwner }); + await call("grant.set", { principalId: rootOwner, treePath: "", access: 3 }); + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(rootOwner, space.id); + const asOwner = { principalId: rootOwner, treeAccess: ta, admin: false }; + await expectAppError( + call( + "invite.create", + { email: `x_${rand(8)}@example.com`, admin: false, shareAccess: 1 }, + asOwner, + ), + "FORBIDDEN", + ); + await expectAppError(call("invite.list", {}, asOwner), "FORBIDDEN"); + await expectAppError( + call("invite.revoke", { email: `x_${rand(8)}@example.com` }, asOwner), + "FORBIDDEN", + ); +}); + +test("roster/group management requires admin or owner", async () => { + // a plain member: write access on a subtree, not an admin, not a root owner + const member = await makeUser(); + const as = { + principalId: member, + treeAccess: [{ tree_path: "sub", access: 2 }] as TreeAccess, + admin: false, + }; + await expectAppError(call("principal.list", {}, as), "FORBIDDEN"); + await expectAppError(call("group.create", { name: "x" }, as), "FORBIDDEN"); +}); + +test("a space admin (without owner@root) has management authority", async () => { + // an admin member with read on a path (plus its own home from joining), but no + // owner@root — so the management authority here comes from admin, not ownership + const adminMember = await makeUser(); + await call("principal.add", { principalId: adminMember, admin: true }); + await call("grant.set", { + principalId: adminMember, + treePath: "x", + access: 1, + }); + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(adminMember, space.id); + const as = { principalId: adminMember, treeAccess: ta, admin: true }; + + // can manage the roster and grant anywhere despite holding no owner grant + expect( + (await call<{ principals: unknown[] }>("principal.list", {}, as)).principals + .length, + ).toBeGreaterThan(0); + const stranger = await makeUser(); + expect( + ( + await call<{ granted: boolean }>( + "grant.set", + { principalId: stranger, treePath: "anywhere", access: 2 }, + as, + ) + ).granted, + ).toBe(true); +}); + +test("group.listForMember: own memberships are self-service, others need admin", async () => { + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "squad", + }); + const member = await makeUser(); + await call("group.addMember", { groupId, memberId: member }); + const as = { + principalId: member, + treeAccess: [] as TreeAccess, + admin: false, + }; + + // the member can see their own memberships + const mine = await call<{ groups: { groupId: string }[] }>( + "group.listForMember", + { memberId: member }, + as, + ); + expect(mine.groups.some((g) => g.groupId === groupId)).toBe(true); + + // but not someone else's + await expectAppError( + call("group.listForMember", { memberId: ownerId }, as), + "FORBIDDEN", + ); +}); + +test("group member management allows a group admin (not a space admin)", async () => { + // owner creates a group and makes `lead` an admin of it + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "team", + }); + const lead = await makeUser(); + // lead is only a group admin (not added to principal_space) — group + // membership is transitive, so this is enough authority over the group + await call("group.addMember", { groupId, memberId: lead, admin: true }); + const as = { principalId: lead, treeAccess: [] as TreeAccess, admin: false }; + + // lead can manage THIS group's membership + const other = await makeUser(); + expect( + ( + await call<{ added: boolean }>( + "group.addMember", + { groupId, memberId: other }, + as, + ) + ).added, + ).toBe(true); + const members = await call<{ members: { memberId: string }[] }>( + "group.listMembers", + { groupId }, + as, + ); + expect(members.members.some((m) => m.memberId === other)).toBe(true); + expect( + ( + await call<{ removed: boolean }>( + "group.removeMember", + { groupId, memberId: other }, + as, + ) + ).removed, + ).toBe(true); + + // but structural ops (create) still require a space admin + await expectAppError(call("group.create", { name: "nope" }, as), "FORBIDDEN"); + + // and a non-admin, non-member can't manage the group + const stranger = await makeUser(); + await expectAppError( + call( + "group.listMembers", + { groupId }, + { principalId: stranger, treeAccess: [] as TreeAccess, admin: false }, + ), + "FORBIDDEN", + ); +}); + +test("group.listForMember: an agent's owner can list its groups", async () => { + // owner sets up: a member who owns an agent, the agent is in a group + const member = await makeUser(); + const agentId = await makeAgent(member); + await call("principal.add", { principalId: agentId }); + const { id: groupId } = await call<{ id: string }>("group.create", { + name: "bots", + }); + await call("group.addMember", { groupId, memberId: agentId }); + + // the member (agent owner, not a space admin) can list their agent's groups + const as = { + principalId: member, + treeAccess: [] as TreeAccess, + admin: false, + }; + const res = await call<{ groups: { groupId: string }[] }>( + "group.listForMember", + { memberId: agentId }, + as, + ); + expect(res.groups.some((g) => g.groupId === groupId)).toBe(true); + + // a stranger who doesn't own the agent cannot + const stranger = await makeUser(); + await expectAppError( + call( + "group.listForMember", + { memberId: agentId }, + { principalId: stranger, treeAccess: [] as TreeAccess, admin: false }, + ), + "FORBIDDEN", + ); +}); + +test("structural mutations require admin — owner@root is not enough", async () => { + // a member who owns the whole data tree (owner@root) but is NOT a space admin + const member = await makeUser(); + await call("principal.add", { principalId: member }); + await call("grant.set", { principalId: member, treePath: "", access: 3 }); + const ta = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(member, space.id); + const as = { principalId: member, treeAccess: ta, admin: false }; + + // owner@root can manage grants on its data — e.g. list them... + expect( + (await call<{ grants: unknown[] }>("grant.list", {}, as)).grants.length, + ).toBeGreaterThan(0); + // ...but the roster (enumeration + add/remove) and groups are admin-only: + // owning the data tree is not structural authority. + await expectAppError(call("principal.list", {}, as), "FORBIDDEN"); + const stranger = await makeUser(); + await expectAppError( + call("principal.add", { principalId: stranger }, as), + "FORBIDDEN", + ); + await expectAppError( + call("principal.remove", { principalId: member }, as), + "FORBIDDEN", + ); + await expectAppError(call("group.create", { name: "g" }, as), "FORBIDDEN"); +}); + +test("grant authority is path-scoped: a subtree owner delegates within it", async () => { + // a member who owns "proj" (not the root) can manage access under proj only + const member = await makeUser(); + await call("principal.add", { principalId: member }); + await call("grant.set", { principalId: member, treePath: "proj", access: 3 }); + const memberTA = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(member, space.id); + const as = { principalId: member, treeAccess: memberTA }; + + const stranger = await makeUser(); + // within the owned subtree → allowed + expect( + ( + await call<{ granted: boolean }>( + "grant.set", + { principalId: stranger, treePath: "proj.sub", access: 1 }, + as, + ) + ).granted, + ).toBe(true); + // outside it → FORBIDDEN + await expectAppError( + call( + "grant.set", + { principalId: stranger, treePath: "other", access: 1 }, + as, + ), + "FORBIDDEN", + ); + + // can list grants under the owned subtree, but not the whole space + const underProj = await call<{ + grants: { treePath: string }[]; + }>("grant.list", { treePath: "proj" }, as); + expect(underProj.grants.some((g) => g.treePath === "proj.sub")).toBe(true); + await expectAppError(call("grant.list", {}, as), "FORBIDDEN"); +}); + +test("self-service: a non-owner member brings their own agent into the space", async () => { + // owner onboards a second user with write (not owner) on a subtree + const member = await makeUser(); + await call("principal.add", { principalId: member }); + await call("grant.set", { principalId: member, treePath: "proj", access: 2 }); + const memberTA = await engineCore + .coreStore(sql, coreSchema) + .buildTreeAccess(member, space.id); + const as = { principalId: member, treeAccess: memberTA }; + + // the member created their agent on the user endpoint (simulated via core); + // they bring it into the space (self-service principal.add) without owner rights + const agentId = await makeAgent(member); + expect( + ( + await call<{ added: boolean }>( + "principal.add", + { principalId: agentId }, + as, + ) + ).added, + ).toBe(true); + + // and self-grant it (capped by their own access). Minting the agent's api key + // is a user-endpoint op (apiKey.* — see rpc/user/api-key.integration.test.ts). + expect( + ( + await call<{ granted: boolean }>( + "grant.set", + { principalId: agentId, treePath: "proj", access: 2 }, + as, + ) + ).granted, + ).toBe(true); + + // but the member cannot manage the roster, add a stranger, or grant to others + await expectAppError(call("principal.list", {}, as), "FORBIDDEN"); + const stranger = await makeUser(); + await expectAppError( + call("principal.add", { principalId: stranger }, as), + "FORBIDDEN", + ); + await expectAppError( + call( + "grant.set", + { principalId: stranger, treePath: "proj", access: 1 }, + as, + ), + "FORBIDDEN", + ); +}); diff --git a/packages/server/rpc/memory/memory.integration.test.ts b/packages/server/rpc/memory/memory.integration.test.ts new file mode 100644 index 0000000..d3bdce0 --- /dev/null +++ b/packages/server/rpc/memory/memory.integration.test.ts @@ -0,0 +1,504 @@ +// Integration test for the memory RPC data-plane handlers (4C-1). +// +// Provisions a space + owner, then drives the memory.* handlers through the +// registry against a real space schema (me_). Semantic search is not +// exercised (embeddings are generated by the worker — Phase 4D); fulltext +// (bm25) and filter search are. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/memory/memory.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import type { TreeAccess } from "@memory.build/engine/core"; +import * as engineCore from "@memory.build/engine/core"; +import type { SpaceStore } from "@memory.build/engine/space"; +import * as engineSpace from "@memory.build/engine/space"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import { provisionUser } from "../../provision"; +import type { HandlerContext } from "../types"; +import { memoryDataMethods } from "./memory"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let authSchema: string; +let coreSchema: string; +const createdSpaceSchemas: string[] = []; + +// Per-test space context. +let store: SpaceStore; +let treeAccess: TreeAccess; +let space: { id: string; slug: string }; +let principalId: string; + +function call( + method: string, + params: unknown, + ta: TreeAccess = treeAccess, +): Promise { + const registered = memoryDataMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/memory/rpc"), + store, + core: engineCore.coreStore(sql, coreSchema), + space, + principalId, + apiKeyId: null, + treeAccess: ta, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +/** Assert a handler call rejects with a specific AppError code. */ +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + await bootstrapSpaceDatabase(sql); + await migrateAuth(sql, { schema: authSchema }); + await migrateCore(sql, { schema: coreSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); +}); + +// Fresh space per test so trees don't bleed across cases. +beforeEach(async () => { + const core = engineCore.coreStore(sql, coreSchema); + const r = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: `mem_${crypto.randomUUID().slice(0, 8)}@example.com`, + name: "Owner", + provider: "github", + accountId: crypto.randomUUID(), + }, + ); + createdSpaceSchemas.push(`me_${r.spaceSlug}`); + store = engineSpace.spaceStore(sql, `me_${r.spaceSlug}`); + // The default creator owns its home + the shared root (`share`), not the whole + // tree — so these tests write under `share.*`. (The normalization test, which + // needs arbitrary paths, elevates itself to owner@root.) + treeAccess = await core.buildTreeAccess(r.userId, r.spaceId); + space = { id: r.spaceId, slug: r.spaceSlug }; + principalId = r.userId; +}); + +test("~ home + lenient separators normalize on input, reverse-map on output", async () => { + // This test writes to arbitrary (non-home, non-share) paths to exercise path + // normalization, so elevate the owner to owner@root for it. + const core = engineCore.coreStore(sql, coreSchema); + await core.grantTreeAccess( + space.id, + principalId, + engineCore.ROOT_PATH, + engineCore.ACCESS.owner, + ); + treeAccess = await core.buildTreeAccess(principalId, space.id); + + const home = `home.${principalId.replace(/-/g, "")}`; + + // `~/notes` (slash accepted on input) stores under the caller's home and + // reads back in canonical dot form as `~.notes`. + const a = await call<{ id: string; tree: string }>("memory.create", { + content: "home note", + tree: "~/notes", + }); + expect(a.tree).toBe("~.notes"); + expect((await call<{ tree: string }>("memory.get", { id: a.id })).tree).toBe( + "~.notes", + ); + + // The raw stored ltree is home..notes — proves real expansion, not just display. + const [row] = await sql.unsafe( + `select tree::text as tree from me_${space.slug}.memory where id = $1`, + [a.id], + ); + expect(row?.tree).toBe(`${home}.notes`); + + // Slash input normalizes to dot ltree; non-home paths display canonically. + const b = await call<{ tree: string }>("memory.create", { + content: "slashy", + tree: "/work/projects/", + }); + expect(b.tree).toBe("work.projects"); + + // memory.tree with a `~` base finds the home node, reverse-mapped to `~.notes`. + const tree = await call<{ nodes: { path: string; count: number }[] }>( + "memory.tree", + { tree: "~" }, + ); + expect(tree.nodes.some((n) => n.path === "~.notes")).toBe(true); + + // An illegal label is a validation error. + await expectAppError( + call("memory.create", { content: "x", tree: "bad label" }), + "VALIDATION_ERROR", + ); +}); + +test("create without a tree is a validation error (tree is required)", async () => { + // `tree` is required on the wire — callers must choose `share` vs `~` + // explicitly. The file importers default to `share`, but a bare RPC create + // does not. + await expectAppError( + call("memory.create", { content: "no tree given" }), + "VALIDATION_ERROR", + ); +}); + +test("create → get round-trips content/tree/meta and createdBy is null", async () => { + const created = await call<{ id: string; createdBy: string | null }>( + "memory.create", + { content: "hello world", tree: "share.notes.work", meta: { tag: "a" } }, + ); + expect(created.createdBy).toBeNull(); + + const got = await call<{ + id: string; + content: string; + tree: string; + meta: Record; + hasEmbedding: boolean; + }>("memory.get", { id: created.id }); + expect(got.content).toBe("hello world"); + expect(got.tree).toBe("share.notes.work"); + expect(got.meta).toEqual({ tag: "a" }); + expect(got.hasEmbedding).toBe(false); +}); + +test("create with temporal round-trips as {start,end}", async () => { + const created = await call<{ id: string }>("memory.create", { + content: "temporal one", + tree: "share", + temporal: { start: "2024-01-01T00:00:00Z", end: "2024-01-02T00:00:00Z" }, + }); + const got = await call<{ temporal: { start: string; end: string } | null }>( + "memory.get", + { id: created.id }, + ); + expect(got.temporal).toEqual({ + start: "2024-01-01T00:00:00.000Z", + end: "2024-01-02T00:00:00.000Z", + }); +}); + +test("update patches fields", async () => { + const created = await call<{ id: string }>("memory.create", { + content: "before", + tree: "share.a", + }); + const updated = await call<{ content: string; tree: string }>( + "memory.update", + { id: created.id, content: "after", tree: "share.a.b" }, + ); + expect(updated.content).toBe("after"); + expect(updated.tree).toBe("share.a.b"); +}); + +test("delete removes; get then NOT_FOUND", async () => { + const created = await call<{ id: string }>("memory.create", { + content: "doomed", + tree: "share", + }); + const res = await call<{ deleted: boolean }>("memory.delete", { + id: created.id, + }); + expect(res.deleted).toBe(true); + await expectAppError(call("memory.get", { id: created.id }), "NOT_FOUND"); +}); + +test("get / delete unknown id → NOT_FOUND", async () => { + const ghost = crypto.randomUUID(); + await expectAppError(call("memory.get", { id: ghost }), "NOT_FOUND"); + await expectAppError(call("memory.delete", { id: ghost }), "NOT_FOUND"); +}); + +test("batchCreate inserts all and is retrievable", async () => { + const res = await call<{ ids: string[]; updatedIds: string[] }>( + "memory.batchCreate", + { + memories: [ + { content: "one", tree: "share.batch" }, + { content: "two", tree: "share.batch" }, + { content: "three", tree: "share.batch.sub" }, + ], + }, + ); + expect(res.ids).toHaveLength(3); + expect(res.updatedIds).toHaveLength(0); + const count = await call<{ count: number }>("memory.countTree", { + tree: "share.batch", + }); + expect(count.count).toBe(3); +}); + +test("create with a duplicate explicit id → CONFLICT", async () => { + const id = "01941000-0000-7000-8000-00000000c0f1"; + await call("memory.create", { id, content: "first", tree: "share.dup" }); + await expectAppError( + call("memory.create", { id, content: "second", tree: "share.dup" }), + "CONFLICT", + ); +}); + +test("batchCreate without replaceIfMetaDiffers skips duplicates", async () => { + const id = "01941000-0000-7000-8000-00000000c0f2"; + await call("memory.batchCreate", { + memories: [{ id, content: "original", tree: "share.skip" }], + }); + const res = await call<{ ids: string[]; updatedIds: string[] }>( + "memory.batchCreate", + { memories: [{ id, content: "replacement", tree: "share.skip" }] }, + ); + expect(res.ids).toHaveLength(0); + expect(res.updatedIds).toHaveLength(0); + const got = await call<{ content: string }>("memory.get", { id }); + expect(got.content).toBe("original"); +}); + +test("batchCreate with replaceIfMetaDiffers splits insert/update/skip", async () => { + const stale = "01941000-0000-7000-8000-00000000c0f3"; + const fresh = "01941000-0000-7000-8000-00000000c0f4"; + const brandNew = "01941000-0000-7000-8000-00000000c0f5"; + await call("memory.batchCreate", { + memories: [ + { id: stale, content: "old render", tree: "share.up", meta: { v: "1" } }, + { id: fresh, content: "current", tree: "share.up", meta: { v: "2" } }, + ], + }); + + const res = await call<{ ids: string[]; updatedIds: string[] }>( + "memory.batchCreate", + { + memories: [ + { + id: stale, + content: "new render", + tree: "share.up", + meta: { v: "2" }, + }, + { id: fresh, content: "untouched", tree: "share.up", meta: { v: "2" } }, + { id: brandNew, content: "added", tree: "share.up", meta: { v: "2" } }, + ], + replaceIfMetaDiffers: "v", + }, + ); + expect(res.ids).toEqual([brandNew]); + expect(res.updatedIds).toEqual([stale]); + + const updated = await call<{ content: string }>("memory.get", { id: stale }); + expect(updated.content).toBe("new render"); + const skipped = await call<{ content: string }>("memory.get", { id: fresh }); + expect(skipped.content).toBe("current"); +}); + +test("tree returns descendant node counts under a path", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "x", tree: "share.root.a" }, + { content: "y", tree: "share.root.a.deep" }, + { content: "z", tree: "share.root.b" }, + ], + }); + const res = await call<{ nodes: { path: string; count: number }[] }>( + "memory.tree", + { tree: "share.root" }, + ); + const byPath = Object.fromEntries(res.nodes.map((n) => [n.path, n.count])); + expect(byPath["share.root.a"]).toBe(2); + expect(byPath["share.root.a.deep"]).toBe(1); + expect(byPath["share.root.b"]).toBe(1); + // the base path itself is excluded + expect(byPath["share.root"]).toBeUndefined(); +}); + +test("tree respects levels depth limit", async () => { + await call("memory.batchCreate", { + memories: [{ content: "deep", tree: "share.t.a.b.c" }], + }); + const res = await call<{ nodes: { path: string }[] }>("memory.tree", { + tree: "share.t", + levels: 1, + }); + const paths = res.nodes.map((n) => n.path); + expect(paths).toContain("share.t.a"); + expect(paths).not.toContain("share.t.a.b"); +}); + +test("move relocates a subtree (dryRun counts without moving)", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "m1", tree: "share.src.x" }, + { content: "m2", tree: "share.src.y" }, + ], + }); + const dry = await call<{ count: number }>("memory.move", { + source: "share.src", + destination: "share.dst", + dryRun: true, + }); + expect(dry.count).toBe(2); + // still under share.src + expect( + (await call<{ count: number }>("memory.countTree", { tree: "share.src" })) + .count, + ).toBe(2); + + const moved = await call<{ count: number }>("memory.move", { + source: "share.src", + destination: "share.dst", + }); + expect(moved.count).toBe(2); + expect( + (await call<{ count: number }>("memory.countTree", { tree: "share.src" })) + .count, + ).toBe(0); + expect( + (await call<{ count: number }>("memory.countTree", { tree: "share.dst" })) + .count, + ).toBe(2); +}); + +test("deleteTree removes a subtree (dryRun counts without deleting)", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "d1", tree: "share.gone.a" }, + { content: "d2", tree: "share.gone.b" }, + ], + }); + const dry = await call<{ count: number }>("memory.deleteTree", { + tree: "share.gone", + dryRun: true, + }); + expect(dry.count).toBe(2); + const del = await call<{ count: number }>("memory.deleteTree", { + tree: "share.gone", + }); + expect(del.count).toBe(2); + expect( + (await call<{ count: number }>("memory.countTree", { tree: "share.gone" })) + .count, + ).toBe(0); +}); + +test("search: fulltext (bm25) finds matching content", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "the quick brown fox", tree: "share.s" }, + { content: "lazy dogs sleep", tree: "share.s" }, + ], + }); + const res = await call<{ results: { content: string }[]; total: number }>( + "memory.search", + { fulltext: "fox" }, + ); + expect(res.total).toBeGreaterThanOrEqual(1); + expect(res.results.some((r) => r.content.includes("fox"))).toBe(true); +}); + +test("search: tree filter only (no ranking) returns matches", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "in scope", tree: "share.scope.a" }, + { content: "out of scope", tree: "share.other" }, + ], + }); + const res = await call<{ results: { tree: string }[] }>("memory.search", { + tree: "share.scope", + }); + expect(res.results.length).toBe(1); + expect(res.results[0]?.tree).toBe("share.scope.a"); +}); + +test("search: tree lquery wildcard matches descendants", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "direct child", tree: "share.proj.a" }, + { content: "deep descendant", tree: "share.proj.a.deep" }, + { content: "sibling root", tree: "share.proj" }, + { content: "elsewhere", tree: "share.other" }, + ], + }); + // `share.proj.*` is lquery (not a literal ltree) — it must bind to the + // lquery param, not cast to ::ltree (which would throw). lquery `*` matches + // zero-or-more labels, so it matches share.proj and everything under it, but + // not share.other. + const res = await call<{ results: { tree: string }[] }>("memory.search", { + tree: "share.proj.*", + }); + const trees = res.results.map((r) => r.tree).sort(); + expect(trees).toEqual(["share.proj", "share.proj.a", "share.proj.a.deep"]); +}); + +test("search: tree ltxtquery (label boolean) matches by label", async () => { + await call("memory.batchCreate", { + memories: [ + { content: "both labels", tree: "share.alpha.beta" }, + { content: "one label", tree: "share.alpha" }, + ], + }); + // `alpha & beta` is ltxtquery — must bind to the ltxtquery param. + const res = await call<{ results: { tree: string }[] }>("memory.search", { + tree: "alpha & beta", + }); + expect(res.results.map((r) => r.tree)).toEqual(["share.alpha.beta"]); +}); + +test("search: grep alone is rejected", async () => { + await expectAppError( + call("memory.search", { grep: "anything" }), + "VALIDATION_ERROR", + ); +}); + +test("search: semantic without embedding config → EMBEDDING_NOT_CONFIGURED", async () => { + await expectAppError( + call("memory.search", { semantic: "meaning" }), + "EMBEDDING_NOT_CONFIGURED", + ); +}); + +test("create without write access → FORBIDDEN", async () => { + // read-only on the root path + const readOnly: TreeAccess = [ + { tree_path: "", access: engineCore.ACCESS.read }, + ]; + await expectAppError( + call("memory.create", { content: "nope", tree: "x" }, readOnly), + "FORBIDDEN", + ); +}); diff --git a/packages/server/rpc/memory/memory.ts b/packages/server/rpc/memory/memory.ts new file mode 100644 index 0000000..264b387 --- /dev/null +++ b/packages/server/rpc/memory/memory.ts @@ -0,0 +1,520 @@ +/** + * Memory RPC data-plane methods (new model) — served at `/api/v1/memory/rpc`. + * + * Adapts the stable memory.* wire protocol onto the space data-plane store + * (spaceStore). The wire is unchanged from the legacy engine RPC; the mapping + * is handler-local. Lossy by design (see Phase 4C): `createdBy` is always null + * (the space model has no per-memory creator) and search `total` is the returned + * row count. `orderBy` applies to unranked (filter-only) search — chronological + * by id, desc (default, newest first) or asc; ranked/hybrid search ignores it + * (score-desc). + */ +import { generateEmbedding } from "@memory.build/embedding"; +import { ACCESS } from "@memory.build/engine/core"; +import type { + SearchResultItem, + Memory as SpaceMemory, +} from "@memory.build/engine/space"; +import type { + MemoryBatchCreateParams, + MemoryBatchCreateResult, + MemoryCountTreeParams, + MemoryCountTreeResult, + MemoryCreateParams, + MemoryDeleteParams, + MemoryDeleteResult, + MemoryDeleteTreeParams, + MemoryDeleteTreeResult, + MemoryGetParams, + MemoryMoveParams, + MemoryMoveResult, + MemoryResponse, + MemorySearchParams, + MemorySearchResult, + MemoryTreeParams, + MemoryTreeResult, + MemoryUpdateParams, +} from "@memory.build/protocol/memory"; +import { + memoryBatchCreateParams, + memoryCountTreeParams, + memoryCreateParams, + memoryDeleteParams, + memoryDeleteTreeParams, + memoryGetParams, + memoryMoveParams, + memorySearchParams, + memoryTreeParams, + memoryUpdateParams, +} from "@memory.build/protocol/memory"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { displayTreePath, inputTreeFilter, inputTreePath } from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +// ============================================================================= +// Helpers +// ============================================================================= + +/** + * Translate a space SQL error into an AppError. The space functions raise + * `insufficient_privilege` (42501) on access violations and + * `invalid_parameter_value` (22023) / `invalid_text_representation` (22P02) + * on malformed input; everything else propagates as an internal error. + */ +function mapSpaceError(e: unknown): never { + const code = (e as { code?: string }).code; + if (code === "42501") { + throw new AppError("FORBIDDEN", "Insufficient tree access"); + } + if (code === "22023" || code === "22P02") { + throw new AppError( + "VALIDATION_ERROR", + e instanceof Error ? e.message : "Invalid parameter", + ); + } + throw e instanceof Error ? e : new Error(String(e)); +} + +/** Run a space-store call, mapping its SQL errors to AppErrors. */ +async function guard(fn: () => Promise): Promise { + try { + return await fn(); + } catch (e) { + return mapSpaceError(e); + } +} + +/** + * Format a wire temporal `{start, end?}` into a PostgreSQL tstzrange string. + * Point-in-time (no end / end == start) → `[t,t]`; otherwise `[start,end)`. + * Mirrors the legacy engine's tstzrange formatting. + */ +function formatTemporal( + t: { start: string; end?: string | null } | null | undefined, +): string | undefined { + if (!t) return undefined; + const start = t.start; + const end = t.end ?? start; + return start === end ? `[${start},${end}]` : `[${start},${end})`; +} + +/** + * Parse a PostgreSQL tstzrange string into a wire `{start, end}` (ISO), + * normalizing the timestamps. Mirrors the legacy engine's parser. + */ +function parseTemporal( + range: string | null, +): { start: string; end: string } | null { + if (!range) return null; + const m = range.match(/[[(]"?([^",]+)"?,"?([^",\])]+)"?[\])]/); + if (!m) return null; + const [, start, end] = m; + if (!start || !end) return null; + return { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + }; +} + +/** ltree depth (label count); root ("") is 0. */ +function nlevel(path: string): number { + return path === "" ? 0 : path.split(".").length; +} + +function toMemoryResponse( + m: SpaceMemory, + ctx: SpaceRpcContext, +): MemoryResponse { + return { + id: m.id, + content: m.content, + meta: m.meta, + tree: displayTreePath(ctx, m.tree), + temporal: parseTemporal(m.temporal), + hasEmbedding: m.hasEmbedding, + createdAt: m.createdAt.toISOString(), + // The space model does not track a per-memory creator (4C decision). + createdBy: null, + updatedAt: m.updatedAt?.toISOString() ?? null, + }; +} + +/** + * Map the wire temporal filter (contains | overlaps | within — mutually + * exclusive) onto the space search's temporal range params. A `contains` + * point becomes an inclusive point-range overlap (true iff the memory's range + * spans the instant). + */ +function mapTemporalFilter(tf: MemorySearchParams["temporal"]): { + temporalWithin?: string; + temporalOverlaps?: string; +} { + if (!tf) return {}; + if (tf.within) { + return { temporalWithin: `[${tf.within.start},${tf.within.end})` }; + } + if (tf.overlaps) { + return { temporalOverlaps: `[${tf.overlaps.start},${tf.overlaps.end})` }; + } + if (tf.contains) { + return { temporalOverlaps: `[${tf.contains},${tf.contains}]` }; + } + return {}; +} + +// ============================================================================= +// Method Handlers +// ============================================================================= + +/** memory.create */ +async function memoryCreate( + params: MemoryCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const created = await guard(() => + store.createMemory(treeAccess, { + id: params.id ?? undefined, + content: params.content, + meta: params.meta ?? undefined, + tree: inputTreePath(ctx, params.tree), + temporal: formatTemporal(params.temporal), + }), + ); + if (created === null) { + // The store skips an explicit id that already exists (no replace key is + // passed here). For a single create that's a caller error, not a skip. + throw new AppError("CONFLICT", `Memory already exists: ${params.id}`); + } + const memory = await store.getMemory(treeAccess, created.id); + if (!memory) { + throw new AppError("INTERNAL_ERROR", "Created memory could not be read"); + } + return toMemoryResponse(memory, ctx); +} + +/** + * memory.batchCreate — atomic across the batch (one set-based statement, + * `batch_create_memory`). + * + * `ids` carries the inserted memories; `updatedIds` the existing rows + * rewritten via `replaceIfMetaDiffers` (conditional upsert). A submitted + * explicit id in neither array was skipped — deterministic-id importers + * re-submit freely and classify the missing ids as already imported. An id + * repeated within one batch collapses to its first occurrence. + */ +async function memoryBatchCreate( + params: MemoryBatchCreateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const rows = await guard(() => + store.batchCreateMemories( + treeAccess, + params.memories.map((m) => ({ + id: m.id ?? undefined, + content: m.content, + meta: m.meta ?? undefined, + tree: inputTreePath(ctx, m.tree), + temporal: formatTemporal(m.temporal), + })), + params.replaceIfMetaDiffers ?? undefined, + ), + ); + const ids: string[] = []; + const updatedIds: string[] = []; + for (const r of rows) { + (r.inserted ? ids : updatedIds).push(r.id); + } + return { ids, updatedIds }; +} + +/** memory.get */ +async function memoryGet( + params: MemoryGetParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const memory = await guard(() => store.getMemory(treeAccess, params.id)); + if (!memory) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + return toMemoryResponse(memory, ctx); +} + +/** memory.update */ +async function memoryUpdate( + params: MemoryUpdateParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const patch: { + content?: string; + meta?: Record; + tree?: string; + temporal?: string | null; + } = {}; + if (params.content !== undefined && params.content !== null) { + patch.content = params.content; + } + if (params.meta !== undefined && params.meta !== null) { + patch.meta = params.meta; + } + if (params.tree !== undefined && params.tree !== null) { + patch.tree = inputTreePath(ctx, params.tree); + } + if (params.temporal !== undefined) { + patch.temporal = + params.temporal === null + ? null + : (formatTemporal(params.temporal) ?? null); + } + + const ok = await guard(() => store.patchMemory(treeAccess, params.id, patch)); + if (!ok) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + const memory = await store.getMemory(treeAccess, params.id); + if (!memory) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + return toMemoryResponse(memory, ctx); +} + +/** memory.delete */ +async function memoryDelete( + params: MemoryDeleteParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const { store, treeAccess } = context as SpaceRpcContext; + + const deleted = await guard(() => store.deleteMemory(treeAccess, params.id)); + if (!deleted) { + throw new AppError("NOT_FOUND", `Memory not found: ${params.id}`); + } + return { deleted }; +} + +/** memory.search — hybrid (fulltext+semantic) or single-arm / filter-only. */ +async function memorySearch( + params: MemorySearchParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess, embeddingConfig } = ctx; + + // Generate the query embedding for semantic search. + let vec: number[] | undefined; + if (params.semantic) { + if (!embeddingConfig) { + throw new AppError( + "EMBEDDING_NOT_CONFIGURED", + "Semantic search requires embedding configuration. Set EMBEDDING_API_KEY.", + ); + } + try { + vec = (await generateEmbedding(params.semantic, embeddingConfig)) + .embedding; + } catch (error) { + throw new AppError( + "EMBEDDING_FAILED", + `Failed to generate embedding: ${error instanceof Error ? error.message : "Unknown error"}`, + ); + } + } + + const bm25 = params.fulltext ?? undefined; + + // grep alone would force a full table scan — require another indexed filter. + if ( + params.grep && + !params.fulltext && + !params.semantic && + !params.tree && + !params.meta && + !params.temporal + ) { + throw new AppError( + "VALIDATION_ERROR", + "grep cannot be used alone (full table scan). Combine with semantic, fulltext, tree, meta, or temporal.", + ); + } + + // semanticThreshold is a cosine similarity (0..1); the space search filters by + // cosine distance (= 1 - similarity), and only when a vector is present. + const maxVecDist = + vec && params.semanticThreshold != null + ? 1 - params.semanticThreshold + : undefined; + + // Classify the tree filter so a wildcard (`foo.*`) binds to lquery and a + // boolean label search (`a & b`) to ltxtquery, rather than all casting to + // ltree (which throws on query syntax). + const treeFilter = params.tree ? inputTreeFilter(ctx, params.tree) : null; + const filters = { + ltree: treeFilter?.kind === "ltree" ? treeFilter.value : undefined, + lquery: treeFilter?.kind === "lquery" ? treeFilter.value : undefined, + ltxtquery: treeFilter?.kind === "ltxtquery" ? treeFilter.value : undefined, + metaContains: params.meta ?? undefined, + regexp: params.grep ?? undefined, + ...mapTemporalFilter(params.temporal), + }; + const limit = params.limit ?? 10; + + let items: SearchResultItem[]; + if (bm25 && vec) { + items = await guard(() => + store.hybridSearch(treeAccess, { + bm25, + vec, + maxVecDist, + candidateLimit: params.candidateLimit, + fulltextWeight: params.weights?.fulltext, + semanticWeight: params.weights?.semantic, + limit, + ...filters, + }), + ); + } else { + items = await guard(() => + store.search(treeAccess, { + bm25, + vec, + maxVecDist, + limit, + order: params.orderBy ?? undefined, + ...filters, + }), + ); + } + + return { + results: items.map((item) => ({ + ...toMemoryResponse(item, ctx), + score: item.score, + })), + total: items.length, + limit, + }; +} + +/** memory.tree — node counts under a path, down to `levels` depth. */ +async function memoryTree( + params: MemoryTreeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const base = params.tree ? inputTreePath(ctx, params.tree) : ""; + // `a.b.*` matches a.b and everything under it; `*` matches all paths. + const lquery = base === "" ? "*" : `${base}.*`; + const entries = await guard(() => store.listTree(treeAccess, lquery)); + + const baseDepth = nlevel(base); + const nodes = entries + .filter((e) => { + const depth = nlevel(e.tree); + // strict descendants of the base path (exclude the base and its ancestors) + if (depth <= baseDepth) return false; + if (params.levels !== undefined && depth - baseDepth > params.levels) { + return false; + } + return true; + }) + .map((e) => ({ path: displayTreePath(ctx, e.tree), count: e.count })); + + return { nodes }; +} + +/** memory.move */ +async function memoryMove( + params: MemoryMoveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const count = await guard(() => + store.moveTree( + treeAccess, + inputTreePath(ctx, params.source), + inputTreePath(ctx, params.destination), + params.dryRun ?? false, + ), + ); + return { count }; +} + +/** memory.deleteTree */ +async function memoryDeleteTree( + params: MemoryDeleteTreeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const count = await guard(() => + store.deleteTree( + treeAccess, + inputTreePath(ctx, params.tree), + params.dryRun ?? false, + ), + ); + return { count }; +} + +/** memory.countTree */ +async function memoryCountTree( + params: MemoryCountTreeParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + const { store, treeAccess } = ctx; + + const count = await guard(() => + store.countTree( + treeAccess, + { tree: inputTreePath(ctx, params.tree) }, + ACCESS.read, + ), + ); + return { count }; +} + +// ============================================================================= +// Registry +// ============================================================================= + +export const memoryDataMethods = buildRegistry() + .register("memory.create", memoryCreateParams, memoryCreate) + .register("memory.batchCreate", memoryBatchCreateParams, memoryBatchCreate) + .register("memory.get", memoryGetParams, memoryGet) + .register("memory.update", memoryUpdateParams, memoryUpdate) + .register("memory.delete", memoryDeleteParams, memoryDelete) + .register("memory.search", memorySearchParams, memorySearch) + .register("memory.tree", memoryTreeParams, memoryTree) + .register("memory.move", memoryMoveParams, memoryMove) + .register("memory.deleteTree", memoryDeleteTreeParams, memoryDeleteTree) + .register("memory.countTree", memoryCountTreeParams, memoryCountTree) + .build(); diff --git a/packages/server/rpc/memory/principal.ts b/packages/server/rpc/memory/principal.ts new file mode 100644 index 0000000..8369872 --- /dev/null +++ b/packages/server/rpc/memory/principal.ts @@ -0,0 +1,132 @@ +/** + * Space membership handlers (principal.*) — the space roster (principal_space). + */ +import type { + PrincipalAddParams, + PrincipalAddResult, + PrincipalListParams, + PrincipalListResult, + PrincipalLookupParams, + PrincipalLookupResult, + PrincipalRemoveParams, + PrincipalRemoveResult, + PrincipalResolveParams, + PrincipalResolveResult, +} from "@memory.build/protocol/space"; +import { + principalAddParams, + principalListParams, + principalLookupParams, + principalRemoveParams, + principalResolveParams, +} from "@memory.build/protocol/space"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { + callerOwnsAgentGlobal, + guardCore, + requireSpaceAdmin, + toSpacePrincipalResponse, +} from "./support"; +import { assertSpaceRpcContext, type SpaceRpcContext } from "./types"; + +async function principalList( + params: PrincipalListParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Enumerating the whole roster is structural — admin only. (Targeted name / id + // lookups for any member are principal.resolve / principal.lookup.) + requireSpaceAdmin(ctx); + const principals = await ctx.core.listSpacePrincipals( + ctx.space.id, + params.kind ?? undefined, + ); + return { principals: principals.map(toSpacePrincipalResponse) }; +} + +async function principalAdd( + params: PrincipalAddParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Bringing your OWN agent into a space is self-service (it stays capped by + // your access); adding anyone else is a structural roster change that requires + // space-admin (owner@root is not enough). A member can't grant themselves admin + // on their own agent membership. + const ownAgent = + params.admin !== true && + (await callerOwnsAgentGlobal(ctx, params.principalId)); + if (!ownAgent) { + requireSpaceAdmin(ctx); + } + await guardCore(() => + ctx.core.addPrincipalToSpace( + ctx.space.id, + params.principalId, + params.admin ?? false, + ), + ); + return { added: true }; +} + +async function principalRemove( + params: PrincipalRemoveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Removing a roster member is structural, like adding — space-admin only. + requireSpaceAdmin(ctx); + const removed = await guardCore(() => + ctx.core.removePrincipalFromSpace(ctx.space.id, params.principalId), + ); + return { removed }; +} + +async function principalResolve( + params: PrincipalResolveParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // No authority gate beyond space participation: reaching this handler means the + // caller has access in this space (the authenticate-space membership gate). This + // is a targeted name->id lookup, not roster enumeration (that is principal.list). + const principals = await ctx.core.listSpacePrincipals( + ctx.space.id, + params.kind ?? undefined, + ); + const lower = params.name.trim().toLowerCase(); + const matches = principals + .filter((p) => p.name.toLowerCase() === lower) + .map((p) => ({ id: p.id, kind: p.kind, name: p.name })); + return { principals: matches }; +} + +async function principalLookup( + params: PrincipalLookupParams, + context: HandlerContext, +): Promise { + assertSpaceRpcContext(context); + const ctx = context as SpaceRpcContext; + // Member-accessible reverse lookup (id -> name/kind) for display; only ids that + // are in the space come back. Same gating rationale as principalResolve. + const ids = new Set(params.ids); + if (ids.size === 0) return { principals: [] }; + const principals = await ctx.core.listSpacePrincipals(ctx.space.id); + const found = principals + .filter((p) => ids.has(p.id)) + .map((p) => ({ id: p.id, kind: p.kind, name: p.name })); + return { principals: found }; +} + +export const principalMethods = buildRegistry() + .register("principal.list", principalListParams, principalList) + .register("principal.add", principalAddParams, principalAdd) + .register("principal.remove", principalRemoveParams, principalRemove) + .register("principal.resolve", principalResolveParams, principalResolve) + .register("principal.lookup", principalLookupParams, principalLookup) + .build(); diff --git a/packages/server/rpc/memory/support.ts b/packages/server/rpc/memory/support.ts new file mode 100644 index 0000000..ec7dce4 --- /dev/null +++ b/packages/server/rpc/memory/support.ts @@ -0,0 +1,260 @@ +/** + * Shared helpers for the space management handlers (member/group/grant/invite): + * the owner authorization gate, core SQL error mapping, and response + * serializers. + */ + +import { + classifyTreeFilter, + denormalizeTreePath, + normalizeTreePath, + type TreeFilter, + TreePathError, +} from "@memory.build/database"; +import type { + Group, + GroupMember, + GroupMembership, + SpaceInvitation, + SpacePrincipal, + TreeGrant, +} from "@memory.build/engine/core"; +import { ACCESS, ROOT_PATH } from "@memory.build/engine/core"; +import type { + GroupMemberResponse, + GroupMembershipResponse, + GroupResponse, + SpaceInvitationResponse, + SpacePrincipalResponse, + TreeGrantResponse, +} from "@memory.build/protocol/space"; +import { guardCore } from "../core-error"; +import { AppError } from "../errors"; +import type { SpaceRpcContext } from "./types"; + +export { guardCore }; + +// ============================================================================= +// Tree-path normalization at the user-facing boundary +// ============================================================================= + +/** + * Normalize a concrete tree path from the wire to canonical ltree, expanding a + * leading `~` to the caller's home. Maps malformed input to a validation error. + */ +export function inputTreePath(ctx: SpaceRpcContext, raw: string): string { + try { + return normalizeTreePath(raw, { home: ctx.principalId }); + } catch (e) { + throw asValidationError(e); + } +} + +/** + * Like `inputTreePath` but for a search filter: normalizes `~`/slashes and + * classifies the result as an ltree path, an `lquery` pattern, or an + * `ltxtquery` label search, so the handler can bind the right SQL parameter. + * Returns `null` when there is no filter. + */ +export function inputTreeFilter( + ctx: SpaceRpcContext, + raw: string, +): TreeFilter | null { + try { + return classifyTreeFilter(raw, { home: ctx.principalId }); + } catch (e) { + throw asValidationError(e); + } +} + +/** Reverse the home expansion for display: the caller's home shows as `~/…`. */ +export function displayTreePath(ctx: SpaceRpcContext, stored: string): string { + return denormalizeTreePath(stored, { home: ctx.principalId }); +} + +function asValidationError(e: unknown): AppError { + if (e instanceof TreePathError) { + return new AppError("VALIDATION_ERROR", e.message); + } + return e instanceof AppError + ? e + : new AppError("VALIDATION_ERROR", "Invalid tree path"); +} + +/** + * Structural authority over the space (principal_space.admin). Required for + * managing groups — a structural construct of the space, distinct from data + * ownership: owning the data tree (owner@root) is NOT sufficient. + */ +export function requireSpaceAdmin(context: SpaceRpcContext): void { + if (!context.admin) { + throw new AppError("FORBIDDEN", "This action requires being a space admin"); + } +} + +/** + * Authority to manage a group's membership: a space admin, or an admin of the + * group itself (group_member.admin). Used by group.addMember / removeMember / + * listMembers. (Creating/renaming/deleting groups stays space-admin only.) + */ +export async function requireGroupAdmin( + context: SpaceRpcContext, + groupId: string, +): Promise { + if (context.admin) return; + const groupAdmin = await context.core.isGroupAdmin( + context.principalId, + groupId, + context.space.id, + ); + if (!groupAdmin) { + throw new AppError( + "FORBIDDEN", + "Managing group members requires being a space admin or an admin of the group", + ); + } +} + +/** True if `ancestor` is an ancestor-or-self of `path` (ltree `@>`). */ +function isAncestorOrSelf(ancestor: string, path: string): boolean { + return ( + ancestor === ROOT_PATH || + path === ancestor || + path.startsWith(`${ancestor}.`) + ); +} + +/** + * Owner authority at a specific tree path: the caller holds an owner grant (3) + * at the path or any ancestor of it. This is how grants are delegated — owning + * a subtree lets you manage access within it. Owner@root is the case that + * covers the whole space. + */ +export function ownsTreePath( + context: SpaceRpcContext, + treePath: string, +): boolean { + return context.treeAccess.some( + (g) => g.access >= ACCESS.owner && isAncestorOrSelf(g.tree_path, treePath), + ); +} + +export function requireTreeOwner( + context: SpaceRpcContext, + treePath: string, +): void { + if (!ownsTreePath(context, treePath)) { + throw new AppError( + "FORBIDDEN", + `Granting access at "${treePath}" requires owner access on that path`, + ); + } +} + +/** + * True if `principalId` is an agent in this space owned by the caller. Agents + * are user-owned and capped by their owner's access (agent_tree_access), so a + * member managing their own agents (create/keys/self-grants) is self-service + * and safe — it can't escalate beyond the caller's own access. + */ +export async function callerOwnsAgent( + context: SpaceRpcContext, + principalId: string, +): Promise { + const agents = await context.core.listSpacePrincipals(context.space.id, "a"); + const agent = agents.find((a) => a.id === principalId); + return agent !== undefined && agent.ownerId === context.principalId; +} + +/** + * True if `principalId` is an agent owned by the caller, checked globally (not + * scoped to the current space). Used by principal.add so a member can bring + * their OWN agent into a space before it is a member there. + */ +export async function callerOwnsAgentGlobal( + context: SpaceRpcContext, + principalId: string, +): Promise { + const principal = await context.core.getPrincipal(principalId); + return ( + principal !== null && + principal.kind === "a" && + principal.ownerId === context.principalId + ); +} + +// ============================================================================= +// Serializers (Date → ISO) +// ============================================================================= + +export function toSpacePrincipalResponse( + m: SpacePrincipal, +): SpacePrincipalResponse { + return { + id: m.id, + kind: m.kind, + name: m.name, + ownerId: m.ownerId, + direct: m.direct, + admin: m.admin, + createdAt: m.createdAt.toISOString(), + updatedAt: m.updatedAt?.toISOString() ?? null, + }; +} + +export function toGroupResponse(g: Group): GroupResponse { + return { + id: g.id, + name: g.name, + createdAt: g.createdAt.toISOString(), + updatedAt: g.updatedAt?.toISOString() ?? null, + }; +} + +export function toGroupMemberResponse(m: GroupMember): GroupMemberResponse { + return { + memberId: m.memberId, + kind: m.kind, + name: m.name, + admin: m.admin, + createdAt: m.createdAt.toISOString(), + }; +} + +export function toGroupMembershipResponse( + m: GroupMembership, +): GroupMembershipResponse { + return { + groupId: m.groupId, + name: m.name, + admin: m.admin, + createdAt: m.createdAt.toISOString(), + }; +} + +export function toTreeGrantResponse( + g: TreeGrant, + ctx: SpaceRpcContext, +): TreeGrantResponse { + return { + principalId: g.principalId, + treePath: displayTreePath(ctx, g.treePath), + access: g.access, + createdAt: g.createdAt.toISOString(), + updatedAt: g.updatedAt?.toISOString() ?? null, + }; +} + +export function toSpaceInvitationResponse( + i: SpaceInvitation, +): SpaceInvitationResponse { + return { + id: i.id, + email: i.email, + admin: i.admin, + shareAccess: i.shareAccess, + invitedBy: i.invitedBy, + invitedByName: i.invitedByName, + createdAt: i.createdAt.toISOString(), + }; +} diff --git a/packages/server/rpc/memory/types.ts b/packages/server/rpc/memory/types.ts new file mode 100644 index 0000000..b6c3420 --- /dev/null +++ b/packages/server/rpc/memory/types.ts @@ -0,0 +1,62 @@ +/** + * Memory RPC context types. + * + * The context for `/api/v1/memory/rpc` — populated by authenticateSpace. Memory + * (data-plane) methods use `store` + `treeAccess`; management (control-plane) + * methods use `core` + `space`. + */ +import type { EmbeddingConfig } from "@memory.build/embedding"; +import type { CoreStore, Space, TreeAccess } from "@memory.build/engine/core"; +import type { SpaceStore } from "@memory.build/engine/space"; +import type { HandlerContext } from "../types"; + +export interface SpaceRpcContext extends HandlerContext { + /** Space data-plane store bound to the `me_` schema. */ + store: SpaceStore; + /** Core control-plane store (management methods). */ + core: CoreStore; + /** The resolved space. */ + space: Space; + /** Authenticated principal id (user id for sessions, agent id for api keys). */ + principalId: string; + /** Api key id when authenticated by api key; null for sessions. */ + apiKeyId: string | null; + /** The principal's effective grants in this space — the access gate. */ + treeAccess: TreeAccess; + /** Whether the principal is a space admin (principal_space.admin). */ + admin: boolean; + /** Embedding config for semantic search (optional). */ + embeddingConfig?: EmbeddingConfig; +} + +/** + * Type guard for the memory RPC context. + */ +export function isSpaceRpcContext(ctx: HandlerContext): ctx is SpaceRpcContext { + return ( + "store" in ctx && + typeof ctx.store === "object" && + ctx.store !== null && + "core" in ctx && + typeof ctx.core === "object" && + ctx.core !== null && + "space" in ctx && + typeof ctx.space === "object" && + ctx.space !== null && + "principalId" in ctx && + typeof ctx.principalId === "string" && + "treeAccess" in ctx && + Array.isArray(ctx.treeAccess) + ); +} + +/** + * Assert that context is a SpaceRpcContext, throwing if not. + */ +export function assertSpaceRpcContext( + ctx: HandlerContext, +): asserts ctx is SpaceRpcContext { + if (!isSpaceRpcContext(ctx)) { + throw new Error("Space context not initialized (authentication required)"); + } +} diff --git a/packages/server/rpc/user/agent.integration.test.ts b/packages/server/rpc/user/agent.integration.test.ts new file mode 100644 index 0000000..abe0133 --- /dev/null +++ b/packages/server/rpc/user/agent.integration.test.ts @@ -0,0 +1,297 @@ +// Integration test for the user RPC agent handlers (agent.* lifecycle). +// User-scoped (no space): a user manages their own global service accounts. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/user/agent.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import { ACCESS, coreStore, ROOT_PATH } from "@memory.build/engine/core"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import type { HandlerContext } from "../types"; +import { userMethods } from "./index"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let coreSchema: string; +let authSchema: string; +let userId: string; +const createdSpaceSchemas: string[] = []; + +function call( + method: string, + params: unknown, + asUser: string = userId, +): Promise { + const registered = userMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/user/rpc"), + core: coreStore(sql, coreSchema), + auth: authStore(sql, authSchema), + userId: asUser, + db: sql, + coreSchema, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +async function makeUser(): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await coreStore(sql, coreSchema).createUser(id, `u_${rand(8)}@example.com`); + return id; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + authSchema = `auth_test_${rand(8)}`; + await bootstrapSpaceDatabase(sql); // extensions for me_ (space.create) + await migrateCore(sql, { schema: coreSchema }); + await migrateAuth(sql, { schema: authSchema }); +}); + +afterAll(async () => { + for (const s of createdSpaceSchemas) { + await sql.unsafe(`drop schema if exists ${s} cascade`); + } + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + userId = await makeUser(); +}); + +test("whoami returns the session's identity", async () => { + // a user in the auth schema; whoami resolves it from ctx.auth by id + const email = `who_${rand(8)}@example.com`; + const id = await authStore(sql, authSchema).createUser(email, "Who Am I"); + + const me = await call<{ id: string; email: string; name: string }>( + "whoami", + {}, + id, + ); + expect(me).toEqual({ id, email, name: "Who Am I" }); +}); + +test("whoami is UNAUTHORIZED when the user row is gone", async () => { + const [row] = await sql`select uuidv7() as id`; + await expectAppError(call("whoami", {}, row?.id as string), "UNAUTHORIZED"); +}); + +test("create / list / rename / delete the caller's agents", async () => { + const { id } = await call<{ id: string }>("agent.create", { name: "bot" }); + + let agents = await call<{ agents: { id: string; name: string }[] }>( + "agent.list", + {}, + ); + expect(agents.agents).toHaveLength(1); + expect(agents.agents[0]?.id).toBe(id); + expect(agents.agents[0]?.name).toBe("bot"); + + expect( + (await call<{ renamed: boolean }>("agent.rename", { id, name: "bot2" })) + .renamed, + ).toBe(true); + agents = await call("agent.list", {}); + expect(agents.agents[0]?.name).toBe("bot2"); + + expect( + (await call<{ deleted: boolean }>("agent.delete", { id })).deleted, + ).toBe(true); + expect( + (await call<{ agents: unknown[] }>("agent.list", {})).agents, + ).toHaveLength(0); +}); + +test("agent.list is scoped to the caller", async () => { + await call("agent.create", { name: "mine" }); + const other = await makeUser(); + const otherList = await call<{ agents: unknown[] }>("agent.list", {}, other); + expect(otherList.agents).toHaveLength(0); +}); + +test("cannot rename/delete another user's agent", async () => { + const { id } = await call<{ id: string }>("agent.create", { name: "mine" }); + const intruder = await makeUser(); + await expectAppError( + call("agent.rename", { id, name: "hijacked" }, intruder), + "FORBIDDEN", + ); + await expectAppError(call("agent.delete", { id }, intruder), "FORBIDDEN"); +}); + +test("rename/delete of a non-existent agent → NOT_FOUND", async () => { + const [row] = await sql`select uuidv7() as id`; + const ghost = row?.id as string; + await expectAppError( + call("agent.rename", { id: ghost, name: "x" }), + "NOT_FOUND", + ); +}); + +test("space.list returns the spaces the user belongs to (with admin flag)", async () => { + const core = coreStore(sql, coreSchema); + const spaceId = await core.createSpace(rand(12), "My Space"); + await core.addPrincipalToSpace(spaceId, userId, true); + + const res = await call<{ + spaces: { id: string; name: string; admin: boolean }[]; + }>("space.list", {}); + const mine = res.spaces.find((s) => s.id === spaceId); + expect(mine).toBeDefined(); + expect(mine?.name).toBe("My Space"); + expect(mine?.admin).toBe(true); + + // a brand-new user with no memberships sees no spaces + const other = await makeUser(); + const otherList = await call<{ spaces: unknown[] }>("space.list", {}, other); + expect(otherList.spaces).toHaveLength(0); +}); + +test("space.list includes spaces reached only via group membership", async () => { + const core = coreStore(sql, coreSchema); + const spaceId = await core.createSpace(rand(12), "Group Space"); + const groupId = await core.createGroup(spaceId, "team"); + // the user is NOT added to principal_space — only to a group in the space + await core.addGroupMember(spaceId, groupId, userId); + + const res = await call<{ spaces: { id: string; admin: boolean }[] }>( + "space.list", + {}, + ); + const mine = res.spaces.find((s) => s.id === spaceId); + expect(mine).toBeDefined(); // group membership confers space membership + expect(mine?.admin).toBe(false); // but not direct-membership admin +}); + +test("space.create provisions a space the caller owns + admins", async () => { + const res = await call<{ id: string; slug: string }>("space.create", { + name: "Fresh Space", + }); + createdSpaceSchemas.push(`me_${res.slug}`); + expect(res.slug).toMatch(/^[a-z0-9]{12}$/); + + // the me_ data schema was provisioned + const [row] = await sql.unsafe( + `select exists (select 1 from information_schema.schemata where schema_name = $1) as e`, + [`me_${res.slug}`], + ); + expect(Boolean(row?.e)).toBe(true); + + // it shows up in the caller's spaces, as admin + const list = await call<{ spaces: { id: string; admin: boolean }[] }>( + "space.list", + {}, + ); + expect(list.spaces.find((s) => s.id === res.id)?.admin).toBe(true); + + // the creator owns the shared root (and its home), not owner@root + const ta = await coreStore(sql, coreSchema).buildTreeAccess(userId, res.id); + expect(ta).toContainEqual({ tree_path: "share", access: ACCESS.owner }); + expect(ta).not.toContainEqual({ tree_path: ROOT_PATH, access: ACCESS.owner }); +}); + +test("space.rename renames; space.delete removes the space + schema", async () => { + const created = await call<{ id: string; slug: string }>("space.create", { + name: "Temp Space", + }); + const schema = `me_${created.slug}`; + createdSpaceSchemas.push(schema); + + // rename + expect( + ( + await call<{ renamed: boolean }>("space.rename", { + slug: created.slug, + name: "Renamed Space", + }) + ).renamed, + ).toBe(true); + const after = await call<{ spaces: { id: string; name: string }[] }>( + "space.list", + {}, + ); + expect(after.spaces.find((s) => s.id === created.id)?.name).toBe( + "Renamed Space", + ); + + // delete: core row gone + data schema dropped + expect( + (await call<{ deleted: boolean }>("space.delete", { slug: created.slug })) + .deleted, + ).toBe(true); + const gone = await call<{ spaces: { id: string }[] }>("space.list", {}); + expect(gone.spaces.some((s) => s.id === created.id)).toBe(false); + const [row] = await sql.unsafe( + `select exists (select 1 from information_schema.schemata where schema_name = $1) as e`, + [schema], + ); + expect(Boolean(row?.e)).toBe(false); +}); + +test("space.rename/delete require space admin", async () => { + const created = await call<{ id: string; slug: string }>("space.create", { + name: "Owned Space", + }); + createdSpaceSchemas.push(`me_${created.slug}`); + // a different user who is not a member/admin + const intruder = await makeUser(); + await expectAppError( + call("space.rename", { slug: created.slug, name: "Hijacked" }, intruder), + "FORBIDDEN", + ); + await expectAppError( + call("space.delete", { slug: created.slug }, intruder), + "FORBIDDEN", + ); +}); + +test("space.list reflects admin inherited via an admin group", async () => { + const core = coreStore(sql, coreSchema); + const spaceId = await core.createSpace(rand(12), "Admin Group Space"); + const groupId = await core.createGroup(spaceId, "admins"); + // designate the group itself as an admin member of the space + await core.addPrincipalToSpace(spaceId, groupId, true); + // the user is only in that group (no direct principal_space row) + await core.addGroupMember(spaceId, groupId, userId); + + const res = await call<{ spaces: { id: string; admin: boolean }[] }>( + "space.list", + {}, + ); + const mine = res.spaces.find((s) => s.id === spaceId); + expect(mine).toBeDefined(); + expect(mine?.admin).toBe(true); // admin transfers transitively through the group +}); diff --git a/packages/server/rpc/user/agent.ts b/packages/server/rpc/user/agent.ts new file mode 100644 index 0000000..406d876 --- /dev/null +++ b/packages/server/rpc/user/agent.ts @@ -0,0 +1,107 @@ +/** + * Agent handlers (agent.*) for the user RPC. + * + * Agents are a user's global service accounts. The lifecycle here is purely + * user-scoped: create / list / rename / delete the caller's own agents, and + * mint their (global) api keys (apiKey.* — see ./api-key.ts). Bringing an agent + * into a space (principal.add) is a space-endpoint operation. + */ +import type { Principal } from "@memory.build/engine/core"; +import type { + AgentCreateParams, + AgentCreateResult, + AgentDeleteParams, + AgentDeleteResult, + AgentListParams, + AgentListResult, + AgentRenameParams, + AgentRenameResult, + AgentResponse, +} from "@memory.build/protocol/user"; +import { + agentCreateParams, + agentDeleteParams, + agentListParams, + agentRenameParams, +} from "@memory.build/protocol/user"; +import { guardCore } from "../core-error"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +function toAgentResponse(p: Principal): AgentResponse { + return { + id: p.id, + name: p.name, + createdAt: p.createdAt.toISOString(), + updatedAt: p.updatedAt?.toISOString() ?? null, + }; +} + +/** Assert the caller owns this agent (globally). */ +export async function requireOwnAgent( + ctx: UserRpcContext, + agentId: string, +): Promise { + const principal = await ctx.core.getPrincipal(agentId); + if (!principal || principal.kind !== "a") { + throw new AppError("NOT_FOUND", `Agent not found: ${agentId}`); + } + if (principal.ownerId !== ctx.userId) { + throw new AppError("FORBIDDEN", "Not the owner of this agent"); + } +} + +async function agentCreate( + params: AgentCreateParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const id = await guardCore(() => + ctx.core.createAgent(ctx.userId, params.name), + ); + return { id }; +} + +async function agentList( + _params: AgentListParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const agents = await ctx.core.listAgents(ctx.userId); + return { agents: agents.map(toAgentResponse) }; +} + +async function agentRename( + params: AgentRenameParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireOwnAgent(ctx, params.id); + const renamed = await guardCore(() => + ctx.core.renamePrincipal(params.id, params.name), + ); + return { renamed }; +} + +async function agentDelete( + params: AgentDeleteParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireOwnAgent(ctx, params.id); + const deleted = await guardCore(() => ctx.core.deletePrincipal(params.id)); + return { deleted }; +} + +export const agentMethods = buildRegistry() + .register("agent.create", agentCreateParams, agentCreate) + .register("agent.list", agentListParams, agentList) + .register("agent.rename", agentRenameParams, agentRename) + .register("agent.delete", agentDeleteParams, agentDelete) + .build(); diff --git a/packages/server/rpc/user/api-key.integration.test.ts b/packages/server/rpc/user/api-key.integration.test.ts new file mode 100644 index 0000000..a192874 --- /dev/null +++ b/packages/server/rpc/user/api-key.integration.test.ts @@ -0,0 +1,154 @@ +// Integration test for the user RPC api-key handlers (apiKey.* lifecycle). +// Keys are agent-only and global: minting one needs only agent ownership — no +// space membership — and the key string carries no space slug. +// TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:5432/postgres" \ +// bun test --timeout 30000 \ +// packages/server/rpc/user/api-key.integration.test.ts +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import { coreStore } from "@memory.build/engine/core"; +import { type AppErrorCode, isAppError } from "@memory.build/protocol/errors"; +import postgres, { type Sql } from "postgres"; +import type { HandlerContext } from "../types"; +import { userMethods } from "./index"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = (n: number) => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(n)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +let sql: Sql; +let coreSchema: string; +let authSchema: string; +let userId: string; + +function call( + method: string, + params: unknown, + asUser: string = userId, +): Promise { + const registered = userMethods.get(method); + if (!registered) throw new Error(`no handler for ${method}`); + const context = { + request: new Request("http://localhost/api/v1/user/rpc"), + core: coreStore(sql, coreSchema), + auth: authStore(sql, authSchema), + userId: asUser, + db: sql, + coreSchema, + } as unknown as HandlerContext; + return registered.handler(params, context) as Promise; +} + +async function expectAppError(p: Promise, code: AppErrorCode) { + try { + await p; + throw new Error(`expected AppError(${code}), but it resolved`); + } catch (e) { + if (!isAppError(e)) throw e; + expect(e.code).toBe(code); + } +} + +async function makeUser(): Promise { + const [row] = await sql`select uuidv7() as id`; + const id = row?.id as string; + await coreStore(sql, coreSchema).createUser(id, `u_${rand(8)}@example.com`); + return id; +} + +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + coreSchema = `core_test_${rand(8)}`; + authSchema = `auth_test_${rand(8)}`; + await bootstrapSpaceDatabase(sql); + await migrateCore(sql, { schema: coreSchema }); + await migrateAuth(sql, { schema: authSchema }); +}); + +afterAll(async () => { + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.end(); +}); + +beforeEach(async () => { + userId = await makeUser(); +}); + +test("create (global, no space needed) / list / get / delete", async () => { + // The agent is owned by the caller but is NOT a member of any space — key + // creation depends only on ownership, not space membership. + const { id: agentId } = await call<{ id: string }>("agent.create", { + name: "bot", + }); + + const created = await call<{ id: string; key: string }>("apiKey.create", { + agentId, + name: "ci", + expiresAt: null, + }); + // Global format: me.. — no space slug. + expect(created.key).toMatch(/^me\.[A-Za-z0-9_-]{16}\.[A-Za-z0-9_-]{32}$/); + + const list = await call<{ apiKeys: { id: string }[] }>("apiKey.list", { + memberId: agentId, + }); + expect(list.apiKeys.map((k) => k.id)).toContain(created.id); + + const got = await call<{ apiKey: { id: string } | null }>("apiKey.get", { + id: created.id, + }); + expect(got.apiKey?.id).toBe(created.id); + + expect( + (await call<{ deleted: boolean }>("apiKey.delete", { id: created.id })) + .deleted, + ).toBe(true); + expect( + (await call<{ apiKey: unknown }>("apiKey.get", { id: created.id })).apiKey, + ).toBeNull(); +}); + +test("apiKey.create rejects a non-agent member", async () => { + // a user id is not an agent → NOT_FOUND + await expectAppError( + call("apiKey.create", { agentId: userId, name: "nope", expiresAt: null }), + "NOT_FOUND", + ); +}); + +test("cannot manage keys for another user's agent", async () => { + const { id: agentId } = await call<{ id: string }>("agent.create", { + name: "mine", + }); + const intruder = await makeUser(); + await expectAppError( + call("apiKey.create", { agentId, name: "x", expiresAt: null }, intruder), + "FORBIDDEN", + ); + await expectAppError( + call("apiKey.list", { memberId: agentId }, intruder), + "FORBIDDEN", + ); +}); + +test("apiKey.get is null for an unknown key id", async () => { + const [row] = await sql`select uuidv7() as id`; + const got = await call<{ apiKey: unknown }>("apiKey.get", { + id: row?.id as string, + }); + expect(got.apiKey).toBeNull(); +}); diff --git a/packages/server/rpc/user/api-key.ts b/packages/server/rpc/user/api-key.ts new file mode 100644 index 0000000..83c4f0f --- /dev/null +++ b/packages/server/rpc/user/api-key.ts @@ -0,0 +1,106 @@ +/** + * Api key handlers (apiKey.*) for the user RPC. + * + * Keys are agent-only and self-service: the caller manages keys for agents they + * own. Keys are global per-principal (not space-bound) — the same key works in + * any space the agent is admitted to. The plaintext key is returned once by + * create. Revoke ≡ delete (no soft-revoke state). + */ +import type { ApiKeyInfo } from "@memory.build/engine/core"; +import { formatApiKey } from "@memory.build/engine/core"; +import type { + ApiKeyCreateParams, + ApiKeyCreateResult, + ApiKeyDeleteParams, + ApiKeyDeleteResult, + ApiKeyGetParams, + ApiKeyGetResult, + ApiKeyInfoResponse, + ApiKeyListParams, + ApiKeyListResult, +} from "@memory.build/protocol/user"; +import { + apiKeyCreateParams, + apiKeyDeleteParams, + apiKeyGetParams, + apiKeyListParams, +} from "@memory.build/protocol/user"; +import { guardCore } from "../core-error"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { requireOwnAgent } from "./agent"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +function toApiKeyInfoResponse(k: ApiKeyInfo): ApiKeyInfoResponse { + return { + id: k.id, + memberId: k.memberId, + lookupId: k.lookupId, + name: k.name, + createdAt: k.createdAt.toISOString(), + expiresAt: k.expiresAt?.toISOString() ?? null, + }; +} + +async function apiKeyCreate( + params: ApiKeyCreateParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + // Keys are agent-only; the caller must own the agent (checked globally). + await requireOwnAgent(ctx, params.agentId); + + const created = await guardCore(() => + ctx.core.createApiKey(params.agentId, params.name, { + expiresAt: params.expiresAt ? new Date(params.expiresAt) : undefined, + }), + ); + // The full key string is global (no space slug); returned once. + const key = formatApiKey(created.lookupId, created.secret); + return { id: created.id, key }; +} + +async function apiKeyList( + params: ApiKeyListParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireOwnAgent(ctx, params.memberId); + const keys = await ctx.core.listApiKeys(params.memberId); + return { apiKeys: keys.map(toApiKeyInfoResponse) }; +} + +async function apiKeyGet( + params: ApiKeyGetParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const key = await ctx.core.getApiKey(params.id); + if (!key) return { apiKey: null }; + // Only the owning user of the key's agent may see it. + await requireOwnAgent(ctx, key.memberId); + return { apiKey: toApiKeyInfoResponse(key) }; +} + +async function apiKeyDelete( + params: ApiKeyDeleteParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const key = await ctx.core.getApiKey(params.id); + if (!key) return { deleted: false }; + await requireOwnAgent(ctx, key.memberId); + const deleted = await guardCore(() => ctx.core.deleteApiKey(params.id)); + return { deleted }; +} + +export const apiKeyMethods = buildRegistry() + .register("apiKey.create", apiKeyCreateParams, apiKeyCreate) + .register("apiKey.list", apiKeyListParams, apiKeyList) + .register("apiKey.get", apiKeyGetParams, apiKeyGet) + .register("apiKey.delete", apiKeyDeleteParams, apiKeyDelete) + .build(); diff --git a/packages/server/rpc/user/index.ts b/packages/server/rpc/user/index.ts new file mode 100644 index 0000000..87e60b9 --- /dev/null +++ b/packages/server/rpc/user/index.ts @@ -0,0 +1,26 @@ +/** + * User RPC method registry — served at `/api/v1/user/rpc` (session-only, + * user-scoped): the lifecycle of a user's agents and their global api keys. + */ +import type { MethodRegistry } from "../types"; +import { agentMethods } from "./agent"; +import { apiKeyMethods } from "./api-key"; +import { spaceMethods } from "./space"; +import { whoamiMethods } from "./whoami"; + +export { + assertUserRpcContext, + isUserRpcContext, + type UserRpcContext, +} from "./types"; + +/** + * The user-endpoint registry: identity + agent lifecycle + api keys + space + * discovery. + */ +export const userMethods: MethodRegistry = new Map([ + ...whoamiMethods, + ...agentMethods, + ...apiKeyMethods, + ...spaceMethods, +]); diff --git a/packages/server/rpc/user/space.ts b/packages/server/rpc/user/space.ts new file mode 100644 index 0000000..388284f --- /dev/null +++ b/packages/server/rpc/user/space.ts @@ -0,0 +1,148 @@ +/** + * Space handlers (space.*) for the user RPC. + * + * User-scoped space discovery: the spaces the calling user belongs to. The CLI + * uses this to pick the X-Me-Space that scopes the rest of its commands. + */ +import { + generateSlug, + provisionSpace, + slugToSchema, +} from "@memory.build/database"; +import { + coreStore, + type MemberSpace, + type Space, +} from "@memory.build/engine/core"; +import type { + MemberSpaceResponse, + SpaceCreateParams, + SpaceCreateResult, + SpaceDeleteParams, + SpaceDeleteResult, + SpaceListParams, + SpaceListResult, + SpaceRenameParams, + SpaceRenameResult, +} from "@memory.build/protocol/user"; +import { + spaceCreateParams, + spaceDeleteParams, + spaceListParams, + spaceRenameParams, +} from "@memory.build/protocol/user"; +import type { Sql } from "postgres"; +import { addSpaceCreator } from "../../provision"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +/** Resolve a space by slug and require the caller to be its admin. */ +async function requireSpaceAdminFor( + ctx: UserRpcContext, + slug: string, +): Promise { + const space = await ctx.core.getSpace(slug); + if (!space) { + throw new AppError("NOT_FOUND", `Space not found: ${slug}`); + } + if (!(await ctx.core.isSpaceAdmin(ctx.userId, space.id))) { + throw new AppError("FORBIDDEN", "This action requires being a space admin"); + } + return space; +} + +function toMemberSpaceResponse(s: MemberSpace): MemberSpaceResponse { + return { + id: s.id, + slug: s.slug, + name: s.name, + language: s.language, + admin: s.admin, + createdAt: s.createdAt.toISOString(), + updatedAt: s.updatedAt?.toISOString() ?? null, + }; +} + +async function spaceList( + _params: SpaceListParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const spaces = await ctx.core.listSpacesForMember(ctx.userId); + return { spaces: spaces.map(toMemberSpaceResponse) }; +} + +/** + * Create a new space. The creator becomes a space admin and owner of its own + * home (via add_principal_to_space) and of the shared root (`share`) — but NOT + * owner@root, so it sees `/share` and `~` but not other members' homes. As an + * admin it can self-grant owner@root later if it wants the whole tree. Atomic: + * the core.space row, the me_ data schema, the membership, and the grant + * all land in one transaction (any failure rolls the schema back). + */ +async function spaceCreate( + params: SpaceCreateParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const slug = generateSlug(); + + const id = (await ctx.db.begin(async (tx) => { + const core = coreStore(tx as unknown as Sql, ctx.coreSchema); + const spaceId = await core.createSpace(slug, params.name); + await provisionSpace(tx, { slug }); // creates the me_ data schema + await addSpaceCreator(core, spaceId, ctx.userId); + return spaceId; + })) as string; + + return { id, slug }; +} + +/** Rename a space's display name (admin only); the slug is immutable. */ +async function spaceRename( + params: SpaceRenameParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + await requireSpaceAdminFor(ctx, params.slug); + const renamed = await ctx.core.renameSpace(params.slug, params.name); + return { renamed }; +} + +/** + * Delete a space (admin only): drop its core row (cascading memberships/groups/ + * grants) and its me_ data schema, atomically. + */ +async function spaceDelete( + params: SpaceDeleteParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const space = await requireSpaceAdminFor(ctx, params.slug); + + const deleted = (await ctx.db.begin(async (tx) => { + const core = coreStore(tx as unknown as Sql, ctx.coreSchema); + const ok = await core.deleteSpace(space.slug); + // slug came from the DB (validated by the slug check constraint); safe to + // interpolate into the DDL. + await tx.unsafe( + `drop schema if exists ${slugToSchema(space.slug)} cascade`, + ); + return ok; + })) as boolean; + + return { deleted }; +} + +export const spaceMethods = buildRegistry() + .register("space.list", spaceListParams, spaceList) + .register("space.create", spaceCreateParams, spaceCreate) + .register("space.rename", spaceRenameParams, spaceRename) + .register("space.delete", spaceDeleteParams, spaceDelete) + .build(); diff --git a/packages/server/rpc/user/types.ts b/packages/server/rpc/user/types.ts new file mode 100644 index 0000000..16a470f --- /dev/null +++ b/packages/server/rpc/user/types.ts @@ -0,0 +1,39 @@ +/** + * User RPC context — populated by authenticateUser. User-scoped (no space): + * the calling user manages their own global service accounts (agents). + */ +import type { AuthStore } from "@memory.build/auth"; +import type { CoreStore } from "@memory.build/engine/core"; +import type { Sql } from "postgres"; +import type { HandlerContext } from "../types"; + +export interface UserRpcContext extends HandlerContext { + /** Core control-plane store. */ + core: CoreStore; + /** Auth store (auth schema) — for the caller's identity (whoami). */ + auth: AuthStore; + /** The authenticated user id (== the core user-principal id). */ + userId: string; + /** New-model pool — for transactional provisioning (space.create). */ + db: Sql; + /** The core control-plane schema name. */ + coreSchema: string; +} + +export function isUserRpcContext(ctx: HandlerContext): ctx is UserRpcContext { + return ( + "core" in ctx && + typeof ctx.core === "object" && + ctx.core !== null && + "userId" in ctx && + typeof ctx.userId === "string" + ); +} + +export function assertUserRpcContext( + ctx: HandlerContext, +): asserts ctx is UserRpcContext { + if (!isUserRpcContext(ctx)) { + throw new Error("User context not initialized (authentication required)"); + } +} diff --git a/packages/server/rpc/user/whoami.ts b/packages/server/rpc/user/whoami.ts new file mode 100644 index 0000000..15830ca --- /dev/null +++ b/packages/server/rpc/user/whoami.ts @@ -0,0 +1,27 @@ +/** + * whoami handler for the user RPC — the identity behind the session token. + */ +import type { WhoamiParams, WhoamiResult } from "@memory.build/protocol/user"; +import { whoamiParams } from "@memory.build/protocol/user"; +import { AppError } from "../errors"; +import { buildRegistry } from "../registry"; +import type { HandlerContext } from "../types"; +import { assertUserRpcContext, type UserRpcContext } from "./types"; + +async function whoami( + _params: WhoamiParams, + context: HandlerContext, +): Promise { + assertUserRpcContext(context); + const ctx = context as UserRpcContext; + const user = await ctx.auth.getUser(ctx.userId); + if (!user) { + // The session validated but the user row is gone — treat as unauthenticated. + throw new AppError("UNAUTHORIZED", "User not found"); + } + return { id: user.id, email: user.email, name: user.name }; +} + +export const whoamiMethods = buildRegistry() + .register("whoami", whoamiParams, whoami) + .build(); diff --git a/packages/server/server.integration.test.ts b/packages/server/server.integration.test.ts index f9b0fd2..9cb7e73 100644 --- a/packages/server/server.integration.test.ts +++ b/packages/server/server.integration.test.ts @@ -1,7 +1,8 @@ import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { SQL } from "bun"; +import type { CoreStore } from "@memory.build/engine/core"; +import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; import { MAX_BODY_SIZE } from "./middleware/size-limit"; @@ -13,22 +14,27 @@ let baseUrl: string; // Mock ServerContext for testing function createMockContext(): ServerContext { return { - accountsDb: { + db: {} as Sql, + auth: { + // Session validation: no session → user RPC stays 401. validateSession: mock(() => Promise.resolve(null)), - getEngineBySlug: mock(() => Promise.resolve(null)), - // Device auth operations for auth endpoint tests - create: mock(() => Promise.resolve({})), - getByDeviceCode: mock(() => Promise.resolve(null)), - getByUserCode: mock(() => Promise.resolve(null)), - getByOAuthState: mock(() => Promise.resolve(null)), - updateLastPoll: mock(() => Promise.resolve(null)), - authorize: mock(() => Promise.resolve(false)), - deny: mock(() => Promise.resolve(false)), - delete: mock(() => Promise.resolve(false)), - deleteExpired: mock(() => Promise.resolve(0)), - } as unknown as AccountsDB, - accountsSql: {} as SQL, - engineSql: {} as SQL, + // Device flow operations exercised by the auth endpoint tests. + createDeviceAuth: mock((_provider: string) => + Promise.resolve({ + deviceCode: "test-device-code", + userCode: "WXYZ-2345", + oauthState: "test-oauth-state", + expiresIn: 900, + }), + ), + // Unknown device codes poll as expired. + pollDevice: mock(() => + Promise.resolve({ status: "expired", userId: null }), + ), + } as unknown as AuthStore, + core: {} as unknown as CoreStore, + authSchema: "auth", + coreSchema: "core", embeddingConfig: { provider: "openai", model: "text-embedding-3-small", @@ -115,7 +121,7 @@ describe("server integration", () => { test("rejects requests with oversized Content-Length header", async () => { // Create a request with a misleading Content-Length header // In real scenarios, the header would match the actual body - const request = new Request(`${baseUrl}/api/v1/accounts/rpc`, { + const request = new Request(`${baseUrl}/api/v1/memory/rpc`, { method: "POST", headers: { "Content-Length": String(MAX_BODY_SIZE + 1), @@ -131,12 +137,12 @@ describe("server integration", () => { }); test("allows normal sized requests", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`, { + const response = await fetch(`${baseUrl}/api/v1/user/rpc`, { method: "POST", headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ jsonrpc: "2.0", method: "test", id: 1 }), + body: JSON.stringify({ jsonrpc: "2.0", method: "whoami", id: 1 }), }); // Should get 401 (auth required) not 413 (size limit) // Auth is checked after size limit passes @@ -145,37 +151,38 @@ describe("server integration", () => { }); describe("RPC endpoints", () => { - test("POST /api/v1/accounts/rpc returns 401 without auth", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`, { + test("POST /api/v1/memory/rpc returns 401 without auth", async () => { + const response = await fetch(`${baseUrl}/api/v1/memory/rpc`, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + "X-Me-Space": "abc123def456", + }, body: JSON.stringify({}), }); // Auth is required before JSON-RPC processing expect(response.status).toBe(401); }); - test("POST /api/v1/accounts/rpc returns 401 for unauthenticated requests", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`, { + test("POST /api/v1/memory/rpc returns 400 when X-Me-Space is missing", async () => { + const response = await fetch(`${baseUrl}/api/v1/memory/rpc`, { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - jsonrpc: "2.0", - method: "unknown.method", - id: 1, - }), + headers: { + "Content-Type": "application/json", + Authorization: "Bearer some-token", + }, + body: JSON.stringify({ jsonrpc: "2.0", method: "memory.get", id: 1 }), }); - // Auth is required before method lookup - expect(response.status).toBe(401); + expect(response.status).toBe(400); }); - test("POST /api/v1/engine/rpc returns 401 without auth", async () => { - const response = await fetch(`${baseUrl}/api/v1/engine/rpc`, { + test("POST /api/v1/user/rpc returns 401 for unauthenticated requests", async () => { + const response = await fetch(`${baseUrl}/api/v1/user/rpc`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", - method: "unknown.method", + method: "whoami", id: 1, }), }); @@ -183,8 +190,8 @@ describe("server integration", () => { expect(response.status).toBe(401); }); - test("GET /api/v1/accounts/rpc returns 404 (wrong method)", async () => { - const response = await fetch(`${baseUrl}/api/v1/accounts/rpc`); + test("GET /api/v1/memory/rpc returns 404 (wrong method)", async () => { + const response = await fetch(`${baseUrl}/api/v1/memory/rpc`); expect(response.status).toBe(404); }); }); diff --git a/packages/server/start.integration.test.ts b/packages/server/start.integration.test.ts new file mode 100644 index 0000000..e2c63bb --- /dev/null +++ b/packages/server/start.integration.test.ts @@ -0,0 +1,173 @@ +// Integration test for the extracted startServer() bootstrap. +// +// Boots the real server stack (pools → migrate → worker → Bun.serve) against +// isolated auth/core test schemas on a port-0 listener, then hits /health and +// /ready. No real embeddings are exercised (a placeholder key suffices — the +// worker idles), so this needs no OpenAI key. +// TEST_DATABASE_URL="$(ghost connect testing_me)" \ +// bun test --timeout 30000 packages/server/start.integration.test.ts + +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, +} from "@memory.build/database"; +import type { EmbeddingConfig } from "@memory.build/embedding"; +import postgres, { type Sql } from "postgres"; +import { startServer } from "./lib"; +import { provisionUser } from "./provision"; + +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; + +const rand = () => { + const a = "abcdefghijklmnopqrstuvwxyz0123456789"; + const bytes = crypto.getRandomValues(new Uint8Array(8)); + let s = ""; + for (const b of bytes) s += a[b % 36]; + return s; +}; + +const embeddingConfig: EmbeddingConfig = { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + apiKey: "test-key-not-used", + options: {}, +}; + +let sql: Sql; +let srv: Awaited>; +let authSchema: string; +let coreSchema: string; +let spaceSchema: string; +let tamperedDef: string; +let bootedDef: string; +let prevSchemaPrefix: string | undefined; + +/** Current definition of a space schema's create_memory function. */ +async function createMemoryDef(schema: string): Promise { + const [row] = await sql` + select pg_get_functiondef(p.oid) as def + from pg_proc p + join pg_namespace n on n.oid = p.pronamespace + where n.nspname = ${schema} and p.proname = 'create_memory'`; + return (row?.def as string) ?? ""; +} + +beforeAll(async () => { + // Space schemas created by this suite land under metest_ (not + // production me_) so leftovers are reclaimable by name. Scoped to + // this suite and restored in afterAll: CI runs every integration file in + // ONE bun process (`find … | xargs bun test`), so a module-scope + // assignment would leak the prefix into suites that expect me_. + prevSchemaPrefix = process.env.SPACE_SCHEMA_PREFIX; + process.env.SPACE_SCHEMA_PREFIX = "metest_"; + + authSchema = `auth_test_${rand()}`; + coreSchema = `core_test_${rand()}`; + sql = postgres(URL, { onnotice: () => {} }); + + // Simulate an existing deployment: a fully provisioned space whose + // create_memory predates the current idempotent SQL. Migrate the control + // plane, provision a user (+ its default space), then tamper the space's + // create_memory so we can prove boot re-applies the real definition. + await bootstrapSpaceDatabase(sql); + await migrateCore(sql, { schema: coreSchema }); + await migrateAuth(sql, { schema: authSchema }); + const provisioned = await provisionUser( + sql, + { auth: authSchema, core: coreSchema }, + { + email: "boot@example.test", + name: "Boot", + provider: "github", + accountId: `boot-${rand()}`, + emailVerified: true, + }, + ); + spaceSchema = `metest_${provisioned.spaceSlug}`; + await sql.unsafe(` + create or replace function ${spaceSchema}.create_memory + ( _tree_access jsonb + , _tree ltree + , _content text + , _id uuid default null + , _meta jsonb default '{}' + , _temporal tstzrange default null + ) + returns uuid + as $func$ + begin + return null; -- stale stand-in, must be replaced by the boot sweep + end; + $func$ language plpgsql volatile + `); + tamperedDef = await createMemoryDef(spaceSchema); + + srv = await startServer({ + port: 0, + databaseUrl: URL, + apiBaseUrl: "http://localhost", + authSchema, + coreSchema, + embeddingConfig, + workerCount: 1, + workerIdleDelayMs: 250, + workerRefreshIntervalMs: 500, + enableCleanupCron: false, + // migrate defaults to true — startServer migrates the isolated schemas + // and re-migrates the pre-existing space. + }); + bootedDef = await createMemoryDef(spaceSchema); +}); + +afterAll(async () => { + await srv?.stop(); + await sql.unsafe(`drop schema if exists ${spaceSchema} cascade`); + await sql.unsafe(`drop schema if exists ${authSchema} cascade`); + await sql.unsafe(`drop schema if exists ${coreSchema} cascade`); + await sql.end(); + if (prevSchemaPrefix === undefined) { + delete process.env.SPACE_SCHEMA_PREFIX; + } else { + process.env.SPACE_SCHEMA_PREFIX = prevSchemaPrefix; + } +}); + +test("boots on a random port and serves /health", async () => { + expect(srv.port).toBeGreaterThan(0); + const res = await fetch(`${srv.url}/health`); + expect(res.status).toBe(200); + expect(await res.text()).toBe("ok"); +}); + +test("/ready reports the database is reachable", async () => { + const res = await fetch(`${srv.url}/ready`); + expect(res.status).toBe(200); +}); + +test("migrated the configured isolated schemas", async () => { + const [authRow] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${authSchema} + ) as e`; + const [coreRow] = await sql` + select exists ( + select 1 from information_schema.schemata where schema_name = ${coreSchema} + ) as e`; + expect(Boolean(authRow?.e)).toBe(true); + expect(Boolean(coreRow?.e)).toBe(true); +}); + +test("re-migrates existing space schemas on boot", async () => { + // The tampered function was in place just before boot… + expect(tamperedDef).toContain("stale stand-in"); + expect(tamperedDef).not.toContain("on conflict"); + // …and boot's space sweep re-applied the idempotent SQL over it + // (create_memory is the one-row wrapper delegating to batch_create_memory). + expect(bootedDef).not.toContain("stale stand-in"); + expect(bootedDef).toContain("batch_create_memory"); +}); diff --git a/packages/server/start.ts b/packages/server/start.ts new file mode 100644 index 0000000..c5977b2 --- /dev/null +++ b/packages/server/start.ts @@ -0,0 +1,501 @@ +// packages/server/start.ts +// +// Callable server bootstrap. `startServer()` stands up the same stack the +// production entrypoint (index.ts) runs — pools → bootstrap/migrate → router → +// worker pool → Bun.serve — but with **no** process-level side effects +// (no SIGINT/SIGTERM/unhandledRejection handlers, no process.exit, no +// telemetry configure()). It returns a `RunningServer` handle whose `stop()` +// tears everything down. index.ts is the thin entrypoint that wraps this with +// telemetry + signal handling; the e2e harness calls it directly. +import { authStore } from "@memory.build/auth"; +import { + bootstrapSpaceDatabase, + migrateAuth, + migrateCore, + migrateSpace, + slugToSchema as spaceSlugToSchema, +} from "@memory.build/database"; +import type { EmbeddingConfig } from "@memory.build/embedding"; +import { type CoreStore, coreStore } from "@memory.build/engine/core"; +import { + DEFAULT_WORKER_TIMEOUTS, + WorkerPool, + type WorkerTimeouts, +} from "@memory.build/worker"; +import { info, reportError, span } from "@pydantic/logfire-node"; +import postgres, { type Sql } from "postgres"; +import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; +import { embeddingConstants } from "./config"; +import type { ServerContext } from "./context"; +import { checkSizeLimit } from "./middleware"; +import { createRouter } from "./router"; +import { internalError } from "./util/response"; + +/** + * Parse an integer from an environment variable with NaN guard. + */ +function parseIntEnv( + name: string, + value: string, + defaultValue: string, +): number { + const raw = value || defaultValue; + const parsed = parseInt(raw, 10); + if (Number.isNaN(parsed)) { + throw new Error( + `Invalid value for ${name}: "${raw}" is not a valid integer`, + ); + } + return parsed; +} + +/** + * Build the embedding config from environment variables. Requires + * EMBEDDING_API_KEY (the server won't boot without it). Used as the default + * when `StartServerOptions.embeddingConfig` is not supplied. + */ +export function buildEmbeddingConfig(): EmbeddingConfig { + const apiKey = process.env.EMBEDDING_API_KEY; + if (!apiKey) { + throw new Error("EMBEDDING_API_KEY is required"); + } + + const options: EmbeddingConfig["options"] = {}; + + if (process.env.EMBEDDING_TIMEOUT_MS) { + options.timeoutMs = parseIntEnv( + "EMBEDDING_TIMEOUT_MS", + process.env.EMBEDDING_TIMEOUT_MS, + "0", + ); + } + if (process.env.EMBEDDING_MAX_RETRIES) { + options.maxRetries = parseIntEnv( + "EMBEDDING_MAX_RETRIES", + process.env.EMBEDDING_MAX_RETRIES, + "0", + ); + } + if (process.env.EMBEDDING_MAX_PARALLEL_CALLS) { + options.maxParallelCalls = parseIntEnv( + "EMBEDDING_MAX_PARALLEL_CALLS", + process.env.EMBEDDING_MAX_PARALLEL_CALLS, + "0", + ); + } + + return { + provider: "openai", + model: embeddingConstants.model, + dimensions: embeddingConstants.dimensions, + apiKey, + baseUrl: process.env.EMBEDDING_BASE_URL, + options, + }; +} + +export interface StartServerOptions { + /** HTTP port. Default PORT env or 3000; 0 = OS-assigned random port. */ + port?: number; + /** Application pool connection string. Default DATABASE_URL. */ + databaseUrl?: string; + /** Worker pool connection string. Default WORKER_DATABASE_URL ?? databaseUrl. */ + workerDatabaseUrl?: string; + /** Public URL for OAuth callbacks. Default API_BASE_URL. */ + apiBaseUrl?: string; + /** Auth schema name. Default AUTH_SCHEMA ?? "auth". */ + authSchema?: string; + /** Core control-plane schema name. Default CORE_SCHEMA ?? "core". */ + coreSchema?: string; + /** Embedding config. Default buildEmbeddingConfig() (reads env). */ + embeddingConfig?: EmbeddingConfig; + /** Number of concurrent embedding workers. Default WORKER_COUNT ?? 2. */ + workerCount?: number; + /** Worker idle poll interval in ms. Default WORKER_IDLE_DELAY_MS ?? 10000. */ + workerIdleDelayMs?: number; + /** Worker space-rediscovery interval in ms. Default WORKER_REFRESH_INTERVAL_MS ?? 60000. */ + workerRefreshIntervalMs?: number; + /** Run the device-flow/session cleanup cron. Default true; harness sets false. */ + enableCleanupCron?: boolean; + /** Run bootstrap + migrate on boot. Default true. */ + migrate?: boolean; +} + +export interface RunningServer { + /** e.g. http://localhost: */ + url: string; + port: number; + context: ServerContext; + /** Tear down: workerPool.stop → cron.stop → server.stop → pools.end. */ + stop(): Promise; +} + +/** + * Re-migrate every existing space schema at boot. + * + * Spaces are otherwise migrated only once, at provision time — so a deploy + * that changes the idempotent space SQL (the function bodies in + * space/migrate/idempotent/*.sql) would never reach existing spaces without + * this sweep. Re-running is cheap: incremental migrations are version-tracked + * no-ops, idempotent files are re-applied (create or replace). Options mirror + * provisionSpace (all defaults), and migrateSpace's per-schema advisory lock + * serializes concurrent replica boots. + * + * Every space is attempted (so one broken space doesn't hide the rest from + * the logs), then any failure aborts boot — the server must not serve spaces + * whose schema may be stale. + */ +async function remigrateSpaces(db: Sql, core: CoreStore): Promise { + const spaces = await core.listSpaces(); + const failed: string[] = []; + for (const space of spaces) { + try { + await migrateSpace(db, { slug: space.slug }); + } catch (error) { + failed.push(space.slug); + reportError( + `Space ${space.slug} re-migration failed`, + error instanceof Error ? error : new Error(String(error)), + ); + } + } + if (failed.length > 0) { + throw new Error( + `space re-migration failed for ${failed.length} of ${spaces.length} space(s): ${failed.join(", ")}`, + ); + } + info(`${spaces.length} space schema(s) re-migrated`); +} + +/** + * Boot the server stack and return a handle. No process-level side effects — + * the caller owns signal handling and process exit (index.ts does this). + */ +export async function startServer( + opts: StartServerOptions = {}, +): Promise { + const port = + opts.port ?? (process.env.PORT ? Number(process.env.PORT) : 3000); + + // TEMPORARY: fall back to the legacy ENGINE_DATABASE_URL so the multiplayer + // branch can deploy to dev before the tiger-agents-deploy helm values are + // migrated to the single-DB env contract. Remove once the deploy config sets + // DATABASE_URL directly. + const databaseUrl = + opts.databaseUrl ?? + process.env.DATABASE_URL ?? + process.env.ENGINE_DATABASE_URL; + if (!databaseUrl) { + throw new Error( + "DATABASE_URL (or legacy ENGINE_DATABASE_URL) environment variable is required", + ); + } + + const apiBaseUrl = opts.apiBaseUrl ?? process.env.API_BASE_URL; + if (!apiBaseUrl) { + throw new Error("API_BASE_URL environment variable is required"); + } + + const deviceFlowCleanupCron = + process.env.DEVICE_FLOW_CLEANUP_CRON || "*/15 * * * *"; + + // Schema names (single DB, postgres.js pool): auth + core control plane. + const authSchema = opts.authSchema ?? process.env.AUTH_SCHEMA ?? "auth"; + const coreSchema = opts.coreSchema ?? process.env.CORE_SCHEMA ?? "core"; + + const workerCount = + opts.workerCount ?? + parseIntEnv("WORKER_COUNT", process.env.WORKER_COUNT || "", "2"); + + // Connection pool settings - database + const dbPoolMax = parseIntEnv( + "DB_POOL_MAX", + process.env.DB_POOL_MAX || "", + "20", + ); + const dbPoolIdleReapSeconds = parseIntEnv( + "DB_POOL_IDLE_REAP_SECONDS", + process.env.DB_POOL_IDLE_REAP_SECONDS || "", + "300", + ); + const dbPoolMaxLifetime = parseIntEnv( + "DB_POOL_MAX_LIFETIME", + process.env.DB_POOL_MAX_LIFETIME || "", + "0", + ); + const dbPoolConnectionTimeout = parseIntEnv( + "DB_POOL_CONNECTION_TIMEOUT", + process.env.DB_POOL_CONNECTION_TIMEOUT || "", + "30", + ); + + // Connection pool settings - embedding worker database + const workerDatabaseUrl = + opts.workerDatabaseUrl ?? process.env.WORKER_DATABASE_URL ?? databaseUrl; + const workerDbPoolMax = parseIntEnv( + "WORKER_DB_POOL_MAX", + process.env.WORKER_DB_POOL_MAX || "", + String(Math.max(workerCount, 1)), + ); + const workerDbPoolIdleReapSeconds = parseIntEnv( + "WORKER_DB_POOL_IDLE_REAP_SECONDS", + process.env.WORKER_DB_POOL_IDLE_REAP_SECONDS || "", + String(dbPoolIdleReapSeconds), + ); + const workerDbPoolMaxLifetime = parseIntEnv( + "WORKER_DB_POOL_MAX_LIFETIME", + process.env.WORKER_DB_POOL_MAX_LIFETIME || "", + String(dbPoolMaxLifetime), + ); + const workerDbPoolConnectionTimeout = parseIntEnv( + "WORKER_DB_POOL_CONNECTION_TIMEOUT", + process.env.WORKER_DB_POOL_CONNECTION_TIMEOUT || "", + String(dbPoolConnectionTimeout), + ); + const workerTimeouts: WorkerTimeouts = { + statementTimeout: + process.env.WORKER_DB_STATEMENT_TIMEOUT ?? + DEFAULT_WORKER_TIMEOUTS.statementTimeout, + lockTimeout: + process.env.WORKER_DB_LOCK_TIMEOUT ?? DEFAULT_WORKER_TIMEOUTS.lockTimeout, + transactionTimeout: + process.env.WORKER_DB_TRANSACTION_TIMEOUT ?? + DEFAULT_WORKER_TIMEOUTS.transactionTimeout, + idleInTransactionSessionTimeout: + process.env.WORKER_DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT ?? + DEFAULT_WORKER_TIMEOUTS.idleInTransactionSessionTimeout, + }; + + const embeddingConfig = opts.embeddingConfig ?? buildEmbeddingConfig(); + + // OAuth provider validation: warn at startup if none configured, rather than + // failing with a confusing error when someone tries to log in. + const configuredProviders: string[] = []; + if (process.env.GITHUB_CLIENT_ID && process.env.GITHUB_CLIENT_SECRET) { + configuredProviders.push("github"); + } + if (process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) { + configuredProviders.push("google"); + } + if (configuredProviders.length === 0) { + console.warn( + "WARNING: No OAuth providers configured. Set GITHUB_CLIENT_ID/GITHUB_CLIENT_SECRET or GOOGLE_CLIENT_ID/GOOGLE_CLIENT_SECRET.", + ); + } else { + info("OAuth providers configured", { providers: configuredProviders }); + } + + // --------------------------------------------------------------------------- + // Database Pools + // --------------------------------------------------------------------------- + + // Dedicated worker pool (postgres.js) — the embedding worker processes the + // per-space me_ schemas. + const workerDb = postgres(workerDatabaseUrl, { + max: workerDbPoolMax, + idle_timeout: workerDbPoolIdleReapSeconds, + max_lifetime: workerDbPoolMaxLifetime, + connect_timeout: workerDbPoolConnectionTimeout, + onnotice: () => {}, + }); + + // The single application pool (postgres.js): the auth + core control plane and + // the per-space me_ data schemas all live in one database, one pool. + const db = postgres(databaseUrl, { + max: dbPoolMax, + idle_timeout: dbPoolIdleReapSeconds, + max_lifetime: dbPoolMaxLifetime, + connect_timeout: dbPoolConnectionTimeout, + onnotice: () => {}, + }); + + // Auth store (auth schema) on the application pool. + const auth = authStore(db, authSchema); + + // Core control-plane store (core schema) on the same pool. + const core = coreStore(db, coreSchema); + + // --------------------------------------------------------------------------- + // Database Bootstrap & Migrations (blocking — server won't serve until current) + // --------------------------------------------------------------------------- + + // Prepare the database for per-space schemas (extensions + roles, idempotent) + // and migrate the auth + core control-plane schemas on the application pool. + // Pass the configured schemas to BOTH the migrations and the stores so an + // isolated (non-default) schema is migrated where the stores read it. + if (opts.migrate ?? true) { + await bootstrapSpaceDatabase(db); + await migrateCore(db, { schema: coreSchema }); + await migrateAuth(db, { schema: authSchema }); + info("Core + auth schemas migrated"); + await remigrateSpaces(db, core); + } + + // --------------------------------------------------------------------------- + // Router + // --------------------------------------------------------------------------- + + const context: ServerContext = { + db, + auth, + core, + authSchema, + coreSchema, + embeddingConfig, + apiBaseUrl, + serverVersion: SERVER_VERSION, + minClientVersion: MIN_CLIENT_VERSION, + }; + + const router = createRouter(context); + + // --------------------------------------------------------------------------- + // Embedding Worker Pool + // --------------------------------------------------------------------------- + + const workerPool = new WorkerPool(workerDb, { + embedding: embeddingConfig, + discover: async () => { + const spaces = await core.listSpaces(); + return spaces.map((s) => ({ schema: spaceSlugToSchema(s.slug) })); + }, + batchSize: parseIntEnv( + "WORKER_BATCH_SIZE", + process.env.WORKER_BATCH_SIZE || "", + "10", + ), + lockDuration: process.env.WORKER_LOCK_DURATION || "5 minutes", + idleDelayMs: + opts.workerIdleDelayMs ?? + parseIntEnv( + "WORKER_IDLE_DELAY_MS", + process.env.WORKER_IDLE_DELAY_MS || "", + "10000", + ), + maxBackoffMs: parseIntEnv( + "WORKER_MAX_BACKOFF_MS", + process.env.WORKER_MAX_BACKOFF_MS || "", + "60000", + ), + refreshIntervalMs: + opts.workerRefreshIntervalMs ?? + parseIntEnv( + "WORKER_REFRESH_INTERVAL_MS", + process.env.WORKER_REFRESH_INTERVAL_MS || "", + "60000", + ), + timeouts: workerTimeouts, + }); + + await workerPool.start(workerCount); + info("Embedding worker pool started", { workers: workerCount }); + + // --------------------------------------------------------------------------- + // Cleanup Jobs + // --------------------------------------------------------------------------- + + // Sweep expired device authorizations and sessions on a cron schedule (UTC). + // Both live in the auth schema now; terminal device states delete themselves + // on poll, so this only reclaims rows abandoned before completing. + const cleanupCron = + (opts.enableCleanupCron ?? true) + ? Bun.cron(deviceFlowCleanupCron, async () => { + try { + const devices = await auth.deleteExpiredDevices(); + if (devices > 0) { + info("Cleaned up expired device authorizations", { + count: devices, + }); + } + } catch (error) { + reportError( + "Failed to cleanup device authorizations", + error as Error, + ); + } + try { + const sessions = await auth.cleanupExpiredSessions(); + if (sessions > 0) { + info("Cleaned up expired sessions", { count: sessions }); + } + } catch (error) { + reportError("Failed to cleanup expired sessions", error as Error); + } + }) + : null; + + // --------------------------------------------------------------------------- + // Server + // --------------------------------------------------------------------------- + + const server = Bun.serve({ + port, + async fetch(request) { + const url = new URL(request.url); + const method = request.method; + const path = url.pathname; + + try { + return await span("http.request", { + attributes: { + "http.method": method, + "http.url": request.url, + "http.path": path, + }, + callback: async () => { + const sizeError = checkSizeLimit(request); + if (sizeError) { + return sizeError; + } + return await router.handleRequest(request); + }, + }); + } catch { + // Error already recorded on http.request span by the helper + return internalError(); + } + }, + }); + + info("Server started", { port: server.port }); + + let stopped = false; + async function stop(): Promise { + if (stopped) return; + stopped = true; + + info("Shutting down server..."); + + // Stop accepting new connections + server.stop(); + + // Stop embedding workers + try { + await workerPool.stop(); + info("Embedding worker pool stopped"); + } catch (error) { + reportError("Error stopping embedding workers", error as Error); + } + + // Stop background jobs + cleanupCron?.stop(); + + // Close database pools + try { + await workerDb.end(); + await db.end(); + } catch (error) { + reportError("Error closing database connections", error as Error); + } + + info("Shutdown complete"); + } + + const boundPort = server.port ?? port; + return { + url: `http://localhost:${boundPort}`, + port: boundPort, + context, + stop, + }; +} diff --git a/packages/server/wiring.test.ts b/packages/server/wiring.test.ts index d6a8346..8d2c76c 100644 --- a/packages/server/wiring.test.ts +++ b/packages/server/wiring.test.ts @@ -1,67 +1,45 @@ /** * Unit tests for server-database wiring. * - * These tests verify that authentication middleware is correctly wired - * to the router using mocked database connections. They test the wiring - * logic, not actual database operations. + * These verify that the new-model authentication middleware is correctly wired + * to the router using mocked stores. They test the wiring (which authenticator + * guards which route, and the shape of its rejections), not real DB operations. * - * For true integration tests with a real database, see the e2e test suite. + * For true integration tests with a real database, see the *.integration.test.ts + * suites under rpc/memory and rpc/user. */ import { describe, expect, mock, test } from "bun:test"; -import type { AccountsDB } from "@memory.build/accounts"; +import type { AuthStore } from "@memory.build/auth"; import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { SQL } from "bun"; +import type { CoreStore } from "@memory.build/engine/core"; +import type { Sql } from "postgres"; import { MIN_CLIENT_VERSION, SERVER_VERSION } from "../../version"; import type { ServerContext } from "./context"; -import type { EngineInfo } from "./middleware/authenticate"; import { createRouter } from "./router"; // ============================================================================= // Test Helpers // ============================================================================= -function createMockAccountsDb(overrides?: { +function createMockAuth(overrides?: { validateSession?: ReturnType; - getEngineBySlug?: ReturnType; -}): AccountsDB { + getUser?: ReturnType; +}): AuthStore { return { validateSession: overrides?.validateSession ?? mock(() => Promise.resolve(null)), - getEngineBySlug: - overrides?.getEngineBySlug ?? mock(() => Promise.resolve(null)), - } as unknown as AccountsDB; -} - -/** - * Create a mock SQL that has enough methods to not throw, but returns - * no results. This allows testing wiring without a real database. - */ -function createMockEngineSql(): SQL { - // Create a function that's also a template tag, returning empty results - const mockSqlTag = mock(() => Promise.resolve([])); - // Add the unsafe method for schema/identifier interpolation - (mockSqlTag as unknown as { unsafe: ReturnType }).unsafe = mock( - (str: string) => str, - ); - - const mockTx = Object.assign( - mock(() => Promise.resolve([])), - { - unsafe: mock((str: string) => str), - }, - ); - - return { - begin: mock((fn: (tx: unknown) => Promise) => fn(mockTx)), - } as unknown as SQL; + getUser: overrides?.getUser ?? mock(() => Promise.resolve(null)), + } as unknown as AuthStore; } function createMockContext(overrides?: Partial): ServerContext { return { - accountsDb: createMockAccountsDb(), - accountsSql: {} as SQL, - engineSql: createMockEngineSql(), + db: {} as Sql, + auth: createMockAuth(), + core: {} as unknown as CoreStore, + authSchema: "auth", + coreSchema: "core", embeddingConfig: { provider: "openai", model: "text-embedding-3-small", @@ -80,14 +58,15 @@ function createMockContext(overrides?: Partial): ServerContext { // ============================================================================= describe("Server-Database Wiring", () => { - describe("Engine RPC Authentication", () => { + describe("Memory RPC authentication (authenticateSpace)", () => { test("returns 401 for missing Authorization header", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/engine/rpc", { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + "X-Me-Space": "abc123def456", + }, body: JSON.stringify({ jsonrpc: "2.0", method: "memory.get", @@ -100,15 +79,13 @@ describe("Server-Database Wiring", () => { expect(response.status).toBe(401); }); - test("returns 401 for invalid API key format", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/engine/rpc", { + test("returns 400 when the X-Me-Space header is missing", async () => { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/memory/rpc", { method: "POST", headers: { "Content-Type": "application/json", - Authorization: "Bearer invalid-key", + Authorization: "Bearer some-token", }, body: JSON.stringify({ jsonrpc: "2.0", @@ -119,21 +96,21 @@ describe("Server-Database Wiring", () => { }); const response = await router.handleRequest(request); - expect(response.status).toBe(401); + expect(response.status).toBe(400); + const body = (await response.json()) as { error: { code: string } }; + expect(body.error.code).toBe("MISSING_SPACE"); }); }); - describe("Accounts RPC Authentication", () => { + describe("User RPC authentication (authenticateUser)", () => { test("returns 401 for missing Authorization header", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/accounts/rpc", { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/user/rpc", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", - method: "me.get", + method: "whoami", params: {}, id: 1, }), @@ -143,11 +120,9 @@ describe("Server-Database Wiring", () => { expect(response.status).toBe(401); }); - test("returns 401 for invalid session token", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - - const request = new Request("http://localhost/api/v1/accounts/rpc", { + test("returns 401 for an invalid session token", async () => { + const router = createRouter(createMockContext()); + const request = new Request("http://localhost/api/v1/user/rpc", { method: "POST", headers: { "Content-Type": "application/json", @@ -155,7 +130,7 @@ describe("Server-Database Wiring", () => { }, body: JSON.stringify({ jsonrpc: "2.0", - method: "me.get", + method: "whoami", params: {}, id: 1, }), @@ -165,26 +140,38 @@ describe("Server-Database Wiring", () => { expect(response.status).toBe(401); }); - test("succeeds with valid session token (happy path)", async () => { - const mockIdentity = { - id: "identity-123", + test("whoami succeeds with a valid session (happy path)", async () => { + const identity = { + id: "01960000-0000-7000-8000-000000000000", email: "test@example.com", name: "Test User", }; - const ctx = createMockContext({ - accountsDb: createMockAccountsDb({ + auth: createMockAuth({ validateSession: mock(() => Promise.resolve({ - session: { id: "session-1", identityId: mockIdentity.id }, - identity: mockIdentity, + sessionId: "session-1", + userId: identity.id, + email: identity.email, + name: identity.name, + expiresAt: new Date("2026-12-31T00:00:00Z"), + }), + ), + getUser: mock(() => + Promise.resolve({ + id: identity.id, + email: identity.email, + name: identity.name, + emailVerified: true, + createdAt: new Date("2026-01-01T00:00:00Z"), + updatedAt: null, }), ), }), }); const router = createRouter(ctx); - const request = new Request("http://localhost/api/v1/accounts/rpc", { + const request = new Request("http://localhost/api/v1/user/rpc", { method: "POST", headers: { "Content-Type": "application/json", @@ -192,82 +179,29 @@ describe("Server-Database Wiring", () => { }, body: JSON.stringify({ jsonrpc: "2.0", - method: "me.get", + method: "whoami", params: {}, id: 1, }), }); const response = await router.handleRequest(request); - // Should get 200 with RPC response (method may not exist but auth passed) expect(response.status).toBe(200); - const body = await response.json(); - // If method doesn't exist, we get an RPC error, but auth succeeded - expect(body).toHaveProperty("jsonrpc", "2.0"); + const body = (await response.json()) as { + jsonrpc: string; + result: { id: string; email: string; name: string }; + }; + expect(body.jsonrpc).toBe("2.0"); + expect(body.result).toEqual(identity); }); }); describe("Health endpoint (no auth)", () => { test("returns 200 without authentication", async () => { - const ctx = createMockContext(); - const router = createRouter(ctx); - + const router = createRouter(createMockContext()); const request = new Request("http://localhost/health"); - const response = await router.handleRequest(request); expect(response.status).toBe(200); }); }); - - describe("Engine RPC wiring verification", () => { - test("engine lookup is called with slug from API key", async () => { - const mockEngine: EngineInfo = { - id: "engine-123", - orgId: "org-456", - slug: "abc123xyz789", - name: "Test Engine", - shardId: 1, - status: "active", - }; - - // Verify router correctly extracts slug from API key and calls accountsDb - // The full auth flow will fail because engineSql is a mock, but we verify - // the wiring is correct by checking getEngineBySlug was called with the right slug - const getEngineBySlug = mock(() => Promise.resolve(mockEngine)); - const ctx = createMockContext({ - accountsDb: createMockAccountsDb({ getEngineBySlug }), - }); - const router = createRouter(ctx); - - // Valid API key format: me.{slug}.{lookupId}.{secret} - const validApiKey = - "me.abc123xyz789.Sh00uLs5rmSHHun3.pREy3xfnbCpgUXiaBcDefghij1234567"; - - const request = new Request("http://localhost/api/v1/engine/rpc", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${validApiKey}`, - }, - body: JSON.stringify({ - jsonrpc: "2.0", - method: "memory.get", - params: { id: "test-id" }, - id: 1, - }), - }); - - // Request will fail downstream (mock engineSql lacks required methods), - // but the wiring we're testing happens before that failure - const response = await router.handleRequest(request); - - // The response will be an error (500 or similar) because the mock SQL - // doesn't have required methods, but we're testing the wiring, not the - // full flow. The important thing is getEngineBySlug was called. - expect(response.status).toBeGreaterThanOrEqual(400); - - // Verify the engine lookup was called with the correct slug extracted from API key - expect(getEngineBySlug).toHaveBeenCalledWith("abc123xyz789"); - }); - }); }); diff --git a/packages/web/src/api/client.ts b/packages/web/src/api/client.ts index a99592d..6255867 100644 --- a/packages/web/src/api/client.ts +++ b/packages/web/src/api/client.ts @@ -2,12 +2,13 @@ * Shared Memory Engine client for the web UI. * * The browser talks to the same-origin `/rpc` proxy exposed by `me serve`. - * That proxy injects the stored API key, so this client intentionally has no - * API key configured. Vite proxies `/rpc` to `me serve` during local dev. + * That proxy injects the session token (Authorization) and the active space + * (X-Me-Space), so this client carries neither. Vite proxies `/rpc` to + * `me serve` during local dev. */ -import { createClient } from "@memory.build/client"; +import { createMemoryClient } from "@memory.build/client"; -export const memoryEngineClient = createClient({ +export const memoryClient = createMemoryClient({ url: "", rpcPath: "/rpc", retries: 0, diff --git a/packages/web/src/api/queries.ts b/packages/web/src/api/queries.ts index 2e8739f..b3de309 100644 --- a/packages/web/src/api/queries.ts +++ b/packages/web/src/api/queries.ts @@ -15,7 +15,7 @@ import { useQuery, useQueryClient, } from "@tanstack/react-query"; -import { memoryEngineClient } from "./client.ts"; +import { memoryClient } from "./client.ts"; const SEARCH_LIMIT = 1000; @@ -49,7 +49,7 @@ export function useMemories(params: MemorySearchParams, enabled = true) { return useQuery({ enabled, queryKey: ["memories", normalized], - queryFn: () => memoryEngineClient.memory.search(normalized), + queryFn: () => memoryClient.memory.search(normalized), }); } @@ -62,7 +62,7 @@ export function useMemories(params: MemorySearchParams, enabled = true) { export function useTree() { return useQuery({ queryKey: ["memory-tree"], - queryFn: () => memoryEngineClient.memory.tree(), + queryFn: () => memoryClient.memory.tree(), }); } @@ -78,7 +78,7 @@ export function useMemoriesAtExactPath(path: string, enabled: boolean) { enabled, queryKey: ["memories-at-exact-path", path], queryFn: () => - memoryEngineClient.memory.search({ + memoryClient.memory.search({ tree: exactTreeLquery(path), limit: SEARCH_LIMIT, }), @@ -94,7 +94,7 @@ export function useMemory(id: string | null) { return useQuery({ enabled: id !== null, queryKey: ["memory", id], - queryFn: () => memoryEngineClient.memory.get({ id: id as string }), + queryFn: () => memoryClient.memory.get({ id: id as string }), }); } @@ -104,7 +104,7 @@ export function useMemory(id: string | null) { export function useUpdateMemory(queryClient: QueryClient) { return useMutation({ mutationFn: (params: MemoryUpdateParams) => - memoryEngineClient.memory.update(params), + memoryClient.memory.update(params), onSuccess: (memory) => { invalidateTreeQueries(queryClient); queryClient.setQueryData(["memory", memory.id], memory); @@ -117,7 +117,7 @@ export function useUpdateMemory(queryClient: QueryClient) { */ export function useDeleteMemory(queryClient: QueryClient) { return useMutation({ - mutationFn: (id: string) => memoryEngineClient.memory.delete({ id }), + mutationFn: (id: string) => memoryClient.memory.delete({ id }), onSuccess: (_result, id) => { invalidateTreeQueries(queryClient); queryClient.removeQueries({ queryKey: ["memory", id] }); @@ -132,7 +132,7 @@ export function useDeleteMemory(queryClient: QueryClient) { export function useDeleteTree(queryClient: QueryClient) { return useMutation({ mutationFn: (args: { tree: string; dryRun?: boolean }) => - memoryEngineClient.memory.deleteTree(args), + memoryClient.memory.deleteTree(args), onSuccess: (_result, args) => { if (!args.dryRun) { invalidateTreeQueries(queryClient); diff --git a/packages/web/src/components/dialogs/DeleteTreeDialog.tsx b/packages/web/src/components/dialogs/DeleteTreeDialog.tsx index d364774..42c6ee7 100644 --- a/packages/web/src/components/dialogs/DeleteTreeDialog.tsx +++ b/packages/web/src/components/dialogs/DeleteTreeDialog.tsx @@ -6,7 +6,7 @@ * be removed. On confirm, re-issues the call with `dryRun: false`. */ import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { memoryEngineClient } from "../../api/client.ts"; +import { memoryClient } from "../../api/client.ts"; import { useDeleteTree } from "../../api/queries.ts"; import { useSelection } from "../../store/selection.ts"; import { useUi } from "../../store/ui.ts"; @@ -26,7 +26,7 @@ export function DeleteTreeDialog() { enabled: treePath !== null, queryKey: ["deleteTreeDryRun", treePath], queryFn: () => - memoryEngineClient.memory.deleteTree({ + memoryClient.memory.deleteTree({ tree: treePath as string, dryRun: true, }), diff --git a/packages/worker/index.ts b/packages/worker/index.ts index 20da2f0..e461eee 100644 --- a/packages/worker/index.ts +++ b/packages/worker/index.ts @@ -1,9 +1,11 @@ export { WorkerPool } from "./pool"; export { processBatch } from "./process"; -export type { - EngineTarget, - ProcessResult, - WorkerConfig, - WorkerStats, +export { + DEFAULT_WORKER_TIMEOUTS, + type ProcessResult, + type SpaceTarget, + type WorkerConfig, + type WorkerStats, + type WorkerTimeouts, } from "./types"; export { Worker } from "./worker"; diff --git a/packages/worker/package.json b/packages/worker/package.json index 9adb437..7e661ce 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -5,8 +5,9 @@ "type": "module", "main": "index.ts", "dependencies": { - "@memory.build/engine": "workspace:*", + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", - "@pydantic/logfire-node": "^0.13.1" + "@pydantic/logfire-node": "^0.13.1", + "postgres": "^3.4.9" } } diff --git a/packages/worker/pool.ts b/packages/worker/pool.ts index 0d4a806..000340f 100644 --- a/packages/worker/pool.ts +++ b/packages/worker/pool.ts @@ -1,19 +1,19 @@ -import type { SQL } from "bun"; +import type { Sql } from "postgres"; import type { WorkerConfig, WorkerStats } from "./types"; import { Worker } from "./worker"; /** * Pool of N embedding workers using the provided SQL connection pool. - * Each worker independently discovers engines, shuffles its target list, + * Each worker independently discovers spaces, shuffles its target list, * and polls queues. FOR UPDATE SKIP LOCKED prevents double-processing. */ export class WorkerPool { - private readonly sql: SQL; + private readonly sql: Sql; private readonly config: WorkerConfig; private workers: Worker[] = []; private running = false; - constructor(sql: SQL, config: WorkerConfig) { + constructor(sql: Sql, config: WorkerConfig) { this.sql = sql; this.config = config; } @@ -44,7 +44,7 @@ export class WorkerPool { totalProcessed: 0, totalFailed: 0, totalPruned: 0, - enginesDropped: 0, + spacesDropped: 0, consecutiveErrors: 0, }; for (const worker of this.workers) { @@ -53,7 +53,7 @@ export class WorkerPool { agg.totalProcessed += s.totalProcessed; agg.totalFailed += s.totalFailed; agg.totalPruned += s.totalPruned; - agg.enginesDropped += s.enginesDropped; + agg.spacesDropped += s.spacesDropped; agg.consecutiveErrors = Math.max( agg.consecutiveErrors, s.consecutiveErrors, diff --git a/packages/worker/process.integration.test.ts b/packages/worker/process.integration.test.ts index 63712cf..24bd4cd 100644 --- a/packages/worker/process.integration.test.ts +++ b/packages/worker/process.integration.test.ts @@ -1,270 +1,193 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { + bootstrapSpaceDatabase, + generateSlug, + migrateSpace, + slugToSchema, +} from "@memory.build/database"; import { RateLimitError } from "@memory.build/embedding"; -import { createEngineDB } from "@memory.build/engine/db"; -import { bootstrap } from "@memory.build/engine/migrate/bootstrap"; -import { provisionEngine } from "@memory.build/engine/migrate/provision"; -import { TestDatabase } from "@memory.build/engine/migrate/test-utils"; -import { SQL } from "bun"; +import postgres, { type Sql } from "postgres"; import { processBatch, pruneQueue } from "./process"; -import type { WorkerConfig } from "./types"; +import type { SpaceTarget, WorkerConfig } from "./types"; // --------------------------------------------------------------------------- -// Test setup +// Test setup — a real space schema (me_) on the new-model pool. // --------------------------------------------------------------------------- -const testDb = new TestDatabase(); -let connectionString: string; -let sql: SQL; -const slug = "tstworker001"; -const schema = `me_${slug}`; -const target = { schema, shard: 1 }; -const discover = async () => [target]; - -beforeAll(async () => { - connectionString = await testDb.create(); - sql = new SQL(connectionString); - await bootstrap(sql); - await provisionEngine(sql, slug, undefined, "0.1.0"); - - // Create a superuser principal for inserting memories - const db = createEngineDB(sql, schema); - const su = await db.createSuperuser("worker-test-admin"); - db.setUser(su.id); - - // Grant me_embed SELECT/UPDATE on memory (already done by migration 005) - // but we need to ensure the embedding_queue trigger is active -}); +const URL = + process.env.TEST_DATABASE_URL ?? + "postgresql://postgres@127.0.0.1:5432/postgres"; -afterAll(async () => { - await sql.close(); - await testDb.drop(); -}); - -// --------------------------------------------------------------------------- -// Helper: insert a memory and return its id + queue state -// --------------------------------------------------------------------------- +let sql: Sql; +let slug: string; +let schema: string; +let target: SpaceTarget; +const discover = async () => [target]; -function getDb() { - return createEngineDB(sql, schema); +/** A config whose embedding calls hit the given mock base URL. */ +function mockConfig(baseUrl: string, maxRetries?: number): WorkerConfig { + return { + embedding: { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + apiKey: "test-key", + baseUrl, + ...(maxRetries !== undefined ? { options: { maxRetries } } : {}), + }, + discover, + batchSize: 10, + }; } -async function withDb() { - const db = getDb(); - const su = await db.getUserByName("worker-test-admin"); - db.setUser(su!.id); - return db; +/** A mock OpenAI embeddings server returning a fixed vector. */ +function embedServer(): ReturnType { + const embedding = Array.from({ length: 1536 }, (_, i) => i * 0.001); + return Bun.serve({ + port: 0, + fetch() { + return Response.json({ + object: "list", + data: [{ object: "embedding", embedding, index: 0 }], + model: "text-embedding-3-small", + usage: { prompt_tokens: 5, total_tokens: 5 }, + }); + }, + }); } async function insertMemory(content: string): Promise { - const db = await withDb(); - const memory = await db.createMemory({ content, tree: "test.worker" }); - return memory.id; + const [row] = await sql.unsafe( + `INSERT INTO ${schema}.memory (content, tree) VALUES ($1, ''::ltree) RETURNING id`, + [content], + ); + return row?.id as string; } -async function getQueueEntries(memoryId: string) { +function getQueueEntries(memoryId: string) { return sql.unsafe( `SELECT * FROM ${schema}.embedding_queue WHERE memory_id = $1 ORDER BY id`, [memoryId], - ); + ) as Promise[]>; } -async function getMemoryEmbedding(memoryId: string) { - const [row] = await sql.unsafe( - `SELECT embedding, embedding_version FROM ${schema}.memory WHERE id = $1`, - [memoryId], +async function clearPending() { + await sql.unsafe( + `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, ); - return row; } -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- +beforeAll(async () => { + sql = postgres(URL, { onnotice: () => {} }); + slug = generateSlug(); + schema = slugToSchema(slug); + target = { schema }; + await bootstrapSpaceDatabase(sql); + await migrateSpace(sql, { slug }); +}); + +afterAll(async () => { + await sql.unsafe(`DROP SCHEMA IF EXISTS ${schema} CASCADE`); + await sql.end(); +}); + +describe("processBatch integration (space model)", () => { + beforeEach(clearPending); -describe("processBatch integration", () => { test("processes queue entries and writes embeddings", async () => { const memoryId = await insertMemory("Hello world embedding test"); - // Verify queue entry was created by trigger const queueBefore = await getQueueEntries(memoryId); expect(queueBefore.length).toBeGreaterThanOrEqual(1); - expect(queueBefore[0].outcome).toBeNull(); - - // We need to mock generateEmbeddings at the module level - // Instead, use a real-ish approach: create a processBatch wrapper - // that intercepts. For integration test, we'll use the actual processBatch - // but with a test embedding provider. - - // Create a mock embedding server using Bun.serve - const mockEmbedding = Array.from({ length: 1536 }, (_, i) => i * 0.001); - const server = Bun.serve({ - port: 0, - fetch() { - return Response.json({ - object: "list", - data: [{ object: "embedding", embedding: mockEmbedding, index: 0 }], - model: "text-embedding-3-small", - usage: { prompt_tokens: 5, total_tokens: 5 }, - }); - }, - }); + expect(queueBefore[0]?.outcome).toBeNull(); + const server = embedServer(); try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`), + ); expect(result.claimed).toBeGreaterThanOrEqual(1); expect(result.succeeded).toBeGreaterThanOrEqual(1); expect(result.failed).toBe(0); - // Verify embedding was written - const mem = await getMemoryEmbedding(memoryId); - expect(mem.embedding).toBeDefined(); + const [mem] = await sql.unsafe( + `SELECT embedding FROM ${schema}.memory WHERE id = $1`, + [memoryId], + ); + expect(mem?.embedding).toBeDefined(); - // Verify queue entry marked completed const queueAfter = await getQueueEntries(memoryId); - const completed = queueAfter.find( - (q: Record) => q.outcome === "completed", - ); - expect(completed).toBeDefined(); + expect(queueAfter.some((q) => q.outcome === "completed")).toBe(true); } finally { server.stop(); } }); - test("handles stale version (content changed during embed)", async () => { - await insertMemory("Original content for version test"); - - // Manually bump embedding_version to simulate content change after claim - // First, process to clear the initial queue entry - const server = Bun.serve({ - port: 0, - fetch() { - return Response.json({ - object: "list", - data: [ - { - object: "embedding", - embedding: Array.from({ length: 1536 }, (_, i) => i * 0.001), - index: 0, - }, - ], - model: "text-embedding-3-small", - usage: { prompt_tokens: 5, total_tokens: 5 }, - }); - }, - }); + test("cancels stale version at claim time", async () => { + const staleId = await insertMemory("Stale version content"); + // Bump the memory's version so the queued row (v1) is stale. + await sql.unsafe( + `UPDATE ${schema}.memory SET embedding_version = embedding_version + 1 WHERE id = $1`, + [staleId], + ); + const server = embedServer(); try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - // Clear any pending entries first - await processBatch(sql, target, config); - - // Now insert a new memory and manually create a stale queue entry - const staleId = await insertMemory("Stale version content"); - - // Bump the memory's embedding_version to make queue entry stale - await sql.unsafe( - `UPDATE ${schema}.memory SET embedding_version = embedding_version + 1 WHERE id = $1`, - [staleId], + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`), ); - - const result = await processBatch(sql, target, config); - - // Stale row cancelled at claim time — not counted as claimed expect(result.claimed).toBe(0); - - // Queue entry should be cancelled (version mismatch detected at claim) const queue = await getQueueEntries(staleId); - const cancelled = queue.find( - (q: Record) => q.outcome === "cancelled", - ); - expect(cancelled).toBeDefined(); + expect(queue.some((q) => q.outcome === "cancelled")).toBe(true); } finally { server.stop(); } }); - test("handles non-rate-limit embedding errors gracefully", async () => { + test("non-rate-limit error records last_error, leaves outcome NULL for retry", async () => { const memoryId = await insertMemory("Error test content"); - - // Mock server that returns a non-rate-limit error (400 bad request) const server = Bun.serve({ port: 0, fetch() { return new Response( JSON.stringify({ - error: { - message: "Invalid input", - type: "invalid_request_error", - }, + error: { message: "Invalid input", type: "invalid_request_error" }, }), { status: 400 }, ); }, }); - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - options: { maxRetries: 0 }, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`, 0), + ); expect(result.claimed).toBeGreaterThanOrEqual(1); expect(result.failed).toBeGreaterThanOrEqual(1); - // Queue entry should still have NULL outcome (for retry) but have last_error set const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === null && q.last_error, - ); + const entry = queue.find((q) => q.outcome === null && q.last_error); expect(entry).toBeDefined(); - expect(entry.last_error).toBeTruthy(); + expect(entry?.last_error).toBeTruthy(); } finally { server.stop(); } }); test("rate limit (429) throws RateLimitError and decrements attempts", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - const memoryId = await insertMemory("Rate limit test content"); - - // Mock server that returns 429 const server = Bun.serve({ port: 0, fetch() { @@ -272,141 +195,83 @@ describe("processBatch integration", () => { JSON.stringify({ error: { message: "Rate limited", type: "rate_limit_error" }, }), - { - status: 429, - headers: { "retry-after-ms": "5000" }, - }, + { status: 429, headers: { "retry-after-ms": "5000" } }, ); }, }); - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - options: { maxRetries: 0 }, - }, - discover, - batchSize: 10, - }; - - // processBatch should throw RateLimitError - await expect(processBatch(sql, target, config)).rejects.toBeInstanceOf( - RateLimitError, - ); - - // Queue entry should have attempts back to 0 (claim incremented to 1, - // then RateLimitError handler decremented back to 0) + await expect( + processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`, 0), + ), + ).rejects.toBeInstanceOf(RateLimitError); + + // claim incremented attempts to 1 and locked the row (vt in the future); + // the RateLimitError handler released it — attempts back to 0 (the + // transient failure isn't charged) and vt reset so it's immediately + // claimable again rather than waiting out the full claim lock. const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === null, - ); - expect(entry).toBeDefined(); - expect(entry.attempts).toBe(0); + const entry = queue.find((q) => q.outcome === null); + expect(entry?.attempts).toBe(0); + expect((entry?.vt as Date).getTime()).toBeLessThanOrEqual(Date.now()); } finally { server.stop(); } }); - test("marks queue row as failed after max attempts exhausted (non-rate-limit)", async () => { - // Clear pending entries + test("claim sweeps exhausted rows to 'failed'", async () => { + const memoryId = await insertMemory("Exhausted attempts content"); + // Simulate a crash that left the row at max attempts with an expired lock. await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - - const memoryId = await insertMemory("Exhaust attempts content"); - - // Set attempts = 2 so next claim brings it to 3 (== max_attempts) - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET attempts = 2 WHERE memory_id = $1 AND outcome IS NULL`, + `UPDATE ${schema}.embedding_queue + SET attempts = 3, vt = now() - interval '1 minute' + WHERE memory_id = $1 AND outcome IS NULL`, [memoryId], ); - // Mock server that returns 400 (non-rate-limit) so embedding fails - const server = Bun.serve({ - port: 0, - fetch() { - return new Response( - JSON.stringify({ - error: { - message: "Invalid input", - type: "invalid_request_error", - }, - }), - { status: 400 }, - ); - }, - }); - - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - options: { maxRetries: 0 }, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - - expect(result.claimed).toBeGreaterThanOrEqual(1); - expect(result.failed).toBeGreaterThanOrEqual(1); - - // Queue entry should now be finalized as 'failed' - const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === "failed", - ); - expect(entry).toBeDefined(); - expect(entry.last_error).toBeTruthy(); - } finally { - server.stop(); - } - }); - - test("cancels stale queue rows at claim time", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, + // Base URL is never reached — the row is swept, not embedded. + const result = await processBatch( + sql, + target, + mockConfig("http://localhost:1/v1"), ); + expect(result.claimed).toBe(0); - // Insert a memory — trigger creates queue row at version 1 - const memoryId = await insertMemory("Stale claim-time v1"); - - // Update content twice more — each triggers a new queue row (v2, v3) - const db = await withDb(); - await db.updateMemory(memoryId, { content: "Stale claim-time v2" }); - await db.updateMemory(memoryId, { content: "Stale claim-time v3" }); + const queue = await getQueueEntries(memoryId); + const entry = queue.find((q) => q.outcome === "failed"); + expect(entry).toBeDefined(); + expect(String(entry?.last_error)).toContain("exceeded max attempts"); + }); - // Verify 3 pending queue rows exist - const queueBefore = await getQueueEntries(memoryId); - const pending = queueBefore.filter( - (q: Record) => q.outcome === null, + test("cancels superseded versions, embeds only the latest", async () => { + const memoryId = await insertMemory("claim-time v1"); + await sql.unsafe(`UPDATE ${schema}.memory SET content = $1 WHERE id = $2`, [ + "claim-time v2", + memoryId, + ]); + await sql.unsafe(`UPDATE ${schema}.memory SET content = $1 WHERE id = $2`, [ + "claim-time v3", + memoryId, + ]); + + const pending = (await getQueueEntries(memoryId)).filter( + (q) => q.outcome === null, ); expect(pending.length).toBe(3); - // Mock server tracks call count — only version 3 should be embedded - let embedCallCount = 0; - const mockEmbedding = Array.from({ length: 1536 }, () => 0); + let embedCalls = 0; const server = Bun.serve({ port: 0, fetch() { - embedCallCount++; + embedCalls++; return Response.json({ object: "list", data: [ { object: "embedding", - embedding: mockEmbedding, + embedding: Array.from({ length: 1536 }, () => 0), index: 0, }, ], @@ -415,173 +280,54 @@ describe("processBatch integration", () => { }); }, }); - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - - // Only version 3 should be claimed and processed + const result = await processBatch( + sql, + target, + mockConfig(`http://localhost:${server.port}/v1`), + ); expect(result.claimed).toBe(1); expect(result.succeeded).toBe(1); - expect(embedCallCount).toBe(1); + expect(embedCalls).toBe(1); - // Verify queue outcomes: two cancelled (v1, v2), one completed (v3) const queueAfter = await getQueueEntries(memoryId); - const cancelled = queueAfter.filter( - (q: Record) => q.outcome === "cancelled", + expect(queueAfter.filter((q) => q.outcome === "cancelled").length).toBe( + 2, ); - const completed = queueAfter.filter( - (q: Record) => q.outcome === "completed", + expect(queueAfter.filter((q) => q.outcome === "completed").length).toBe( + 1, ); - expect(cancelled.length).toBe(2); - expect(completed.length).toBe(1); } finally { server.stop(); } }); - test("cancels queue rows for deleted memories at claim time", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - - // Insert a memory — trigger creates a queue entry + test("deleted memory: queue row CASCADE-deleted, nothing to claim", async () => { const memoryId = await insertMemory("Delete claim-time test"); - - // Verify queue entry exists - const queueBefore = await getQueueEntries(memoryId); - expect(queueBefore.length).toBeGreaterThanOrEqual(1); - - // Delete the memory — CASCADE deletes the queue row too await sql.unsafe(`DELETE FROM ${schema}.memory WHERE id = $1`, [memoryId]); + expect((await getQueueEntries(memoryId)).length).toBe(0); - // Queue row should be gone due to CASCADE - const queueAfter = await getQueueEntries(memoryId); - expect(queueAfter.length).toBe(0); - - // processBatch should handle this gracefully (nothing to claim) - const server = Bun.serve({ - port: 0, - fetch() { - return Response.json({ - object: "list", - data: [ - { - object: "embedding", - embedding: Array.from({ length: 1536 }, () => 0), - index: 0, - }, - ], - model: "text-embedding-3-small", - usage: { prompt_tokens: 1, total_tokens: 1 }, - }); - }, - }); - - try { - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: `http://localhost:${server.port}/v1`, - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - expect(result.claimed).toBe(0); - } finally { - server.stop(); - } - }); - - test("sweeps zombie rows as failed when attempts exhausted by crash", async () => { - // Clear pending entries - await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, - ); - - const memoryId = await insertMemory("Zombie crash test content"); - - // Simulate a crash: set attempts = max_attempts and vt in the past, leave outcome NULL - await sql.unsafe( - `UPDATE ${schema}.embedding_queue - SET attempts = max_attempts, vt = now() - interval '1 minute' - WHERE memory_id = $1 AND outcome IS NULL`, - [memoryId], + const result = await processBatch( + sql, + target, + mockConfig("http://localhost:1/v1"), ); - - // processBatch should sweep the zombie row as failed, not claim it - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: "http://localhost:1/v1", // never called - }, - discover, - batchSize: 10, - }; - - const result = await processBatch(sql, target, config); - - // Zombie was swept, not claimed expect(result.claimed).toBe(0); - - // Queue entry should be finalized as 'failed' with crash message - const queue = await getQueueEntries(memoryId); - const entry = queue.find( - (q: Record) => q.outcome === "failed", - ); - expect(entry).toBeDefined(); - expect(entry.last_error).toContain("exceeded max attempts (worker crash)"); }); test("pruneQueue deletes terminal rows older than retention", async () => { - // Clear queue await sql.unsafe(`DELETE FROM ${schema}.embedding_queue`); - const memoryId = await insertMemory("Prune helper test memory"); - // Discard the trigger-enqueued row so we control all rows in the queue await sql.unsafe(`DELETE FROM ${schema}.embedding_queue`); - // Old terminal rows (will be pruned) await sql.unsafe( `INSERT INTO ${schema}.embedding_queue (memory_id, embedding_version, outcome, created_at) VALUES ($1, 1, 'completed', now() - interval '10 days'), ($1, 2, 'failed', now() - interval '10 days'), - ($1, 3, 'cancelled', now() - interval '10 days')`, - [memoryId], - ); - // Recent terminal row (kept) - await sql.unsafe( - `INSERT INTO ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - VALUES ($1, 4, 'completed', now() - interval '1 day')`, - [memoryId], - ); - // Old active row (kept — outcome IS NULL) - await sql.unsafe( - `INSERT INTO ${schema}.embedding_queue - (memory_id, embedding_version, outcome, created_at) - VALUES ($1, 5, null, now() - interval '30 days')`, + ($1, 3, 'cancelled', now() - interval '10 days'), + ($1, 4, 'completed', now() - interval '1 day'), + ($1, 5, null, now() - interval '30 days')`, [memoryId], ); @@ -594,40 +340,135 @@ describe("processBatch integration", () => { [memoryId], )) as { embedding_version: number; outcome: string | null }[]; expect(remaining).toHaveLength(2); - expect(remaining[0]!.embedding_version).toBe(4); - expect(remaining[0]!.outcome).toBe("completed"); - expect(remaining[1]!.embedding_version).toBe(5); - expect(remaining[1]!.outcome).toBeNull(); + expect(remaining[0]?.embedding_version).toBe(4); + expect(remaining[1]?.embedding_version).toBe(5); + expect(remaining[1]?.outcome).toBeNull(); }); test("pruneQueue is a no-op when nothing matches", async () => { await sql.unsafe(`DELETE FROM ${schema}.embedding_queue`); - const pruned = await pruneQueue(sql, target, "7 days"); - expect(pruned).toBe(0); + expect(await pruneQueue(sql, target, "7 days")).toBe(0); }); test("returns zero when queue is empty", async () => { - // Use a dedicated config pointing at a mock server that should never be called - const config: WorkerConfig = { - embedding: { - provider: "openai", - model: "text-embedding-3-small", - dimensions: 1536, - apiKey: "test-key", - baseUrl: "http://localhost:1/v1", - }, - discover, - batchSize: 10, + const result = await processBatch( + sql, + target, + mockConfig("http://localhost:1/v1"), + ); + expect(result).toEqual({ claimed: 0, succeeded: 0, failed: 0 }); + }); +}); + +describe("write-back SQL functions", () => { + beforeEach(clearPending); + + const zeroVec = `[${Array.from({ length: 1536 }, () => 0).join(",")}]`; + + async function pendingRow(content: string) { + const memoryId = await insertMemory(content); + const [q] = await getQueueEntries(memoryId); + return { + memoryId, + queueId: q?.id as string, + version: Number(q?.embedding_version), }; + } + + test("complete_embedding writes the vector and marks 'completed' on a version match", async () => { + const { memoryId, queueId, version } = await pendingRow("complete me"); + + const [r] = (await sql.unsafe( + `SELECT ${schema}.complete_embedding($1, $2, $3, $4::halfvec) AS outcome`, + [queueId, memoryId, version, zeroVec], + )) as { outcome: string }[]; + expect(r?.outcome).toBe("completed"); + + const [mem] = await sql.unsafe( + `SELECT embedding FROM ${schema}.memory WHERE id = $1`, + [memoryId], + ); + expect(mem?.embedding).not.toBeNull(); + const [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBe("completed"); + }); + + test("complete_embedding cancels (no write) when the version no longer matches", async () => { + const { memoryId, queueId, version } = await pendingRow("superseded"); + + const [r] = (await sql.unsafe( + `SELECT ${schema}.complete_embedding($1, $2, $3, $4::halfvec) AS outcome`, + [queueId, memoryId, version + 1, zeroVec], // stale version + )) as { outcome: string }[]; + expect(r?.outcome).toBe("cancelled"); + + const [mem] = await sql.unsafe( + `SELECT embedding FROM ${schema}.memory WHERE id = $1`, + [memoryId], + ); + expect(mem?.embedding).toBeNull(); // not written + const [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBe("cancelled"); + }); + + test("fail_embedding records last_error and leaves outcome NULL; no-op once terminal", async () => { + const { memoryId, queueId } = await pendingRow("fail me"); - // Clear all pending queue entries first + await sql.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + queueId, + "boom", + ]); + let [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBeNull(); + expect(q?.last_error).toBe("boom"); + + // Finalize, then a later fail must not touch the terminal row. await sql.unsafe( - `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE outcome IS NULL`, + `UPDATE ${schema}.embedding_queue SET outcome = 'completed' WHERE id = $1`, + [queueId], ); + await sql.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + queueId, + "later", + ]); + [q] = await getQueueEntries(memoryId); + expect(q?.outcome).toBe("completed"); + expect(q?.last_error).toBe("boom"); // unchanged + }); - const result = await processBatch(sql, target, config); - expect(result.claimed).toBe(0); - expect(result.succeeded).toBe(0); - expect(result.failed).toBe(0); + test("release_embedding decrements attempts, resets vt, floors at 0, no-op once terminal", async () => { + const { memoryId, queueId } = await pendingRow("release me"); + // Simulate a claim: attempt charged + locked (vt pushed into the future). + await sql.unsafe( + `UPDATE ${schema}.embedding_queue + SET attempts = 2, vt = now() + interval '5 minutes' WHERE id = $1`, + [queueId], + ); + + await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); + const [released] = (await sql.unsafe( + `SELECT attempts, (vt <= now()) AS claimable + FROM ${schema}.embedding_queue WHERE id = $1`, + [queueId], + )) as { attempts: number; claimable: boolean }[]; + expect(Number(released?.attempts)).toBe(1); + expect(released?.claimable).toBe(true); // vt reset → eligible again now + + // Floors at 0. + await sql.unsafe( + `UPDATE ${schema}.embedding_queue SET attempts = 0 WHERE id = $1`, + [queueId], + ); + await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); + expect(Number((await getQueueEntries(memoryId))[0]?.attempts)).toBe(0); + + // No-op once terminal. + await sql.unsafe( + `UPDATE ${schema}.embedding_queue + SET outcome = 'completed', attempts = 5 WHERE id = $1`, + [queueId], + ); + await sql.unsafe(`SELECT ${schema}.release_embedding($1)`, [queueId]); + expect(Number((await getQueueEntries(memoryId))[0]?.attempts)).toBe(5); }); }); diff --git a/packages/worker/process.ts b/packages/worker/process.ts index 93ce22b..6a80f3f 100644 --- a/packages/worker/process.ts +++ b/packages/worker/process.ts @@ -3,20 +3,21 @@ import { generateEmbeddings, RateLimitError, } from "@memory.build/embedding"; -import { - DEFAULT_ENGINE_TIMEOUTS, - type EngineTimeouts, - setLocalEngineTimeouts, -} from "@memory.build/engine/ops/_tx"; import { info, reportError, span, warning } from "@pydantic/logfire-node"; -import type { SQL } from "bun"; -import type { EngineTarget, ProcessResult, WorkerConfig } from "./types"; +import type { Sql } from "postgres"; +import { + DEFAULT_WORKER_TIMEOUTS, + type ProcessResult, + type SpaceTarget, + type WorkerConfig, + type WorkerTimeouts, +} from "./types"; -function workerEngineTimeouts(config?: WorkerConfig): EngineTimeouts { - return config?.workerEngineTimeouts ?? DEFAULT_ENGINE_TIMEOUTS; +function workerTimeouts(config?: WorkerConfig): WorkerTimeouts { + return config?.timeouts ?? DEFAULT_WORKER_TIMEOUTS; } -function workerEngineTimeoutAttributes(timeouts: EngineTimeouts) { +function timeoutAttributes(timeouts: WorkerTimeouts) { return { "db.statement_timeout": timeouts.statementTimeout, "db.lock_timeout": timeouts.lockTimeout, @@ -26,6 +27,32 @@ function workerEngineTimeoutAttributes(timeouts: EngineTimeouts) { }; } +/** + * Set transaction-local search_path + timeouts. The new model is not sharded + * and the space functions are security-invoker, so the worker runs as the pool + * user with no SET ROLE. + */ +async function prepareTx( + tx: Sql, + schema: string, + timeouts: WorkerTimeouts, +): Promise { + await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); + await tx.unsafe("SELECT set_config('statement_timeout', $1, true)", [ + timeouts.statementTimeout, + ]); + await tx.unsafe("SELECT set_config('lock_timeout', $1, true)", [ + timeouts.lockTimeout, + ]); + await tx.unsafe("SELECT set_config('transaction_timeout', $1, true)", [ + timeouts.transactionTimeout, + ]); + await tx.unsafe( + "SELECT set_config('idle_in_transaction_session_timeout', $1, true)", + [timeouts.idleInTransactionSessionTimeout], + ); +} + function asError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } @@ -37,24 +64,21 @@ function asError(error: unknown): Error { * Returns the number of rows pruned. */ export async function pruneQueue( - sql: SQL, - target: EngineTarget, + sql: Sql, + target: SpaceTarget, retention: string, config?: WorkerConfig, ): Promise { - const { schema, shard } = target; - const timeouts = workerEngineTimeouts(config); + const { schema } = target; + const timeouts = workerTimeouts(config); return sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + await prepareTx(tx as unknown as Sql, schema, timeouts); const rows = (await tx.unsafe( `SELECT ${schema}.prune_embedding_queue($1::interval) AS pruned`, [retention], )) as { pruned: string | number | null }[]; return Number(rows[0]?.pruned ?? 0); - }); + }) as Promise; } interface ClaimedRow { @@ -69,30 +93,30 @@ interface ClaimedRow { * * Claim and write-back are separate transactions — if the worker crashes * between them, the visibility timeout expires and rows become claimable again. + * Rows that exhaust their attempts are finalized to 'failed' by the claim + * function's sweep, so write-back only records last_error and leaves the + * outcome NULL on transient failure. */ export async function processBatch( - sql: SQL, - target: EngineTarget, + sql: Sql, + target: SpaceTarget, config: WorkerConfig, ): Promise { - const { schema, shard } = target; + const { schema } = target; const batchSize = config.batchSize ?? 10; const lockDuration = config.lockDuration ?? "5 minutes"; - const timeouts = workerEngineTimeouts(config); - const timeoutAttributes = workerEngineTimeoutAttributes(timeouts); + const timeouts = workerTimeouts(config); + const attrs = timeoutAttributes(timeouts); // --- Claim --- const claimStart = performance.now(); - const claimed = await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + const claimed = (await sql.begin(async (tx) => { + await prepareTx(tx as unknown as Sql, schema, timeouts); return tx.unsafe( `SELECT * FROM ${schema}.claim_embedding_batch($1, $2::interval)`, [batchSize, lockDuration], - ) as Promise; - }); + ); + })) as ClaimedRow[]; const claimDurationMs = performance.now() - claimStart; if (claimed.length === 0) { @@ -101,28 +125,25 @@ export async function processBatch( info("Embedding batch claimed", { "worker.schema": schema, - "worker.shard": shard, "batch.claimed": claimed.length, "batch.requested_size": batchSize, "batch.lock_duration": lockDuration, "batch.claim_duration_ms": claimDurationMs, "batch.memoryIds": claimed.map((r) => r.memory_id), "batch.queueIds": claimed.map((r) => r.queue_id), - ...timeoutAttributes, + ...attrs, }); - // Process claimed items with telemetry return span("embedding.batch", { attributes: { "worker.schema": schema, - "worker.shard": shard, "batch.size": claimed.length, "batch.requested_size": batchSize, "batch.lock_duration": lockDuration, "batch.claim_duration_ms": claimDurationMs, "batch.memoryIds": claimed.map((r) => r.memory_id), "batch.queueIds": claimed.map((r) => r.queue_id), - ...timeoutAttributes, + ...attrs, }, callback: async () => { // --- Embed --- @@ -137,19 +158,13 @@ export async function processBatch( } catch (error) { if (error instanceof RateLimitError) { // Undo the attempt increment from claim — rate limits are transient - // and should not consume max_attempts + // and should not consume the attempt budget. await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + await prepareTx(tx as unknown as Sql, schema, timeouts); for (const row of claimed) { - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET attempts = greatest(attempts - 1, 0) - WHERE id = $1 AND outcome IS NULL`, - [row.queue_id], - ); + await tx.unsafe(`SELECT ${schema}.release_embedding($1)`, [ + row.queue_id, + ]); } }); } @@ -158,14 +173,12 @@ export async function processBatch( info("Embedding batch generated", { "worker.schema": schema, - "worker.shard": shard, "batch.claimed": claimed.length, "batch.generated": embedResults.length, "batch.embed_successes": embedResults.filter((r) => !r.error).length, "batch.embed_errors": embedResults.filter((r) => r.error).length, }); - // Build lookup: memory_id → embed result const resultMap = new Map(embedResults.map((r) => [r.id, r])); // --- Write-back --- @@ -175,68 +188,48 @@ export async function processBatch( await span("embedding.write_back", { attributes: { "worker.schema": schema, - "worker.shard": shard, "batch.size": claimed.length, "batch.embed_successes": embedResults.filter((r) => !r.error).length, "batch.embed_errors": embedResults.filter((r) => r.error).length, - ...timeoutAttributes, + ...attrs, }, callback: async () => { for (const row of claimed) { try { await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); + await prepareTx(tx as unknown as Sql, schema, timeouts); const result = resultMap.get(row.memory_id); if (!result || result.error) { - // Embedding failed — record error, leave outcome NULL for retry. - // Queue row may be CASCADE-deleted if memory was deleted; 0 rows is fine. + // Embedding failed — record the error, leave outcome NULL so + // the row retries; the claim sweep fails it once attempts are + // exhausted. (Row may be CASCADE-deleted if the memory was + // deleted; 0 rows updated is fine.) const error = result?.error ?? "No embedding result returned"; - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET last_error = $1 - , outcome = CASE WHEN attempts >= max_attempts THEN 'failed' END - WHERE id = $2`, - [error, row.queue_id], - ); + await tx.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + row.queue_id, + error, + ]); failed++; return; } - // Version-guarded write to memory. + // Version-guarded write-back: writes the memory iff its version + // still matches the claim and finalizes the queue row — + // 'completed', or 'cancelled' if the memory was superseded + // (content changed → newer version) or deleted between claim + // and embed. const vecLiteral = `[${result.embedding.join(",")}]`; - const updated = await tx.unsafe( - `UPDATE ${schema}.memory - SET embedding = $1::halfvec - WHERE id = $2 AND embedding_version = $3 - RETURNING id`, - [vecLiteral, row.memory_id, row.embedding_version], + await tx.unsafe( + `SELECT ${schema}.complete_embedding($1, $2, $3, $4::halfvec)`, + [ + row.queue_id, + row.memory_id, + row.embedding_version, + vecLiteral, + ], ); - - if (updated.length === 0) { - // Content changed or memory deleted between claim and embed — cancel. - // Queue row may already be CASCADE-deleted; 0 rows updated is fine. - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET outcome = 'cancelled' - WHERE id = $1`, - [row.queue_id], - ); - } else { - // Embedding written — mark completed. - // Queue row may be CASCADE-deleted if memory deleted between these two - // statements; 0 rows updated is fine. - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET outcome = 'completed' - WHERE id = $1`, - [row.queue_id], - ); - } succeeded++; }); } catch (error) { @@ -244,7 +237,6 @@ export async function processBatch( failed++; reportError("Embedding row write-back failed", err, { "worker.schema": schema, - "worker.shard": shard, "queue.id": row.queue_id, "memory.id": row.memory_id, "memory.embedding_version": row.embedding_version, @@ -252,22 +244,15 @@ export async function processBatch( try { await sql.begin(async (tx) => { - await tx.unsafe(`SET LOCAL pgdog.shard TO ${shard}`); - await setLocalEngineTimeouts(tx, timeouts); - await tx.unsafe(`SET LOCAL search_path TO ${schema}, public`); - await tx.unsafe("SET LOCAL ROLE me_embed"); - await tx.unsafe( - `UPDATE ${schema}.embedding_queue - SET last_error = $1 - , outcome = CASE WHEN attempts >= max_attempts THEN 'failed' END - WHERE id = $2 AND outcome IS NULL`, - [err.message, row.queue_id], - ); + await prepareTx(tx as unknown as Sql, schema, timeouts); + await tx.unsafe(`SELECT ${schema}.fail_embedding($1, $2)`, [ + row.queue_id, + err.message, + ]); }); } catch (recordError) { warning("Failed to record embedding row write-back error", { "worker.schema": schema, - "worker.shard": shard, "queue.id": row.queue_id, "memory.id": row.memory_id, error: asError(recordError).message, diff --git a/packages/worker/types.ts b/packages/worker/types.ts index 1388cbf..ee423cc 100644 --- a/packages/worker/types.ts +++ b/packages/worker/types.ts @@ -1,15 +1,29 @@ import type { EmbeddingConfig } from "@memory.build/embedding"; -import type { EngineTimeouts } from "@memory.build/engine/ops/_tx"; -export interface EngineTarget { +/** A space schema (me_) the worker should process. */ +export interface SpaceTarget { schema: string; - shard: number; } +/** Transaction-local timeouts applied to each worker DB transaction. */ +export interface WorkerTimeouts { + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export const DEFAULT_WORKER_TIMEOUTS: WorkerTimeouts = { + statementTimeout: "25s", + lockTimeout: "5s", + transactionTimeout: "30s", + idleInTransactionSessionTimeout: "30s", +}; + export interface WorkerConfig { embedding: EmbeddingConfig; - /** Discover active engines (schema + shard) from accounts DB */ - discover: () => Promise; + /** Discover the spaces (me_ schemas) to process. */ + discover: () => Promise; /** Number of queue entries to claim per batch (default: 10) */ batchSize?: number; /** PostgreSQL interval for claim lock duration (default: '5 minutes') */ @@ -18,10 +32,10 @@ export interface WorkerConfig { idleDelayMs?: number; /** Maximum backoff delay on consecutive errors (default: 60_000ms) */ maxBackoffMs?: number; - /** How often to re-discover engines (default: 60_000ms) */ + /** How often to re-discover spaces (default: 60_000ms) */ refreshIntervalMs?: number; - /** PostgreSQL transaction/session timeouts for worker engine DB work */ - workerEngineTimeouts?: EngineTimeouts; + /** PostgreSQL transaction/session timeouts for worker DB work */ + timeouts?: WorkerTimeouts; /** Exit gracefully after this much idle time (optional) */ drainTimeoutMs?: number; /** @@ -43,11 +57,11 @@ export interface WorkerStats { totalFailed: number; totalPruned: number; /** - * Number of times an engine was dropped from the in-memory target list - * because its schema no longer exists in PostgreSQL (e.g. engine deleted + * Number of times a space was dropped from the in-memory target list + * because its schema no longer exists in PostgreSQL (e.g. space deleted * between discover() refreshes). Self-heals on the next refresh. */ - enginesDropped: number; + spacesDropped: number; consecutiveErrors: number; lastError?: string; } diff --git a/packages/worker/worker.test.ts b/packages/worker/worker.test.ts index ebd8de2..22999f4 100644 --- a/packages/worker/worker.test.ts +++ b/packages/worker/worker.test.ts @@ -1,6 +1,5 @@ import { describe, expect, test } from "bun:test"; import { RateLimitError } from "@memory.build/embedding"; -import { SQL } from "bun"; import { WorkerPool } from "./pool"; import { Worker } from "./worker"; @@ -76,7 +75,7 @@ describe("Worker", () => { }, discover: async () => { discoverCalls++; - return [{ schema: "me_test12345678", shard: 1 }]; + return [{ schema: "me_test12345678" }]; }, idleDelayMs: 50, drainTimeoutMs: 100, @@ -144,7 +143,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -204,7 +203,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -243,7 +242,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -281,9 +280,10 @@ describe("Worker", () => { } if (q.includes("claim_embedding_batch")) { if (lastSchema === deadSchema) { - throw new SQL.PostgresError( - `schema "${deadSchema}" does not exist`, - { code: "3F000", errno: "3F000" }, + // postgres.js surfaces SQLSTATE on `.code`; 3F000 = missing schema + throw Object.assign( + new Error(`schema "${deadSchema}" does not exist`), + { code: "3F000" }, ); } claimedBy.push(lastSchema ?? ""); @@ -302,10 +302,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [ - { schema: liveSchema, shard: 1 }, - { schema: deadSchema, shard: 1 }, - ], + discover: async () => [{ schema: liveSchema }, { schema: deadSchema }], idleDelayMs: 50, drainTimeoutMs: 200, refreshIntervalMs: 1_000_000, // don't refresh during test @@ -319,9 +316,9 @@ describe("Worker", () => { expect(worker.stats.consecutiveErrors).toBe(0); expect(worker.stats.lastError).toBeUndefined(); // Dead schema dropped exactly once, then filtered out of the rotation. - expect(worker.stats.enginesDropped).toBe(1); + expect(worker.stats.spacesDropped).toBe(1); // Live schema kept being polled across multiple loop iterations even - // though we only saw one enginesDropped — proves the live target wasn't + // though we only saw one spacesDropped — proves the live target wasn't // collateral damage when its sibling threw. expect(claimedBy.length).toBeGreaterThanOrEqual(2); expect(claimedBy.every((s) => s === liveSchema)).toBe(true); @@ -390,7 +387,7 @@ describe("Worker", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 200, refreshIntervalMs: 1_000_000, @@ -433,7 +430,7 @@ describe("WorkerPool", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 50, drainTimeoutMs: 150, refreshIntervalMs: 1_000_000, @@ -540,7 +537,7 @@ describe("WorkerPool", () => { model: "test", dimensions: 3, }, - discover: async () => [{ schema: "me_test12345678", shard: 1 }], + discover: async () => [{ schema: "me_test12345678" }], idleDelayMs: 60_000, maxBackoffMs: 60_000, refreshIntervalMs: 1_000_000, diff --git a/packages/worker/worker.ts b/packages/worker/worker.ts index 055d45c..530098b 100644 --- a/packages/worker/worker.ts +++ b/packages/worker/worker.ts @@ -1,6 +1,6 @@ import { RateLimitError } from "@memory.build/embedding"; import { info, reportError, warning } from "@pydantic/logfire-node"; -import { SQL } from "bun"; +import type { Sql } from "postgres"; import { processBatch, pruneQueue } from "./process"; import type { WorkerConfig, WorkerStats } from "./types"; @@ -8,12 +8,12 @@ import type { WorkerConfig, WorkerStats } from "./types"; const RATE_LIMIT_FLOOR_MS = 30_000; /** - * SQLSTATE 3F000 = invalid_schema_name. Raised when the engine's schema - * no longer exists — typically because the engine was deleted between - * discover() refreshes. Treated as benign: drop the target and continue. + * SQLSTATE 3F000 = invalid_schema_name. Raised when the space's schema no + * longer exists — typically because the space was deleted between discover() + * refreshes. Treated as benign: drop the target and continue. */ function isMissingSchemaError(error: unknown): boolean { - return error instanceof SQL.PostgresError && error.errno === "3F000"; + return (error as { code?: string }).code === "3F000"; } /** @@ -51,14 +51,14 @@ function shuffle(arr: T[]): T[] { } /** - * Embedding worker. Discovers engines from the accounts DB and polls their - * embedding queues in round-robin, generating embeddings for new memories. + * Embedding worker. Discovers spaces and polls their embedding queues in + * round-robin, generating embeddings for new memories. * * Adaptive delay: loops immediately when work is found, sleeps idleDelayMs * when idle. Exponential backoff on consecutive errors. */ export class Worker { - private readonly sql: SQL; + private readonly sql: Sql; private readonly config: WorkerConfig; private abort: AbortController | null = null; private runPromise: Promise | null = null; @@ -67,11 +67,11 @@ export class Worker { totalProcessed: 0, totalFailed: 0, totalPruned: 0, - enginesDropped: 0, + spacesDropped: 0, consecutiveErrors: 0, }; - constructor(sql: SQL, config: WorkerConfig) { + constructor(sql: Sql, config: WorkerConfig) { this.sql = sql; this.config = config; } @@ -105,7 +105,7 @@ export class Worker { } async function run( - sql: SQL, + sql: Sql, config: WorkerConfig, signal: AbortSignal, stats: WorkerStats, @@ -153,10 +153,9 @@ async function run( if (isMissingSchemaError(err)) { warning("Embedding target schema no longer exists, dropping", { "worker.schema": target.schema, - "worker.shard": target.shard, }); droppedSchemas.add(target.schema); - stats.enginesDropped++; + stats.spacesDropped++; continue; } throw err; @@ -181,11 +180,10 @@ async function run( // Schema dropped between claim and prune in the same cycle. // Drop the target now to avoid re-trying it next cycle. droppedSchemas.add(target.schema); - stats.enginesDropped++; + stats.spacesDropped++; } else { warning("Embedding queue prune failed", { "worker.schema": target.schema, - "worker.shard": target.shard, error: pruneError instanceof Error ? pruneError.message @@ -229,7 +227,7 @@ async function run( warning("Rate limited by embedding provider, backing off", { backoffMs, retryAfterMs: error.retryAfterMs, - engineCount: targets.length, + spaceCount: targets.length, }); if (signal.aborted) break; @@ -243,7 +241,7 @@ async function run( stats.lastError = errorMsg; reportError("Worker batch processing failed", error as Error, { consecutiveErrors, - engineCount: targets.length, + spaceCount: targets.length, }); if (signal.aborted) break; @@ -258,7 +256,7 @@ async function run( } finally { info("Embedding worker stopped", { consecutiveErrors, - engineCount: targets.length, + spaceCount: targets.length, }); } } diff --git a/packs/README.md b/packs/README.md index f88ea02..3737990 100644 --- a/packs/README.md +++ b/packs/README.md @@ -124,7 +124,7 @@ Old-version memories are automatically cleaned up. Memories that exist in both v 3. Generate deterministic memory IDs with `./bun run scripts/pack-uuids.ts ` 4. Validate with `me pack validate packs/your-pack.yaml` 5. Run the cross-pack check: `./bun run scripts/validate-packs.ts` -6. Run the full repo check: `./bun run check` +6. Run the full repo check: `./bun run check:full` 7. Open a pull request All packs must: diff --git a/scripts/clean-test-schemas.ts b/scripts/clean-test-schemas.ts new file mode 100644 index 0000000..f457535 --- /dev/null +++ b/scripts/clean-test-schemas.ts @@ -0,0 +1,199 @@ +#!/usr/bin/env bun +// +// Reclaim orphaned integration-test schemas from the test database. +// +// Integration tests provision throwaway schemas and drop them in teardown, but +// a hard interruption (SIGKILL, OOM, a timed-out `beforeAll`) can leave one +// behind. This script sweeps those leftovers. It runs automatically before +// `test:db` (see package.json) and can be run by hand. +// +// SAFETY — this script issues `drop schema ... cascade`, so it only ever targets +// schema names that are impossible in production: +// +// * `core_test_*` — core tests; production's control plane is the bare `core`. +// * `auth_test_*` — auth tests; production's auth schema is the bare `auth`. +// * `metest_*` — space tests; production spaces are `me_`. Tests +// deliberately provision under the `metest_` prefix (see +// packages/database/space/migrate/test-utils.ts) so they +// never share a name with a real space, and `metest_` does +// not start with the `me_` engine prefix. +// +// The result: pointed at a real database, this script is a no-op — neither +// pattern can match a production schema. +// +// By default it only drops schemas older than --older-than-min (default 60), +// using each schema's `version.at` timestamp. That keeps it safe to run as a +// pre-step even while another `test:db` invocation shares the database: that +// run's freshly-provisioned schemas are younger than the threshold and are left +// alone. Pass --all to ignore age (a deliberate full reset — only do this when +// nothing else is using the database). + +import postgres, { type Sql } from "postgres"; + +const DEFAULT_TEST_DATABASE_URL = + "postgresql://postgres@127.0.0.1:5432/postgres"; + +// Production-impossible test schema patterns. +// core_test_, auth_test_, metest_. +const TEST_SCHEMA_PATTERNS = [ + "^core_test_[a-z0-9]+$", + "^auth_test_[a-z0-9]+$", + "^metest_[a-z0-9]{12}$", +]; + +const SAFE_IDENTIFIER = /^[a-z_][a-z0-9_]*$/; + +interface Options { + all: boolean; + olderThanMin: number; + quiet: boolean; +} + +function parseArgs(argv: string[]): Options { + const opts: Options = { all: false, olderThanMin: 60, quiet: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--help" || arg === "-h") { + console.log( + `Usage: ./bun scripts/clean-test-schemas.ts [--all] [--older-than-min N] [--quiet] + +Drops orphaned integration-test schemas from TEST_DATABASE_URL (default +${DEFAULT_TEST_DATABASE_URL}): core_test_*, auth_test_*, and metest_* schemas. +Safe against production databases (no-op there — no pattern can match a real +schema). + + --all Ignore age; drop every matching schema. Use only when no + other test run shares the database. + --older-than-min N Only drop schemas older than N minutes (default 60). + --quiet Only print when something is dropped or on error.`, + ); + process.exit(0); + } else if (arg === "--all") { + opts.all = true; + } else if (arg === "--quiet") { + opts.quiet = true; + } else if (arg === "--older-than-min") { + const next = argv[++i]; + const n = Number(next); + if (!Number.isFinite(n) || n < 0) { + console.error(`Invalid --older-than-min value: ${next}`); + process.exit(2); + } + opts.olderThanMin = n; + } else { + console.error(`Unknown argument: ${arg}`); + process.exit(2); + } + } + return opts; +} + +function testDatabaseUrl(): string { + return process.env.TEST_DATABASE_URL ?? DEFAULT_TEST_DATABASE_URL; +} + +async function findCandidates(sql: Sql): Promise { + // OR the (constant, production-impossible) patterns as bound `~` tests. + const match = TEST_SCHEMA_PATTERNS.map((p) => sql`n.nspname ~ ${p}`).reduce( + (acc, frag) => sql`${acc} or ${frag}`, + ); + const rows = await sql<{ schema: string }[]>` + select n.nspname as schema + from pg_namespace n + where ${match} + order by n.nspname + `; + return rows.map((r) => r.schema); +} + +/** + * Age of a schema in minutes from its singleton `version.at`, or null when the + * schema has no readable version row (a partial/failed provision). In safe mode + * those are skipped — they may belong to a concurrent run still provisioning. + */ +async function schemaAgeMinutes( + sql: Sql, + schema: string, +): Promise { + try { + const rows = await sql.unsafe<{ age_min: number }[]>( + `select extract(epoch from (now() - at)) / 60 as age_min + from ${schema}.version + limit 1`, + ); + const [row] = rows; + return row ? Number(row.age_min) : null; + } catch { + return null; + } +} + +async function main(): Promise { + const opts = parseArgs(process.argv.slice(2)); + const log = (msg: string) => { + if (!opts.quiet) console.error(msg); + }; + + // Short timeouts: never let cleanup hang a test run on a lock. statement_timeout + // and lock_timeout are libpq startup params here, in milliseconds. + const sql = postgres(testDatabaseUrl(), { + max: 1, + connect_timeout: 10, + idle_timeout: 5, + onnotice: () => {}, + connection: { statement_timeout: 10_000, lock_timeout: 3_000 }, + }); + + const dropped: string[] = []; + try { + const candidates = await findCandidates(sql); + if (candidates.length === 0) { + log("[clean-test-schemas] no test schemas found."); + return; + } + + for (const schema of candidates) { + // Defense in depth: the patterns already constrain this, but never + // interpolate a name that isn't a plain lowercase identifier. + if (!SAFE_IDENTIFIER.test(schema) || schema.length > 63) { + log(`[clean-test-schemas] skip ${schema}: unexpected identifier`); + continue; + } + + if (!opts.all) { + const age = await schemaAgeMinutes(sql, schema); + if (age === null) { + log(`[clean-test-schemas] skip ${schema}: no version row (recent?)`); + continue; + } + if (age < opts.olderThanMin) { + log( + `[clean-test-schemas] skip ${schema}: ${age.toFixed(0)}m old < ${opts.olderThanMin}m`, + ); + continue; + } + } + + try { + await sql.unsafe(`drop schema if exists ${schema} cascade`); + dropped.push(schema); + } catch (error) { + log(`[clean-test-schemas] failed to drop ${schema}: ${error}`); + } + } + } catch (error) { + // Best-effort: never block the test run on a cleanup hiccup. If the database + // is genuinely unreachable, the tests themselves will surface it. + console.error(`[clean-test-schemas] skipped (cleanup error): ${error}`); + } finally { + await sql.end(); + } + + if (dropped.length > 0) { + console.error( + `[clean-test-schemas] dropped ${dropped.length} schema(s): ${dropped.join(", ")}`, + ); + } +} + +await main(); diff --git a/scripts/generate-master-key.ts b/scripts/generate-master-key.ts deleted file mode 100644 index f7433b0..0000000 --- a/scripts/generate-master-key.ts +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bun -/** - * Generate a random 32-byte (256-bit) master key for ACCOUNTS_MASTER_KEY. - * - * Usage: - * ./bun scripts/generate-master-key.ts - */ -import { randomBytes } from "node:crypto"; - -const key = randomBytes(32).toString("hex"); -console.log(key); diff --git a/scripts/integration-test.ts b/scripts/integration-test.ts index 47fc159..1f38cfa 100644 --- a/scripts/integration-test.ts +++ b/scripts/integration-test.ts @@ -1053,6 +1053,18 @@ async function phase5_memory(): Promise { expectEq(json.failed, 0, "failed count"); }); + // Canonical spelling of the same command (`me memory import` is its alias). + await step("me import memories (--dry-run)", async () => { + const { json } = await runJson<{ wouldImport: number; dryRun: boolean }>([ + "import", + "memories", + join(fixturesDir, "sample.md"), + "--dry-run", + ]); + expectEq(json.dryRun, true, "dryRun"); + expectEq(json.wouldImport, 1, "wouldImport"); + }); + await step("me memory import (recursive --dry-run)", async () => { // exclude the pack yaml from the picture by importing only the // sample files via the directory recursion. diff --git a/scripts/migrate-db.ts b/scripts/migrate-db.ts new file mode 100644 index 0000000..20c1b53 --- /dev/null +++ b/scripts/migrate-db.ts @@ -0,0 +1,140 @@ +#!/usr/bin/env bun +import { + bootstrapSpaceDatabase, + CORE_SCHEMA_VERSION, + migrateCore, + migrateSpace, + SPACE_SCHEMA_VERSION, + slugToSchema, +} from "@memory.build/database"; +import postgres from "postgres"; + +const DEFAULT_DATABASE_URL = "postgresql://postgres@127.0.0.1:5432/postgres"; +const DEFAULT_SPACE_SLUG = "dev000000001"; + +type Mode = "all" | "core" | "space-db" | "space"; + +function usage(): string { + return `Usage: ./bun run migrate:db [all|core|space-db|space] + +Environment: + DATABASE_URL Postgres connection string. Defaults to ${DEFAULT_DATABASE_URL} + SPACE_SLUG Space slug to migrate. Defaults to ${DEFAULT_SPACE_SLUG} + +Modes: + all Migrate core, prepare database for spaces, and migrate the dev space. Default. + core Migrate only the core schema. + space-db Prepare only the physical database for spaces. + space Prepare the database for spaces and migrate one space. +`; +} + +function parseMode(arg: string | undefined): Mode { + if (!arg) return "all"; + if (arg === "--help" || arg === "-h") { + console.log(usage()); + process.exit(0); + } + if ( + arg === "all" || + arg === "core" || + arg === "space-db" || + arg === "space" + ) { + return arg; + } + console.error(`Invalid migration mode: ${arg}`); + console.error(usage()); + process.exit(1); +} + +function databaseUrl(): string { + return process.env.DATABASE_URL ?? DEFAULT_DATABASE_URL; +} + +async function main(): Promise { + const mode = parseMode(process.argv[2]); + const url = databaseUrl(); + const spaceSlug = process.env.SPACE_SLUG ?? DEFAULT_SPACE_SLUG; + const sql = postgres(url, { onnotice: () => {} }); + + console.log(`Database: ${url}`); + console.log(`Mode: ${mode}`); + console.log(`Space slug: ${spaceSlug}`); + console.log(`Core schema version: ${CORE_SCHEMA_VERSION}`); + console.log(`Space schema version: ${SPACE_SCHEMA_VERSION}`); + console.log(""); + + try { + if (mode === "all" || mode === "core") { + await migrateCore(sql, { logSqlFiles: true }); + console.log("Migrated core."); + } + + if (mode === "all" || mode === "space-db" || mode === "space") { + await bootstrapSpaceDatabase(sql); + console.log("Prepared database for spaces."); + } + + if (mode === "all" || mode === "space") { + await migrateSpace(sql, { slug: spaceSlug, logSqlFiles: true }); + console.log(`Migrated space ${slugToSchema(spaceSlug)}.`); + } + } finally { + await sql.end(); + } +} + +main().catch((error) => { + console.error(""); + console.error( + "Migration failed:", + error instanceof Error ? error.message : error, + ); + printErrorDetails(error); + process.exit(1); +}); + +function printErrorDetails(error: unknown): void { + if (!error || typeof error !== "object") return; + + const details = error as Record; + const keys = [ + "name", + "errno", + "code", + "severity", + "detail", + "hint", + "position", + "internalPosition", + "internalQuery", + "where", + "schema", + "table", + "column", + "dataType", + "constraint", + "file", + "line", + "routine", + ]; + const seen = new Set(keys); + const extraKeys = [ + ...Object.getOwnPropertyNames(error), + ...Object.keys(details), + ].filter((key) => !seen.has(key) && key !== "message" && key !== "stack"); + + const entries = [...keys, ...extraKeys] + .map((key) => [key, details[key]] as const) + .filter( + ([, value]) => value !== undefined && value !== null && value !== "", + ); + + if (entries.length === 0) return; + + console.error("Postgres error details:"); + for (const [key, value] of entries) { + console.error(` ${key}: ${String(value)}`); + } +} diff --git a/scripts/package.json b/scripts/package.json index 8bd79d2..69cefac 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -3,7 +3,9 @@ "private": true, "type": "module", "dependencies": { + "@memory.build/database": "workspace:*", "@memory.build/embedding": "workspace:*", + "postgres": "^3.4.9", "yaml": "^2.7.0" } } diff --git a/scripts/release-server.ts b/scripts/release-server.ts index 1096d30..41e8bdf 100644 --- a/scripts/release-server.ts +++ b/scripts/release-server.ts @@ -25,11 +25,10 @@ import semver from "semver"; const root = join(import.meta.dirname, ".."); // Server-side package.json files. `packages/server/package.json` is the -// canonical source of truth for SERVER_VERSION; the other four are sibling +// canonical source of truth for SERVER_VERSION; the others are sibling // internal packages consumed by the server via `workspace:*`. They are // bumped in lockstep for visual consistency — none of them publish. const PACKAGE_JSONS = [ - "packages/accounts/package.json", "packages/embedding/package.json", "packages/engine/package.json", "packages/server/package.json", diff --git a/scripts/setup.ts b/scripts/setup.ts index aff03ca..f02a336 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -11,7 +11,7 @@ * * Prerequisites: * 1. Postgres running (`./bun run pg`) - * 2. .env filled in (ACCOUNTS_DATABASE_URL, ENGINE_DATABASE_URL) + * 2. .env filled in (DATABASE_URL) * * Usage: * ./bun run setup @@ -32,8 +32,7 @@ function requireEnv(name: string): string { return value; } -const ACCOUNTS_DATABASE_URL = requireEnv("ACCOUNTS_DATABASE_URL"); -const ENGINE_DATABASE_URL = requireEnv("ENGINE_DATABASE_URL"); +const DATABASE_URL = requireEnv("DATABASE_URL"); // ============================================================================= // Database creation @@ -75,9 +74,8 @@ async function main() { console.log("=== Memory Engine: Local Dev Setup ==="); console.log(""); - console.log("Ensuring databases exist..."); - await ensureDatabase(ACCOUNTS_DATABASE_URL); - await ensureDatabase(ENGINE_DATABASE_URL); + console.log("Ensuring database exists..."); + await ensureDatabase(DATABASE_URL); console.log(""); console.log("Done! Start the server to run bootstrap + migrations:");