fix(rest): REST data API honors sys_api_key — shared verifier with MCP (closes #1633)

.changeset/rest-honors-api-key.md

@@ -0,0 +1,32 @@
+---
+'@objectstack/core': minor
+'@objectstack/rest': patch
+'@objectstack/runtime': patch
+---
+
+fix(rest): REST data API honors sys_api_key — one shared verifier with MCP (closes #1633)
+
+Staging e2e found the MCP surface authenticated a `sys_api_key` but the REST data
+API (`@objectstack/rest`) returned 401 for the same key — its `resolveExecCtx`
+only checked the better-auth session, never the API key.
+
+Converged both surfaces onto ONE verifier so they can't drift:
+
+- **`@objectstack/core/security`** now owns the shared `sys_api_key` primitives
+  (`hashApiKey`, `generateApiKey`, `extractApiKey`, `parseScopes`, `isExpired`)
+  plus a new `resolveApiKeyPrincipal(ql, headers, nowMs?)` that hashes the
+  inbound key, looks it up by the indexed at-rest hash, and rejects unknown /
+  revoked / expired / owner-less keys (fail-closed). `core` is the natural home:
+  both `rest` and `runtime` depend on it, it depends on neither (no cycle), and
+  it's server-side (already uses `node:crypto`).
+- **`@objectstack/runtime`** — `security/api-key.ts` re-exports the primitives
+  from core (stable import surface) and `resolveExecutionContext` now delegates
+  its API-key branch to `resolveApiKeyPrincipal`.
+- **`@objectstack/rest`** — `resolveExecCtx` resolves the data engine once and
+  tries `resolveApiKeyPrincipal` (x-api-key / `Authorization: ApiKey`) BEFORE the
+  session, so `/api/v1/data` + `/api/v1/meta` now authenticate an API key under
+  the key's permissions + RLS, exactly like the dispatcher/MCP path.
+
+Tests: core `api-key.test.ts` (primitives + verifier: valid / revoked / expired /
+unknown / owner-less / plaintext-not-matched / fail-closed-ql). runtime + rest
+suites green.

packages/core/src/security/api-key.test.ts

@@ -0,0 +1,93 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect } from 'vitest';
+
+import {
+  hashApiKey,
+  generateApiKey,
+  extractApiKey,
+  parseScopes,
+  isExpired,
+  resolveApiKeyPrincipal,
+} from './api-key.js';
+
+/** In-memory sys_api_key store exposing the `find` shape the verifier uses. */
+function makeQl(rows: any[]) {
+  return {
+    find: async (object: string, opts: any) => {
+      if (object !== 'sys_api_key') return [];
+      const where = opts?.where ?? {};
+      return rows.filter((r) => Object.entries(where).every(([k, v]) => r[k] === v));
+    },
+  };
+}
+
+const FUTURE = '2999-01-01T00:00:00Z';
+const PAST = '2000-01-01T00:00:00Z';
+
+describe('core api-key primitives', () => {
+  it('hashApiKey is deterministic sha256 hex, never the raw', () => {
+    expect(hashApiKey('osk_a')).toBe(hashApiKey('osk_a'));
+    expect(hashApiKey('osk_a')).toMatch(/^[0-9a-f]{64}$/);
+    expect(hashApiKey('osk_secret')).not.toContain('secret');
+  });
+
+  it('generateApiKey: prefix + base64url secret, hash matches', () => {
+    const k = generateApiKey();
+    expect(k.raw.startsWith('osk_')).toBe(true);
+    expect(k.hash).toBe(hashApiKey(k.raw));
+    expect(k.raw.startsWith(k.prefix)).toBe(true);
+  });
+
+  it('extractApiKey: x-api-key / ApiKey scheme, not Bearer', () => {
+    expect(extractApiKey({ 'x-api-key': 'k' })).toBe('k');
+    expect(extractApiKey({ authorization: 'ApiKey k' })).toBe('k');
+    expect(extractApiKey({ authorization: 'Bearer k' })).toBeUndefined();
+  });
+
+  it('parseScopes + isExpired basics', () => {
+    expect(parseScopes('["a","b"]')).toEqual(['a', 'b']);
+    expect(isExpired(PAST, Date.now())).toBe(true);
+    expect(isExpired(FUTURE, Date.now())).toBe(false);
+    expect(isExpired(null, Date.now())).toBe(false);
+  });
+});
+
+describe('resolveApiKeyPrincipal (shared verifier)', () => {
+  it('resolves a valid key to its principal (x-api-key)', async () => {
+    const raw = 'osk_valid';
+    const ql = makeQl([
+      { key: hashApiKey(raw), revoked: false, user_id: 'u1', organization_id: 'org1', scopes: '["read"]', expires_at: FUTURE },
+    ]);
+    const p = await resolveApiKeyPrincipal(ql, { 'x-api-key': raw });
+    expect(p).toEqual({ userId: 'u1', tenantId: 'org1', scopes: ['read'] });
+  });
+
+  it('resolves via Authorization: ApiKey', async () => {
+    const raw = 'osk_valid';
+    const ql = makeQl([{ key: hashApiKey(raw), revoked: false, user_id: 'u1' }]);
+    const p = await resolveApiKeyPrincipal(ql, { authorization: `ApiKey ${raw}` });
+    expect(p?.userId).toBe('u1');
+  });
+
+  it('returns undefined for no key / revoked / expired / unknown / owner-less', async () => {
+    const raw = 'osk_x';
+    const base = (extra: any) => makeQl([{ key: hashApiKey(raw), revoked: false, user_id: 'u1', ...extra }]);
+    expect(await resolveApiKeyPrincipal(base({}), {})).toBeUndefined(); // no key header
+    expect(await resolveApiKeyPrincipal(makeQl([{ key: hashApiKey(raw), revoked: true, user_id: 'u1' }]), { 'x-api-key': raw })).toBeUndefined();
+    expect(await resolveApiKeyPrincipal(base({ expires_at: PAST }), { 'x-api-key': raw })).toBeUndefined();
+    expect(await resolveApiKeyPrincipal(base({}), { 'x-api-key': 'osk_wrong' })).toBeUndefined();
+    expect(await resolveApiKeyPrincipal(makeQl([{ key: hashApiKey(raw), revoked: false }]), { 'x-api-key': raw })).toBeUndefined(); // no user_id
+  });
+
+  it('never matches a plaintext-stored key (hash lookup only)', async () => {
+    const raw = 'osk_plain';
+    const ql = makeQl([{ key: raw, revoked: false, user_id: 'u1' }]);
+    expect(await resolveApiKeyPrincipal(ql, { 'x-api-key': raw })).toBeUndefined();
+  });
+
+  it('fail-closed when ql is missing/unusable', async () => {
+    expect(await resolveApiKeyPrincipal(undefined, { 'x-api-key': 'osk_x' })).toBeUndefined();
+    expect(await resolveApiKeyPrincipal({}, { 'x-api-key': 'osk_x' })).toBeUndefined();
+  });
+});

packages/core/src/security/api-key.ts

@@ -0,0 +1,195 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * api-key — hand-rolled API-key primitives + verifier for `sys_api_key`.
+ *
+ * better-auth 1.6.x ships no apiKey plugin, so ObjectStack owns the full
+ * lifecycle: generation, at-rest hashing, header extraction, validation, and
+ * the verify-time principal lookup. This is the SINGLE shared source of truth
+ * used by BOTH inbound surfaces — the runtime dispatcher / MCP path
+ * (`resolveExecutionContext`) and the REST data API (`@objectstack/rest`) — so
+ * the two can never drift on how a key authenticates. It lives in
+ * `@objectstack/core` (server-side; both `runtime` and `rest` depend on it,
+ * and `core` depends on neither, so there is no cycle).
+ *
+ * SECURITY (zero-tolerance):
+ *  - The raw key is returned EXACTLY ONCE, by {@link generateApiKey}. It is
+ *    never persisted; only `sha256(raw)` (hex) is stored in `sys_api_key.key`.
+ *  - The raw key and its hash must never enter logs, HTTP responses, error
+ *    messages, commit messages or comments.
+ *  - Validation is fail-closed: anything ambiguous (missing, revoked, expired,
+ *    malformed) resolves to "no principal", never to an elevated one.
+ */
+
+import { createHash, randomBytes } from 'node:crypto';
+
+/** Default visible prefix for generated keys (helps users identify a key). */
+export const API_KEY_PREFIX = 'osk_';
+
+/** Bytes of entropy in the secret portion of a generated key (256 bits). */
+const API_KEY_ENTROPY_BYTES = 32;
+
+/** Length of the human-visible prefix stored in `sys_api_key.prefix`. */
+const VISIBLE_PREFIX_LEN = 12;
+
+/**
+ * Derive the at-rest hash for an API key. Inbound keys are hashed the same way
+ * before the DB lookup. Because the lookup matches an indexed, high-entropy
+ * hash exactly, this doubles as a constant-effort comparison: an attacker
+ * cannot recover the raw key by probing for partial matches.
+ */
+export function hashApiKey(raw: string): string {
+  return createHash('sha256').update(raw, 'utf8').digest('hex');
+}
+
+/** Result of {@link generateApiKey}. `raw` is shown to the user only once. */
+export interface GeneratedApiKey {
+  /** The full secret to hand to the client. NEVER persist this. */
+  raw: string;
+  /** `sha256(raw)` hex — store this in `sys_api_key.key`. */
+  hash: string;
+  /** Short non-secret prefix for display/identification (`sys_api_key.prefix`). */
+  prefix: string;
+}
+
+/**
+ * Generate a fresh API key. Returns the raw secret (caller must surface it to
+ * the user exactly once and then discard it), its at-rest hash, and a short
+ * non-secret prefix for display.
+ */
+export function generateApiKey(prefix: string = API_KEY_PREFIX): GeneratedApiKey {
+  // base64url so the token is URL/header-safe with no padding.
+  const secret = randomBytes(API_KEY_ENTROPY_BYTES).toString('base64url');
+  const raw = `${prefix}${secret}`;
+  return {
+    raw,
+    hash: hashApiKey(raw),
+    prefix: raw.slice(0, VISIBLE_PREFIX_LEN),
+  };
+}
+
+/**
+ * Extract an API key from request headers. Accepts `X-API-Key: <token>` or
+ * `Authorization: ApiKey <token>` (case-insensitive scheme). Bearer tokens are
+ * deliberately NOT treated as API keys — those flow through the session path.
+ */
+export function extractApiKey(headers: any): string | undefined {
+  const x = readHeader(headers, 'x-api-key');
+  if (x && x.trim()) return x.trim();
+  const auth = readHeader(headers, 'authorization');
+  if (!auth) return undefined;
+  const m = auth.match(/^ApiKey\s+(.+)$/i);
+  const token = m?.[1]?.trim();
+  return token || undefined;
+}
+
+/** Parse a `scopes` value that may be a JSON-string textarea or a real array. */
+export function parseScopes(value: unknown): string[] {
+  if (Array.isArray(value)) {
+    return value.filter((s): s is string => typeof s === 'string' && s.length > 0);
+  }
+  if (typeof value === 'string' && value.trim()) {
+    const parsed = safeJsonParse<unknown>(value, []);
+    if (Array.isArray(parsed)) {
+      return parsed.filter((s): s is string => typeof s === 'string' && s.length > 0);
+    }
+  }
+  return [];
+}
+
+/** Return true when an expiry timestamp is in the past (i.e. the key is dead). */
+export function isExpired(value: unknown, nowMs: number): boolean {
+  if (value == null) return false;
+  let ms: number;
+  if (typeof value === 'number') {
+    // Heuristic: seconds vs milliseconds epoch.
+    ms = value < 1e12 ? value * 1000 : value;
+  } else if (value instanceof Date) {
+    ms = value.getTime();
+  } else if (typeof value === 'string') {
+    ms = Date.parse(value);
+  } else {
+    return false;
+  }
+  if (Number.isNaN(ms)) return false;
+  return ms <= nowMs;
+}
+
+/** The principal resolved from a valid `sys_api_key`. */
+export interface ApiKeyPrincipal {
+  userId: string;
+  tenantId?: string;
+  scopes: string[];
+}
+
+/**
+ * Verify an inbound API key against `sys_api_key` and resolve its principal.
+ * This is the ONE verify path shared by the dispatcher/MCP and REST surfaces.
+ *
+ * Fail-closed: returns `undefined` for a missing key, an unusable data engine,
+ * a lookup error, or a key that is unknown / revoked / expired / owner-less.
+ *
+ * @param ql      A data engine with `find(object, { where, limit, context })`.
+ * @param headers Request headers (Web `Headers` or a plain object).
+ * @param nowMs   Clock for expiry checks (injectable for tests).
+ */
+export async function resolveApiKeyPrincipal(
+  ql: any,
+  headers: any,
+  nowMs: number = Date.now(),
+): Promise<ApiKeyPrincipal | undefined> {
+  const apiKey = extractApiKey(headers);
+  if (!apiKey) return undefined;
+  if (!ql || typeof ql.find !== 'function') return undefined;
+
+  // Match by the indexed at-rest hash only — never query by the raw key.
+  let rows: any;
+  try {
+    rows = await ql.find('sys_api_key', {
+      where: { key: hashApiKey(apiKey), revoked: false },
+      limit: 1,
+      context: { isSystem: true },
+    });
+  } catch {
+    return undefined;
+  }
+  if (rows && (rows as any).value) rows = (rows as any).value;
+  const row = Array.isArray(rows) ? rows[0] : undefined;
+  if (!row || row.revoked === true) return undefined;
+
+  const expiresAt = row.expires_at ?? row.expiresAt;
+  if (isExpired(expiresAt, nowMs)) return undefined;
+
+  const userId = row.user_id ?? row.userId;
+  if (!userId || typeof userId !== 'string') return undefined;
+
+  return {
+    userId,
+    tenantId: row.organization_id ?? row.organizationId ?? undefined,
+    scopes: parseScopes(row.scopes),
+  };
+}
+
+function readHeader(headers: any, name: string): string | undefined {
+  if (!headers) return undefined;
+  const lower = name.toLowerCase();
+  if (typeof headers.get === 'function') {
+    const v = headers.get(name) ?? headers.get(lower);
+    return v == null ? undefined : String(v);
+  }
+  for (const key of Object.keys(headers)) {
+    if (key.toLowerCase() === lower) {
+      const v = headers[key];
+      return Array.isArray(v) ? v[0] : v == null ? undefined : String(v);
+    }
+  }
+  return undefined;
+}
+
+function safeJsonParse<T>(s: string, fallback: T): T {
+  try {
+    return JSON.parse(s) as T;
+  } catch {
+    return fallback;
+  }
+}

packages/core/src/security/index.ts

@@ -67,3 +67,15 @@ export {
   type ScanTarget,
   type SecurityIssue,
 } from './security-scanner.js';
+
+export {
+  API_KEY_PREFIX,
+  hashApiKey,
+  generateApiKey,
+  extractApiKey,
+  parseScopes,
+  isExpired,
+  resolveApiKeyPrincipal,
+  type GeneratedApiKey,
+  type ApiKeyPrincipal,
+} from './api-key.js';

packages/rest/src/rest-server.ts

@@ -1,6 +1,6 @@
 // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
 
-import { IHttpServer } from '@objectstack/core';
+import { IHttpServer, resolveApiKeyPrincipal } from '@objectstack/core';
 import { RouteManager } from './route-manager.js';
 import { RestServerConfig, RestApiConfig, CrudEndpointsConfig, MetadataEndpointsConfig, BatchEndpointsConfig, RouteGenerationConfig } from '@objectstack/spec/api';
 import { ObjectStackProtocol } from '@objectstack/spec/api';
@@ -788,13 +788,35 @@ export class RestServer {
                 return undefined;
             }
 
-            const session = await api.getSession({ headers });
-            if (!session?.user?.id) return undefined;
-            const userId = session.user.id;
-            const tenantId = session.session?.activeOrganizationId ?? undefined;
             const permissions: string[] = [];
             const systemPermissions: string[] = [];
             const roles: string[] = [];
+
+            // Resolve the data engine once — needed by the API-key verifier and
+            // reused by the role/permission lookups below.
+            let identityQl: any;
+            if (kernel) identityQl = await kernel.getServiceAsync('objectql').catch(() => undefined);
+            if (!identityQl && this.objectQLProvider) {
+                identityQl = await this.objectQLProvider(environmentId).catch(() => undefined);
+            }
+
+            // ── Identity: API key (sys_api_key) takes precedence, then session.
+            //    Verified by the SAME `resolveApiKeyPrincipal` (@objectstack/core)
+            //    the dispatcher/MCP path uses, so REST + MCP never drift on how a
+            //    key authenticates. Anonymous (neither) → undefined → 401.
+            let userId: string;
+            let tenantId: string | undefined;
+            const keyPrincipal = await resolveApiKeyPrincipal(identityQl, headers).catch(() => undefined);
+            if (keyPrincipal) {
+                userId = keyPrincipal.userId;
+                tenantId = keyPrincipal.tenantId;
+                for (const s of keyPrincipal.scopes) if (!permissions.includes(s)) permissions.push(s);
+            } else {
+                const session = await api.getSession({ headers });
+                if (!session?.user?.id) return undefined;
+                userId = session.user.id;
+                tenantId = session.session?.activeOrganizationId ?? undefined;
+            }
             // Look up the link tables to surface roles + permission set names.
             // Skipping this lookup would silently ignore admin/role grants —
             // including the platform-admin promotion seeded by