feat(packages): consolidate package subsystem so AI-built app packages surface in Studio

.changeset/adr-0033-package-consolidation.md

@@ -0,0 +1,34 @@
+---
+"@objectstack/objectql": minor
+"@objectstack/runtime": patch
+"@objectstack/service-package": patch
+"@objectstack/service-ai": patch
+---
+
+feat(packages): consolidate the package subsystem so AI-built app packages surface in Studio
+
+The package subsystem was split across two stores that never met: the in-memory
+`SchemaRegistry` (what the dispatcher's `/api/v1/packages` list/detail and
+`getMetaItems({type:'package'})` read — i.e. Studio's package selector) and the durable
+`sys_packages` table (where the AI's auto app package, and any `package`-service publish,
+were written). Nothing reconciled the two, so an AI-created `app.<name>` package never
+appeared in Studio.
+
+This unifies them around one write primitive and one read source:
+
+- **`protocol.installPackage`** is now implemented (it was declared-but-missing). It is the
+  single canonical write path: it registers the package in the in-memory registry **and**
+  best-effort persists it to `sys_packages` via the `package` service. Non-fatal when no
+  `package` service is wired (registry write still succeeds).
+- **Dispatcher `POST /api/v1/packages`** routes through `protocol.installPackage` (falling
+  back to the bare registry write when the protocol is unavailable), so HTTP installs are
+  durable too.
+- **`@objectstack/service-package`** reconciles `sys_packages` back into the registry on
+  boot, without clobbering filesystem-registered packages — so persisted packages survive a
+  restart and stay visible in the registry-backed read paths.
+- **`@objectstack/service-ai`** `apply_blueprint` now homes an app via
+  `protocol.installPackage` (falling back to the legacy `package`-service publish), so the
+  app package lands where Studio reads it.
+
+Still the *legacy* `package_id` plane — sealed `sys_package_version` versioning and
+cross-environment promotion remain ADR-0027 follow-ups.

packages/objectql/src/protocol-install-package.test.ts

@@ -0,0 +1,94 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect, vi } from 'vitest';
+import { ObjectStackProtocolImplementation } from './protocol.js';
+
+/**
+ * ADR-0033 package-subsystem consolidation — `protocol.installPackage` is the
+ * single canonical write primitive. It must land a package in BOTH the
+ * in-memory registry (what the dispatcher's `/api/v1/packages` and
+ * `getMetaItems({type:'package'})` read → Studio's selector) AND the durable
+ * `sys_packages` table via the optional `package` service.
+ */
+function makeProtocol(opts: { pkgSvc?: unknown } = {}) {
+  const installed: Array<{ manifest: { id: string } }> = [];
+  const registry = {
+    installPackage: vi.fn((manifest: { id: string }) => {
+      const pkg = { manifest, status: 'installed', enabled: true };
+      installed.push(pkg);
+      return pkg;
+    }),
+    getPackage: vi.fn((id: string) => installed.find((p) => p.manifest.id === id)),
+  };
+  const engine = { registry } as never;
+  const services = new Map<string, unknown>();
+  if (opts.pkgSvc) services.set('package', opts.pkgSvc);
+  const protocol = new ObjectStackProtocolImplementation(engine, () => services);
+  return { protocol, registry };
+}
+
+describe('protocol.installPackage (ADR-0033 consolidation)', () => {
+  it('writes the in-memory registry AND persists via the package service', async () => {
+    const publish = vi.fn(async () => ({ success: true }));
+    const { protocol, registry } = makeProtocol({ pkgSvc: { publish } });
+    const manifest = { id: 'app.demo', name: 'Demo', version: '1.0.0', type: 'application' };
+
+    const res = (await protocol.installPackage({ manifest } as never)) as {
+      package: { manifest: { id: string } };
+      message: string;
+    };
+
+    expect(registry.installPackage).toHaveBeenCalledWith(manifest, undefined);
+    expect(publish).toHaveBeenCalledTimes(1);
+    expect((publish.mock.calls[0] as unknown[])[0]).toMatchObject({ manifest });
+    expect(res.package.manifest.id).toBe('app.demo');
+    expect(res.message).toContain('app.demo');
+  });
+
+  it('forwards install-time settings to the registry', async () => {
+    const { protocol, registry } = makeProtocol();
+    const manifest = { id: 'app.s', name: 'S', version: '1.0.0', type: 'application' };
+    await protocol.installPackage({ manifest, settings: { theme: 'dark' } } as never);
+    expect(registry.installPackage).toHaveBeenCalledWith(manifest, { theme: 'dark' });
+  });
+
+  it('still registers in-memory when no package service is present (non-fatal)', async () => {
+    const { protocol, registry } = makeProtocol();
+    const manifest = { id: 'app.nopkg', name: 'NoPkg', version: '1.0.0', type: 'application' };
+
+    const res = (await protocol.installPackage({ manifest } as never)) as {
+      package: { manifest: { id: string } };
+    };
+
+    expect(registry.installPackage).toHaveBeenCalledOnce();
+    expect(res.package.manifest.id).toBe('app.nopkg');
+  });
+
+  it('does not throw and keeps the registry write when persistence rejects', async () => {
+    const publish = vi.fn(async () => {
+      throw new Error('db down');
+    });
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const { protocol, registry } = makeProtocol({ pkgSvc: { publish } });
+    const manifest = { id: 'app.err', name: 'Err', version: '1.0.0', type: 'application' };
+
+    const res = (await protocol.installPackage({ manifest } as never)) as {
+      package: { manifest: { id: string } };
+    };
+
+    expect(registry.installPackage).toHaveBeenCalledOnce();
+    expect(res.package.manifest.id).toBe('app.err');
+    warn.mockRestore();
+  });
+
+  it('skips persistence when the manifest has no version (cannot key sys_packages)', async () => {
+    const publish = vi.fn(async () => ({ success: true }));
+    const { protocol, registry } = makeProtocol({ pkgSvc: { publish } });
+    const manifest = { id: 'app.nov', name: 'NoVer', type: 'application' };
+
+    await protocol.installPackage({ manifest } as never);
+
+    expect(registry.installPackage).toHaveBeenCalledOnce();
+    expect(publish).not.toHaveBeenCalled();
+  });
+});

packages/objectql/src/protocol.ts

@@ -10,7 +10,9 @@ import type {
     BatchUpdateRequest,
     BatchUpdateResponse,
     UpdateManyDataRequest,
-    DeleteManyDataRequest
+    DeleteManyDataRequest,
+    InstallPackageRequest,
+    InstallPackageResponse
 } from '@objectstack/spec/api';
 import type { MetadataCacheRequest, MetadataCacheResponse, ServiceInfo, ApiRoutes, WellKnownCapabilities } from '@objectstack/spec/api';
 import type { IFeedService } from '@objectstack/spec/contracts';
@@ -4380,4 +4382,44 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
         const unsubscribed = await svc.unsubscribe(request.object, request.recordId, 'current_user');
         return { success: true, data: { object: request.object, recordId: request.recordId, unsubscribed } };
     }
+
+    /**
+     * Install a package from a manifest — the single canonical write primitive
+     * for the package subsystem (ADR-0033 consolidation).
+     *
+     * It writes BOTH stores that the runtime keeps for packages, so a package
+     * surfaces consistently no matter which read path is used:
+     *   1. the in-memory `SchemaRegistry` (what the dispatcher's
+     *      `/api/v1/packages` list/detail and `getMetaItems({type:'package'})`
+     *      read — i.e. what Studio's package selector shows), and
+     *   2. the durable `sys_packages` table via the optional `package` service
+     *      (so the package survives a restart; that service re-hydrates these
+     *      rows back into the registry on boot).
+     *
+     * The DB write is best-effort and non-fatal: when the `package` service is
+     * absent (e.g. the `marketplace` capability is off) the package is still
+     * registered in-memory and visible for the lifetime of the process.
+     */
+    async installPackage(request: InstallPackageRequest): Promise<InstallPackageResponse> {
+        const manifest = request.manifest;
+        const pkg = this.engine.registry.installPackage(manifest as any, request.settings);
+
+        // Best-effort durable persistence to `sys_packages`.
+        try {
+            const services = this.getServicesRegistry?.();
+            const pkgSvc = services?.get('package') as
+                | { publish?: (data: { manifest: unknown; metadata: unknown }) => Promise<unknown> }
+                | undefined;
+            if (pkgSvc?.publish && (manifest as any)?.version) {
+                await pkgSvc.publish({ manifest, metadata: {} });
+            }
+        } catch (e) {
+            // Non-fatal: registry write already succeeded; log and continue.
+            console.warn(
+                `[protocol.installPackage] sys_packages persist skipped for '${(manifest as any)?.id}': ${(e as Error)?.message}`,
+            );
+        }
+
+        return { package: pkg as any, message: `Installed package: ${(manifest as any)?.id}` };
+    }
 }

packages/runtime/src/http-dispatcher.test.ts

@@ -954,6 +954,54 @@ describe('HttpDispatcher', () => {
         });
     });
 
+    // ═══════════════════════════════════════════════════════════════
+    // Package install — POST /packages routes through protocol.installPackage
+    // (ADR-0033 consolidation: registry + sys_packages in one primitive)
+    // ═══════════════════════════════════════════════════════════════
+
+    describe('POST /packages install', () => {
+        it('routes through protocol.installPackage and returns the unwrapped package', async () => {
+            const installPackage = vi.fn().mockResolvedValue({
+                package: { manifest: { id: 'app.demo' }, status: 'installed' },
+                message: 'Installed package: app.demo',
+            });
+            const mockRegistry = { installPackage: vi.fn(), getAllPackages: vi.fn().mockReturnValue([]) };
+            (kernel as any).getService = vi.fn().mockImplementation((name: string) => {
+                if (name === 'protocol') return Promise.resolve({ installPackage });
+                if (name === 'objectql') return Promise.resolve({ registry: mockRegistry });
+                return null;
+            });
+
+            const manifest = { id: 'app.demo', name: 'Demo', version: '1.0.0', type: 'application' };
+            const result = await dispatcher.handlePackages('', 'POST', { manifest, settings: { a: 1 } }, {}, { request: {} });
+
+            expect(result.handled).toBe(true);
+            expect(result.response?.status).toBe(201);
+            expect(installPackage).toHaveBeenCalledWith({ manifest, settings: { a: 1 } });
+            expect(mockRegistry.installPackage).not.toHaveBeenCalled(); // primitive owns the write
+            expect((result.response as any)?.body?.data?.manifest?.id).toBe('app.demo');
+        });
+
+        it('falls back to registry.installPackage when the protocol lacks the method', async () => {
+            const mockRegistry = {
+                installPackage: vi.fn().mockReturnValue({ manifest: { id: 'app.fb' }, status: 'installed' }),
+                getAllPackages: vi.fn().mockReturnValue([]),
+            };
+            (kernel as any).getService = vi.fn().mockImplementation((name: string) => {
+                if (name === 'protocol') return Promise.resolve({}); // no installPackage
+                if (name === 'objectql') return Promise.resolve({ registry: mockRegistry });
+                return null;
+            });
+
+            const manifest = { id: 'app.fb', name: 'FB', version: '1.0.0', type: 'application' };
+            const result = await dispatcher.handlePackages('', 'POST', { manifest }, {}, { request: {} });
+
+            expect(result.response?.status).toBe(201);
+            expect(mockRegistry.installPackage).toHaveBeenCalledWith(manifest, undefined);
+            expect((result.response as any)?.body?.data?.manifest?.id).toBe('app.fb');
+        });
+    });
+
     // ═══════════════════════════════════════════════════════════════
     // Metadata getPublished Endpoint
     // ═══════════════════════════════════════════════════════════════

packages/runtime/src/http-dispatcher.ts

@@ -1383,9 +1383,21 @@ export class HttpDispatcher {
                 return { handled: true, response: this.success({ packages, total: packages.length }) };
             }
 
-            // POST /packages → install package
+            // POST /packages → install package.
+            // Route through the canonical `protocol.installPackage` primitive so
+            // the install lands in BOTH the in-memory registry (what this list/detail
+            // reads) AND the durable `sys_packages` table. Fall back to the bare
+            // registry write only when the protocol service/method is unavailable.
             if (parts.length === 0 && m === 'POST') {
-                const pkg = registry.installPackage(body.manifest || body, body.settings);
+                const manifest = body.manifest || body;
+                let pkg: any;
+                const protocolSvc: any = await this.resolveService('protocol').catch(() => null);
+                if (protocolSvc && typeof protocolSvc.installPackage === 'function') {
+                    const out = await protocolSvc.installPackage({ manifest, settings: body.settings });
+                    pkg = out?.package ?? out;
+                } else {
+                    pkg = registry.installPackage(manifest, body.settings);
+                }
                 const res = this.success(pkg);
                 res.status = 201;
                 return { handled: true, response: res };

packages/services/service-ai/src/__tests__/blueprint-tools.test.ts

@@ -462,3 +462,73 @@ describe('apply_blueprint — auto app package', () => {
     for (const c of proto.saveMetaItem.mock.calls) expect(c[0].packageId).toBeUndefined();
   });
 });
+
+// ═══════════════════════════════════════════════════════════════════
+// ADR-0033 consolidation — when the protocol exposes the canonical
+// `installPackage` primitive, the app package is written through it (registry
+// + sys_packages) instead of the legacy package-service publish, so the app
+// actually surfaces in Studio.
+// ═══════════════════════════════════════════════════════════════════
+
+function createMockProtocolWithInstall(existingObjects: string[] = []) {
+  const base = createMockProtocol(existingObjects);
+  const installPackage = vi.fn(async (req: any) => ({ package: { manifest: req.manifest }, message: 'ok' }));
+  (base.protocol as any).installPackage = installPackage;
+  return { ...base, installPackage };
+}
+
+describe('apply_blueprint — app package via protocol.installPackage', () => {
+  it('prefers protocol.installPackage over the legacy publish and binds artifacts', async () => {
+    const registry = new ToolRegistry();
+    const proto = createMockProtocolWithInstall();
+    const pkg = createMockPackageService();
+    registerBlueprintTools(registry, {
+      ai: createMockAi().ai, protocol: proto.protocol,
+      metadataService: createMockMetadataService(), packageService: pkg.svc as any,
+    });
+
+    const parsed = parse(await registry.execute(call('apply_blueprint', { blueprint: APP_BLUEPRINT })));
+
+    expect(proto.installPackage).toHaveBeenCalledOnce();
+    expect((proto.installPackage.mock.calls[0] as any[])[0].manifest).toMatchObject({
+      id: 'app.project_management', type: 'application', namespace: 'project_management', scope: 'environment',
+    });
+    expect(pkg.publish).not.toHaveBeenCalled(); // canonical primitive wins; legacy publish skipped
+    expect(parsed.package).toEqual({ id: 'app.project_management', name: '项目管理', created: true });
+    expect(proto.saveMetaItem.mock.calls.length).toBe(3); // object + view + app
+    for (const c of proto.saveMetaItem.mock.calls) {
+      expect(c[0].packageId).toBe('app.project_management');
+      expect(c[0].mode).toBe('draft');
+    }
+  });
+
+  it('installs via protocol.installPackage even with no package service wired', async () => {
+    const registry = new ToolRegistry();
+    const proto = createMockProtocolWithInstall();
+    registerBlueprintTools(registry, {
+      ai: createMockAi().ai, protocol: proto.protocol, metadataService: createMockMetadataService(),
+    });
+
+    const parsed = parse(await registry.execute(call('apply_blueprint', { blueprint: APP_BLUEPRINT })));
+
+    expect(proto.installPackage).toHaveBeenCalledOnce();
+    expect(parsed.package).toEqual({ id: 'app.project_management', name: '项目管理', created: true });
+    for (const c of proto.saveMetaItem.mock.calls) expect(c[0].packageId).toBe('app.project_management');
+  });
+
+  it('reuses an existing package (get-guard) without calling installPackage', async () => {
+    const registry = new ToolRegistry();
+    const proto = createMockProtocolWithInstall();
+    const pkg = createMockPackageService(['app.project_management']);
+    registerBlueprintTools(registry, {
+      ai: createMockAi().ai, protocol: proto.protocol,
+      metadataService: createMockMetadataService(), packageService: pkg.svc as any,
+    });
+
+    const parsed = parse(await registry.execute(call('apply_blueprint', { blueprint: APP_BLUEPRINT })));
+
+    expect(proto.installPackage).not.toHaveBeenCalled();
+    expect(pkg.publish).not.toHaveBeenCalled();
+    expect(parsed.package).toEqual({ id: 'app.project_management', name: 'app.project_management', created: false });
+  });
+});