Serve @cardstack/base modules from the host bundle

packages/host/app/lib/bundled-base.ts

@@ -0,0 +1,60 @@
+import type { VirtualNetwork } from '@cardstack/runtime-common';
+
+// Base modules compiled into the host bundle, keyed by their path under
+// `@cardstack/base/`. Each is served to the loader in place of a fetch of the
+// module from the base realm; the literal `import()` per entry is what lets
+// Vite give each module its own chunk.
+//
+// Registered from a table, as the host tools are, rather than as literal
+// `shimAsyncModule` calls in externals.ts: the boxel-cli guard that reads
+// literal shim ids out of that file covers `@cardstack/base/*` through its
+// path alias already, so nothing is lost to it here.
+export const BUNDLED_BASE_MODULES: Record<
+  string,
+  () => Promise<Record<string, unknown>>
+> = {
+  'date/day': () => import('@cardstack/base/date/day'),
+  'date/month': () => import('@cardstack/base/date/month'),
+  'date/month-day': () => import('@cardstack/base/date/month-day'),
+  'date/month-year': () => import('@cardstack/base/date/month-year'),
+  'date/year': () => import('@cardstack/base/date/year'),
+  'date/week': () => import('@cardstack/base/date/week'),
+  'date/quarter': () => import('@cardstack/base/date/quarter'),
+  time: () => import('@cardstack/base/time'),
+  'time/time-range': () => import('@cardstack/base/time/time-range'),
+  'time/duration': () => import('@cardstack/base/time/duration'),
+  'time/relative-time': () => import('@cardstack/base/time/relative-time'),
+  // string.ts is `export default StringField` from card-api, so it is
+  // resolved there. Importing string.ts itself would have TypeScript classify
+  // that `.ts` module as CommonJS (this package declares no `type`) and
+  // retype its default export as a namespace for every host importer.
+  string: () =>
+    import('@cardstack/base/card-api').then(({ StringField }) => ({
+      default: StringField,
+    })),
+  number: () => import('@cardstack/base/number'),
+  boolean: () => import('@cardstack/base/boolean'),
+  'big-integer': () => import('@cardstack/base/big-integer'),
+  email: () => import('@cardstack/base/email'),
+  'ethereum-address': () => import('@cardstack/base/ethereum-address'),
+  'phone-number': () => import('@cardstack/base/phone-number'),
+  'text-area': () => import('@cardstack/base/text-area'),
+  markdown: () => import('@cardstack/base/markdown'),
+  'rich-markdown': () => import('@cardstack/base/rich-markdown'),
+  color: () => import('@cardstack/base/color'),
+  'code-ref': () => import('@cardstack/base/code-ref'),
+  realm: () => import('@cardstack/base/realm'),
+  enum: () => import('@cardstack/base/enum'),
+  searchable: () => import('@cardstack/base/searchable'),
+  'base64-image': () => import('@cardstack/base/base64-image'),
+};
+
+// Registers on the virtual network, so every loader that shares it serves the
+// bundled modules. Must run after the `@cardstack/base/` realm mapping is
+// registered: a shim id resolves at registration time, and it has to land on
+// the same realm URL a loader import of the id resolves to.
+export function shimBundledBase(virtualNetwork: VirtualNetwork) {
+  for (let [name, resolve] of Object.entries(BUNDLED_BASE_MODULES)) {
+    virtualNetwork.shimAsyncModule({ id: `@cardstack/base/${name}`, resolve });
+  }
+}

packages/host/app/services/network.ts

@@ -13,6 +13,7 @@ import {
 
 import config from '@cardstack/host/config/environment';
 
+import { shimBundledBase } from '../lib/bundled-base';
 import { shimExternals } from '../lib/externals';
 import { authErrorEventMiddleware } from '../utils/auth-error-guard';
 import { scheduleNativeTimeout } from '../utils/render-timer-stub';
@@ -97,6 +98,7 @@ export default class NetworkService extends Service {
       virtualNetwork.addRealmMapping(prefix, resolvedRealmURL.href);
     }
     shimExternals(virtualNetwork);
+    shimBundledBase(virtualNetwork);
     virtualNetwork.addImportMap('@cardstack/boxel-icons/', (rest) => {
       return `${config.iconsURL}/@cardstack/boxel-icons/v1/icons/${rest}.js`;
     });

packages/host/tests/integration/bundled-base-modules-test.ts

@@ -0,0 +1,46 @@
+import { getService } from '@universal-ember/test-support';
+
+import { module, test } from 'qunit';
+
+import { BUNDLED_BASE_MODULES } from '@cardstack/host/lib/bundled-base';
+
+import { setupRenderingTest } from '../helpers/setup';
+
+module('Integration | bundled base modules', function (hooks) {
+  setupRenderingTest(hooks);
+
+  test('the loader serves every bundled base module from the host bundle, not the base realm', async function (assert) {
+    let network = getService('network');
+    let loader = getService('loader-service').loader;
+    let baseRealmRequests: string[] = [];
+    let spy = async (request: Request) => {
+      if (request.url.includes('/base/')) {
+        baseRealmRequests.push(request.url);
+      }
+      return null;
+    };
+    network.virtualNetwork.mount(spy, { prepend: true });
+    try {
+      for (let [name, resolveBundled] of Object.entries(BUNDLED_BASE_MODULES)) {
+        let bundled = await resolveBundled();
+        let served = await loader.import<Record<string, unknown>>(
+          `@cardstack/base/${name}`,
+        );
+        for (let key of Object.keys(bundled)) {
+          assert.strictEqual(
+            served[key],
+            bundled[key],
+            `${name}: the loader's ${key} export is the bundled one`,
+          );
+        }
+      }
+      assert.deepEqual(
+        baseRealmRequests,
+        [],
+        'no bundled module was fetched from the base realm',
+      );
+    } finally {
+      network.virtualNetwork.unmount(spy);
+    }
+  });
+});

packages/host/tests/unit/loader-test.ts

@@ -4,7 +4,7 @@ import { getService } from '@universal-ember/test-support';
 
 import { module, test } from 'qunit';
 
-import { baseRealm, Loader } from '@cardstack/runtime-common';
+import { baseRealm, Loader, VirtualNetwork } from '@cardstack/runtime-common';
 
 import {
   testRealmURL,
@@ -396,6 +396,47 @@ module('Unit | loader', function (hooks) {
     );
   });
 
+  test('a module shimmed on the virtual network is served to the loader without a fetch, under every spelling', async function (assert) {
+    // A shim for a realm-mapped identifier is keyed by the realm URL the
+    // identifier resolves to, and a loader import resolves to that same URL.
+    // The network's fetch pipeline only answers shims on the fake packages
+    // origin, so the loader has to ask the network for the shim itself.
+    let virtualNetwork = new VirtualNetwork();
+    let realmURL = 'https://shimmed-realm.example/';
+    let aliasURL = 'https://shimmed-alias.example/';
+    virtualNetwork.addURLMapping(new URL(aliasURL), new URL(realmURL));
+    virtualNetwork.addRealmMapping('@test-loader-shim/', realmURL);
+    class Shimmed {}
+    virtualNetwork.shimAsyncModule({
+      id: '@test-loader-shim/shimmed-module',
+      resolve: async () => ({ default: Shimmed }),
+    });
+    let throwIfFetch = new Loader(
+      async () => {
+        throw new Error(
+          'fetch should not be invoked for a module the virtual network shims',
+        );
+      },
+      virtualNetwork.resolveImport,
+      { virtualNetwork },
+    );
+
+    for (let spelling of [
+      '@test-loader-shim/shimmed-module',
+      `${realmURL}shimmed-module`,
+      `${aliasURL}shimmed-module`,
+    ]) {
+      let module = await throwIfFetch.import<{ default: typeof Shimmed }>(
+        spelling,
+      );
+      assert.strictEqual(
+        module.default,
+        Shimmed,
+        `${spelling} is served from the shim`,
+      );
+    }
+  });
+
   test('identify preserves original module for reexports', function (assert) {
     let throwIfFetch = new Loader(async () => {
       throw new Error(

packages/host/vite.config.mjs

@@ -298,6 +298,16 @@ export default defineConfig(({ mode }) => ({
   },
   resolve: {
     alias: [
+      // Base-realm modules served from the host bundle (see shimBundledBase in
+      // app/lib/bundled-base.ts) import host tools as
+      // `@cardstack/boxel-host/tools/*` or `@cardstack/boxel-host/commands/*`.
+      // At runtime the virtual network shims those specifiers to app/tools
+      // modules (see app/tools/index.ts); this alias gives the bundler the
+      // same 1:1 mapping.
+      {
+        find: /^@cardstack\/boxel-host\/(?:tools|commands)\//,
+        replacement: `${__dirname}/app/tools/`,
+      },
       { find: 'path', replacement: require.resolve('path-browserify') },
       { find: 'stream', replacement: require.resolve('stream-browserify') },
       { find: /^util$/, replacement: require.resolve('util/') },

packages/runtime-common/loader.ts

@@ -1122,16 +1122,21 @@ export class Loader {
     init?: RequestInit,
   ): Promise<MaybeCachedResponse> => {
     try {
-      let shimmedModule = this.moduleShims.get(
-        this.asRequest(urlOrRequest, init).url,
-      );
+      let request = this.asRequest(urlOrRequest, init);
+      // A module shimmed on the virtual network is keyed by the URL its
+      // identifier resolves to, and the network's fetch pipeline only answers
+      // shims on the fake packages origin. This is the one path that knows a
+      // request is for a module (not a card instance that may live at the same
+      // realm URL), so the lookup belongs here, ahead of any fetch.
+      let shimmedModule =
+        this.moduleShims.get(request.url) ??
+        (await this.virtualNetwork?.getShimmedModule(request.url));
       if (shimmedModule) {
         let response = new Response();
         (response as any)[Symbol.for('shimmed-module')] = shimmedModule;
         return response;
       }
 
-      let request = this.asRequest(urlOrRequest, init);
       return await cachedFetch(this.fetchImplementation, request);
     } catch (err: any) {
       let url =

packages/runtime-common/package-shim-handler.ts

@@ -519,6 +519,21 @@ export class PackageShimHandler {
     }
   }
 
+  // Module lookup for the Loader's module-fetch path. That path sees the URL
+  // an identifier resolves to — for a realm-mapped prefix such as
+  // `@cardstack/base/`, the realm URL — which is also the key a shim for such
+  // an identifier is registered under. `handle` only answers on the fake
+  // packages origin because, in the general fetch pipeline, a realm URL may
+  // name a card instance as well as a module; the Loader knows it is asking
+  // for a module, so it may be served a shim registered under any URL.
+  async lookupModule(url: string): Promise<ModuleLike | undefined> {
+    let module =
+      (await this.getModule(url)) ?? (await this.getModuleByPrefix(url));
+    return module
+      ? wrapWithStrictNamespace(url, module, this.findExportSources)
+      : undefined;
+  }
+
   private async getModule(url: string): Promise<ModuleLike | undefined> {
     let key = trimModuleIdentifier(url);
     let resolver = this.moduleIds.get(key);

packages/runtime-common/virtual-network.ts

@@ -196,6 +196,15 @@ export class VirtualNetwork {
     this.packageShimHandler.shimAsyncModule(descriptor);
   }
 
+  // Lets a Loader serve a module shimmed on this network from its module-fetch
+  // path, whatever URL the shim is registered under. The lookup folds every
+  // spelling of the identifier (realm-prefix form, virtual alias, url-mapped
+  // alias) onto the real URL, which is the form shims for realm-mapped
+  // identifiers are keyed by, so all spellings converge on one module.
+  getShimmedModule(url: string): Promise<ModuleLike | undefined> {
+    return this.packageShimHandler.lookupModule(this.toRealURLHref(url));
+  }
+
   addURLMapping(from: URL, to: URL) {
     this.urlMappings.push([from.href, to.href]);
     // unresolveURL and toRealURLHref chase through urlMappings (the latter via