fix(layers): bundle all publishable utilities in the Lambda layer

layers/src/layer-publisher-stack.ts

@@ -1,6 +1,6 @@
 import { execFileSync, execSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
-import { readdirSync } from 'node:fs';
+import { readdirSync, readFileSync } from 'node:fs';
 import { dirname, join, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { CfnOutput, RemovalPolicy, Stack, type StackProps } from 'aws-cdk-lib';
@@ -60,6 +60,50 @@
   }
 };
 
+interface LayerUtility {
+  /** Workspace directory under `packages/`, used with `npm -w packages/<dir>`. */
+  readonly workspace: string;
+  /** Published npm package name, e.g. `@aws-lambda-powertools/logger`. */
+  readonly packageName: string;
+}
+
+/**
+ * Discovers the packages to bundle in the layer by scanning the `packages/`
+ * workspace and keeping every non-private `@aws-lambda-powertools/*` package.
+ *
+ * Deriving the list means a newly added utility is bundled automatically and
+ * the list cannot silently drift out of sync with the workspace (#5576).
+ */
+export const getLayerUtilities = (packagesDir: string): LayerUtility[] =>
+  readdirSync(packagesDir, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .flatMap((entry) => {
+      let manifest: { name?: string; private?: boolean };
+      try {
+        manifest = JSON.parse(
+          readFileSync(join(packagesDir, entry.name, 'package.json'), 'utf-8')
+        );
+      } catch {
+        return [];
+      }
+      if (
+        manifest.private === true ||
+        typeof manifest.name !== 'string' ||
+        !manifest.name.startsWith('@aws-lambda-powertools/')
+      ) {
+        return [];
+      }
+      return [{ workspace: entry.name, packageName: manifest.name }];
+    })
+    .sort((a, b) => a.workspace.localeCompare(b.workspace));
+
+/**
+ * Prefix of the tarball produced by `npm pack` for a scoped package, e.g.
+ * `@aws-lambda-powertools/logger` -> `aws-lambda-powertools-logger-`.
+ */
+export const packTarballPrefix = (packageName: string): string =>
+  `${packageName.replace(/^@/, '').replace(/\//g, '-')}-`;
+
 export interface LayerPublisherStackProps extends StackProps {
   readonly layerName?: string;
   readonly powertoolsPackageVersion?: string;
@@ -102,20 +146,17 @@
               // This folder is the project root, relative to the current file
               const projectRoot = resolve(__dirname, '..', '..');
 
-              // This is the list of packages that we need include in the Lambda Layer
-              // the name is the same as the npm workspace name
-              const utilities = [
-                'commons',
-                'event-handler',
-                'jmespath',
-                'logger',
-                'metrics',
-                'tracer',
-                'parameters',
-                'idempotency',
-                'batch',
-                'parser',
-              ];
+              // The packages to bundle in the Lambda Layer are derived from the
+              // workspace: every non-private @aws-lambda-powertools/* package
+              // under packages/. New utilities are bundled automatically.
+              const utilities = getLayerUtilities(
+                join(projectRoot, 'packages')
+              );
+              if (utilities.length === 0) {
+                throw new Error(
+                  `No publishable utilities found under ${join(projectRoot, 'packages')}`
+                );
+              }
 
               // These files are relative to the tmp folder
               const filesToRemove = [
@@ -162,31 +203,30 @@
               // change across that boundary surfaces as a runtime error.
               //
               // The Node.js runtime already provides a complete, self-consistent
               // AWS SDK v3, so we rely on it instead of exposing a partial one.
               const modulesToInstall: string[] = ['zod'];
 
               if (buildFromLocal) {
                 for (const util of utilities) {
                   buildCommands.push(
                     // Build latest version of the package
-                    `npm run build -w packages/${util}`,
+                    `npm run build -w packages/${util.workspace}`,
                     // Pack the package to a .tgz file
-                    `npm pack -w packages/${util}`,
+                    `npm pack -w packages/${util.workspace}`,
                     // Move the .tgz file to the tmp folder
-                    `mv aws-lambda-powertools-${util}-*.tgz ${tmpBuildDir}`
+                    `mv ${packTarballPrefix(util.packageName)}*.tgz ${tmpBuildDir}`
                   );
                 }
                 filesToRemove.push(
                   ...utilities.map((util) =>
-                    join(`aws-lambda-powertools-${util}-*.tgz`)
+                    join(`${packTarballPrefix(util.packageName)}*.tgz`)
                   )
                 );
               } else {
                 // Dependencies to install in the Lambda Layer
                 modulesToInstall.push(
                   ...utilities.map(
-                    (util) =>
-                      `@aws-lambda-powertools/${util}@${powertoolsPackageVersion}`
+                    (util) => `${util.packageName}@${powertoolsPackageVersion}`
                   )
                 );
               }
@@ -212,13 +252,13 @@
               // than relying on shell glob expansion.
               if (buildFromLocal) {
                 for (const util of utilities) {
-                  const prefix = `aws-lambda-powertools-${util}-`;
+                  const prefix = packTarballPrefix(util.packageName);
                   const tarball = readdirSync(tmpBuildDir).find(
                     (name) => name.startsWith(prefix) && name.endsWith('.tgz')
                   );
                   if (tarball === undefined) {
                     throw new Error(
-                      `Could not find packed tarball for ${util} in ${tmpBuildDir}`
+                      `Could not find packed tarball for ${util.packageName} in ${tmpBuildDir}`
                     );
                   }
                   modulesToInstall.push(join(tmpBuildDir, tarball));

layers/tests/e2e/layerPublisher.class.test.functionCode.ts

@@ -1,15 +1,19 @@
-import { readFile } from 'node:fs/promises';
+import { readdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { BatchProcessor, EventType } from '@aws-lambda-powertools/batch';
+import { DataMasking } from '@aws-lambda-powertools/data-masking';
 import { Router } from '@aws-lambda-powertools/event-handler/http';
 import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb';
+import { kafkaConsumer, SchemaType } from '@aws-lambda-powertools/kafka';
 import { Logger } from '@aws-lambda-powertools/logger';
 import { Metrics } from '@aws-lambda-powertools/metrics';
 import { AppConfigProvider } from '@aws-lambda-powertools/parameters/appconfig';
 import { DynamoDBProvider } from '@aws-lambda-powertools/parameters/dynamodb';
 import { SecretsProvider } from '@aws-lambda-powertools/parameters/secrets';
 import { SSMProvider } from '@aws-lambda-powertools/parameters/ssm';
+import { SigV4Signer } from '@aws-lambda-powertools/signer/sigv4';
 import { Tracer } from '@aws-lambda-powertools/tracer';
+import { validate } from '@aws-lambda-powertools/validation';
 import { AppConfigDataClient } from '@aws-sdk/client-appconfigdata';
 import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
 import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager';
@@ -85,18 +89,17 @@ const getVersionFromModule = async (moduleName: string): Promise<string> => {
 };
 
 export const handler = async (_event: unknown, context: Context) => {
-  // Check that the packages version matches the expected one
-  for (const moduleName of [
-    'commons',
-    'logger',
-    'metrics',
-    'tracer',
-    'parameters',
-    'idempotency',
-    'batch',
-    'parser',
-    'event-handler',
-  ]) {
+  // Check that every Powertools package bundled in the layer matches the
+  // expected version. Reading the layer directory keeps this verification in
+  // sync automatically as utilities are added to the layer.
+  const scopeDir = join(layerPath, '@aws-lambda-powertools');
+  const bundledModules = (await readdir(scopeDir)).filter(
+    (name) => !name.startsWith('.')
+  );
+  if (bundledModules.length === 0) {
+    throw new Error(`No Powertools packages found in the layer at ${scopeDir}`);
+  }
+  for (const moduleName of bundledModules) {
     const moduleVersion = await getVersionFromModule(moduleName);
     if (moduleVersion !== expectedVersion) {
       throw new Error(
@@ -133,4 +136,27 @@ export const handler = async (_event: unknown, context: Context) => {
       throw error;
     }
   }
+
+  // Exercise the remaining utilities to prove they load from the layer at
+  // runtime (both CJS and ESM). These don't emit logs; a missing package or a
+  // broken import would surface as an error and fail the invocation.
+  const masker = new DataMasking();
+  masker.erase({ ssn: '123-45-6789' }, { fields: ['ssn'] });
+
+  validate({ payload: { foo: 'bar' }, schema: { type: 'object' } });
+
+  const kafkaHandler = kafkaConsumer(async () => {}, {
+    value: { type: SchemaType.JSON },
+  });
+  if (typeof kafkaHandler !== 'function') {
+    throw new Error('kafkaConsumer did not return a handler');
+  }
+
+  const signer = new SigV4Signer({
+    service: 'execute-api',
+    region: 'eu-west-1',
+  });
+  if (typeof signer.sign !== 'function') {
+    throw new Error('SigV4Signer.sign is not available');
+  }
 };

layers/tests/unit/getLayerUtilities.test.ts

@@ -0,0 +1,101 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import {
+  getLayerUtilities,
+  packTarballPrefix,
+} from '../../src/layer-publisher-stack.js';
+
+describe('getLayerUtilities', () => {
+  let fixtureDir: string;
+
+  const writePackage = (
+    dir: string,
+    manifest: Record<string, unknown> | string
+  ) => {
+    const pkgDir = join(fixtureDir, dir);
+    mkdirSync(pkgDir, { recursive: true });
+    writeFileSync(
+      join(pkgDir, 'package.json'),
+      typeof manifest === 'string' ? manifest : JSON.stringify(manifest)
+    );
+  };
+
+  beforeAll(() => {
+    fixtureDir = mkdtempSync(join(tmpdir(), 'layer-utils-'));
+
+    writePackage('logger', { name: '@aws-lambda-powertools/logger' });
+    writePackage('data-masking', {
+      name: '@aws-lambda-powertools/data-masking',
+    });
+    // Private packages (e.g. testing-utils) must be excluded.
+    writePackage('testing', {
+      name: '@aws-lambda-powertools/testing-utils',
+      private: true,
+    });
+    // Packages outside the Powertools scope must be excluded.
+    writePackage('some-tool', { name: 'some-tool' });
+    // Unparseable manifests must be skipped, not throw.
+    writePackage('broken', '{ not valid json');
+    // Loose files that aren't directories must be ignored.
+    writeFileSync(join(fixtureDir, 'README.md'), '# not a package');
+  });
+
+  afterAll(() => {
+    rmSync(fixtureDir, { recursive: true, force: true });
+  });
+
+  it('returns only non-private @aws-lambda-powertools packages, sorted', () => {
+    // Act
+    const utilities = getLayerUtilities(fixtureDir);
+
+    // Assess
+    expect(utilities).toEqual([
+      {
+        workspace: 'data-masking',
+        packageName: '@aws-lambda-powertools/data-masking',
+      },
+      { workspace: 'logger', packageName: '@aws-lambda-powertools/logger' },
+    ]);
+  });
+
+  it('bundles every publishable utility in the real workspace', () => {
+    // Prepare
+    const packagesDir = join(import.meta.dirname, '..', '..', '..', 'packages');
+
+    // Act
+    const workspaces = getLayerUtilities(packagesDir).map((u) => u.workspace);
+
+    // Assess
+    expect(workspaces).toEqual([
+      'batch',
+      'commons',
+      'data-masking',
+      'event-handler',
+      'idempotency',
+      'jmespath',
+      'kafka',
+      'logger',
+      'metrics',
+      'parameters',
+      'parser',
+      'signer',
+      'tracer',
+      'validation',
+    ]);
+    // The internal testing package is private and must never ship in the layer.
+    expect(workspaces).not.toContain('testing');
+  });
+});
+
+describe('packTarballPrefix', () => {
+  it('derives the npm pack tarball prefix from a scoped package name', () => {
+    expect(packTarballPrefix('@aws-lambda-powertools/logger')).toBe(
+      'aws-lambda-powertools-logger-'
+    );
+    expect(packTarballPrefix('@aws-lambda-powertools/data-masking')).toBe(
+      'aws-lambda-powertools-data-masking-'
+    );
+  });
+});

layers/tests/unit/layer-publisher.test.ts

@@ -1,7 +1,12 @@
+import { readdirSync } from 'node:fs';
+import { join } from 'node:path';
 import { App } from 'aws-cdk-lib';
 import { Template } from 'aws-cdk-lib/assertions';
-import { describe, it } from 'vitest';
-import { LayerPublisherStack } from '../../src/layer-publisher-stack.js';
+import { describe, expect, it } from 'vitest';
+import {
+  getLayerUtilities,
+  LayerPublisherStack,
+} from '../../src/layer-publisher-stack.js';
 
 describe('Class: LayerPublisherStack', () => {
   it('creates the stack with a layer in it', () => {
@@ -40,5 +45,30 @@ describe('Class: LayerPublisherStack', () => {
       Name: '/layers/powertools-layer-arn',
       Type: 'String',
     });
+
+    // Synthesising the stack above runs the local bundling, which installs the
+    // utilities into the tmp folder. Assert every publishable utility landed in
+    // the layer so the derived list can't silently drop a package.
+    const packagesDir = join(import.meta.dirname, '..', '..', '..', 'packages');
+    const expectedUtilities = getLayerUtilities(packagesDir).map(
+      (util) => util.workspace
+    );
+    const bundledScopeDir = join(
+      import.meta.dirname,
+      '..',
+      '..',
+      'tmp',
+      'nodejs',
+      'node_modules',
+      '@aws-lambda-powertools'
+    );
+    const bundledUtilities = readdirSync(bundledScopeDir).filter(
+      (name) => !name.startsWith('.')
+    );
+    for (const utility of expectedUtilities) {
+      expect(bundledUtilities).toContain(utility);
+    }
+    // The internal testing package is private and must never ship in the layer.
+    expect(bundledUtilities).not.toContain('testing-utils');
   }, 120000);
 });