diff --git a/layers/src/layer-publisher-stack.ts b/layers/src/layer-publisher-stack.ts index 7e3fe9016b..1e0cf0b175 100644 --- a/layers/src/layer-publisher-stack.ts +++ b/layers/src/layer-publisher-stack.ts @@ -1,6 +1,6 @@ import { execFileSync, execSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; -import { readdirSync } from 'node:fs'; +import { readdirSync, readFileSync } from 'node:fs'; import { dirname, join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { CfnOutput, RemovalPolicy, Stack, type StackProps } from 'aws-cdk-lib'; @@ -60,6 +60,50 @@ const execFileWithRetry = ( } }; +interface LayerUtility { + /** Workspace directory under `packages/`, used with `npm -w packages/`. */ + readonly workspace: string; + /** Published npm package name, e.g. `@aws-lambda-powertools/logger`. */ + readonly packageName: string; +} + +/** + * Discovers the packages to bundle in the layer by scanning the `packages/` + * workspace and keeping every non-private `@aws-lambda-powertools/*` package. + * + * Deriving the list means a newly added utility is bundled automatically and + * the list cannot silently drift out of sync with the workspace (#5576). + */ +export const getLayerUtilities = (packagesDir: string): LayerUtility[] => + readdirSync(packagesDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .flatMap((entry) => { + let manifest: { name?: string; private?: boolean }; + try { + manifest = JSON.parse( + readFileSync(join(packagesDir, entry.name, 'package.json'), 'utf-8') + ); + } catch { + return []; + } + if ( + manifest.private === true || + typeof manifest.name !== 'string' || + !manifest.name.startsWith('@aws-lambda-powertools/') + ) { + return []; + } + return [{ workspace: entry.name, packageName: manifest.name }]; + }) + .sort((a, b) => a.workspace.localeCompare(b.workspace)); + +/** + * Prefix of the tarball produced by `npm pack` for a scoped package, e.g. + * `@aws-lambda-powertools/logger` -> `aws-lambda-powertools-logger-`. + */ +export const packTarballPrefix = (packageName: string): string => + `${packageName.replace(/^@/, '').replace(/\//g, '-')}-`; + export interface LayerPublisherStackProps extends StackProps { readonly layerName?: string; readonly powertoolsPackageVersion?: string; @@ -102,20 +146,17 @@ export class LayerPublisherStack extends Stack { // This folder is the project root, relative to the current file const projectRoot = resolve(__dirname, '..', '..'); - // This is the list of packages that we need include in the Lambda Layer - // the name is the same as the npm workspace name - const utilities = [ - 'commons', - 'event-handler', - 'jmespath', - 'logger', - 'metrics', - 'tracer', - 'parameters', - 'idempotency', - 'batch', - 'parser', - ]; + // The packages to bundle in the Lambda Layer are derived from the + // workspace: every non-private @aws-lambda-powertools/* package + // under packages/. New utilities are bundled automatically. + const utilities = getLayerUtilities( + join(projectRoot, 'packages') + ); + if (utilities.length === 0) { + throw new Error( + `No publishable utilities found under ${join(projectRoot, 'packages')}` + ); + } // These files are relative to the tmp folder const filesToRemove = [ @@ -169,24 +210,23 @@ export class LayerPublisherStack extends Stack { for (const util of utilities) { buildCommands.push( // Build latest version of the package - `npm run build -w packages/${util}`, + `npm run build -w packages/${util.workspace}`, // Pack the package to a .tgz file - `npm pack -w packages/${util}`, + `npm pack -w packages/${util.workspace}`, // Move the .tgz file to the tmp folder - `mv aws-lambda-powertools-${util}-*.tgz ${tmpBuildDir}` + `mv ${packTarballPrefix(util.packageName)}*.tgz ${tmpBuildDir}` ); } filesToRemove.push( ...utilities.map((util) => - join(`aws-lambda-powertools-${util}-*.tgz`) + join(`${packTarballPrefix(util.packageName)}*.tgz`) ) ); } else { // Dependencies to install in the Lambda Layer modulesToInstall.push( ...utilities.map( - (util) => - `@aws-lambda-powertools/${util}@${powertoolsPackageVersion}` + (util) => `${util.packageName}@${powertoolsPackageVersion}` ) ); } @@ -212,13 +252,13 @@ export class LayerPublisherStack extends Stack { // than relying on shell glob expansion. if (buildFromLocal) { for (const util of utilities) { - const prefix = `aws-lambda-powertools-${util}-`; + const prefix = packTarballPrefix(util.packageName); const tarball = readdirSync(tmpBuildDir).find( (name) => name.startsWith(prefix) && name.endsWith('.tgz') ); if (tarball === undefined) { throw new Error( - `Could not find packed tarball for ${util} in ${tmpBuildDir}` + `Could not find packed tarball for ${util.packageName} in ${tmpBuildDir}` ); } modulesToInstall.push(join(tmpBuildDir, tarball)); diff --git a/layers/tests/e2e/layerPublisher.class.test.functionCode.ts b/layers/tests/e2e/layerPublisher.class.test.functionCode.ts index 8b68e56a75..736c3101ac 100644 --- a/layers/tests/e2e/layerPublisher.class.test.functionCode.ts +++ b/layers/tests/e2e/layerPublisher.class.test.functionCode.ts @@ -1,15 +1,19 @@ -import { readFile } from 'node:fs/promises'; +import { readdir, readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { BatchProcessor, EventType } from '@aws-lambda-powertools/batch'; +import { DataMasking } from '@aws-lambda-powertools/data-masking'; import { Router } from '@aws-lambda-powertools/event-handler/http'; import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import { kafkaConsumer, SchemaType } from '@aws-lambda-powertools/kafka'; import { Logger } from '@aws-lambda-powertools/logger'; import { Metrics } from '@aws-lambda-powertools/metrics'; import { AppConfigProvider } from '@aws-lambda-powertools/parameters/appconfig'; import { DynamoDBProvider } from '@aws-lambda-powertools/parameters/dynamodb'; import { SecretsProvider } from '@aws-lambda-powertools/parameters/secrets'; import { SSMProvider } from '@aws-lambda-powertools/parameters/ssm'; +import { SigV4Signer } from '@aws-lambda-powertools/signer/sigv4'; import { Tracer } from '@aws-lambda-powertools/tracer'; +import { validate } from '@aws-lambda-powertools/validation'; import { AppConfigDataClient } from '@aws-sdk/client-appconfigdata'; import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager'; @@ -85,18 +89,17 @@ const getVersionFromModule = async (moduleName: string): Promise => { }; export const handler = async (_event: unknown, context: Context) => { - // Check that the packages version matches the expected one - for (const moduleName of [ - 'commons', - 'logger', - 'metrics', - 'tracer', - 'parameters', - 'idempotency', - 'batch', - 'parser', - 'event-handler', - ]) { + // Check that every Powertools package bundled in the layer matches the + // expected version. Reading the layer directory keeps this verification in + // sync automatically as utilities are added to the layer. + const scopeDir = join(layerPath, '@aws-lambda-powertools'); + const bundledModules = (await readdir(scopeDir)).filter( + (name) => !name.startsWith('.') + ); + if (bundledModules.length === 0) { + throw new Error(`No Powertools packages found in the layer at ${scopeDir}`); + } + for (const moduleName of bundledModules) { const moduleVersion = await getVersionFromModule(moduleName); if (moduleVersion !== expectedVersion) { throw new Error( @@ -133,4 +136,27 @@ export const handler = async (_event: unknown, context: Context) => { throw error; } } + + // Exercise the remaining utilities to prove they load from the layer at + // runtime (both CJS and ESM). These don't emit logs; a missing package or a + // broken import would surface as an error and fail the invocation. + const masker = new DataMasking(); + masker.erase({ ssn: '123-45-6789' }, { fields: ['ssn'] }); + + validate({ payload: { foo: 'bar' }, schema: { type: 'object' } }); + + const kafkaHandler = kafkaConsumer(async () => {}, { + value: { type: SchemaType.JSON }, + }); + if (typeof kafkaHandler !== 'function') { + throw new Error('kafkaConsumer did not return a handler'); + } + + const signer = new SigV4Signer({ + service: 'execute-api', + region: 'eu-west-1', + }); + if (typeof signer.sign !== 'function') { + throw new Error('SigV4Signer.sign is not available'); + } }; diff --git a/layers/tests/unit/getLayerUtilities.test.ts b/layers/tests/unit/getLayerUtilities.test.ts new file mode 100644 index 0000000000..f503f9b7f2 --- /dev/null +++ b/layers/tests/unit/getLayerUtilities.test.ts @@ -0,0 +1,101 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { + getLayerUtilities, + packTarballPrefix, +} from '../../src/layer-publisher-stack.js'; + +describe('getLayerUtilities', () => { + let fixtureDir: string; + + const writePackage = ( + dir: string, + manifest: Record | string + ) => { + const pkgDir = join(fixtureDir, dir); + mkdirSync(pkgDir, { recursive: true }); + writeFileSync( + join(pkgDir, 'package.json'), + typeof manifest === 'string' ? manifest : JSON.stringify(manifest) + ); + }; + + beforeAll(() => { + fixtureDir = mkdtempSync(join(tmpdir(), 'layer-utils-')); + + writePackage('logger', { name: '@aws-lambda-powertools/logger' }); + writePackage('data-masking', { + name: '@aws-lambda-powertools/data-masking', + }); + // Private packages (e.g. testing-utils) must be excluded. + writePackage('testing', { + name: '@aws-lambda-powertools/testing-utils', + private: true, + }); + // Packages outside the Powertools scope must be excluded. + writePackage('some-tool', { name: 'some-tool' }); + // Unparseable manifests must be skipped, not throw. + writePackage('broken', '{ not valid json'); + // Loose files that aren't directories must be ignored. + writeFileSync(join(fixtureDir, 'README.md'), '# not a package'); + }); + + afterAll(() => { + rmSync(fixtureDir, { recursive: true, force: true }); + }); + + it('returns only non-private @aws-lambda-powertools packages, sorted', () => { + // Act + const utilities = getLayerUtilities(fixtureDir); + + // Assess + expect(utilities).toEqual([ + { + workspace: 'data-masking', + packageName: '@aws-lambda-powertools/data-masking', + }, + { workspace: 'logger', packageName: '@aws-lambda-powertools/logger' }, + ]); + }); + + it('bundles every publishable utility in the real workspace', () => { + // Prepare + const packagesDir = join(import.meta.dirname, '..', '..', '..', 'packages'); + + // Act + const workspaces = getLayerUtilities(packagesDir).map((u) => u.workspace); + + // Assess + expect(workspaces).toEqual([ + 'batch', + 'commons', + 'data-masking', + 'event-handler', + 'idempotency', + 'jmespath', + 'kafka', + 'logger', + 'metrics', + 'parameters', + 'parser', + 'signer', + 'tracer', + 'validation', + ]); + // The internal testing package is private and must never ship in the layer. + expect(workspaces).not.toContain('testing'); + }); +}); + +describe('packTarballPrefix', () => { + it('derives the npm pack tarball prefix from a scoped package name', () => { + expect(packTarballPrefix('@aws-lambda-powertools/logger')).toBe( + 'aws-lambda-powertools-logger-' + ); + expect(packTarballPrefix('@aws-lambda-powertools/data-masking')).toBe( + 'aws-lambda-powertools-data-masking-' + ); + }); +}); diff --git a/layers/tests/unit/layer-publisher.test.ts b/layers/tests/unit/layer-publisher.test.ts index 554f571829..c6e56237e1 100644 --- a/layers/tests/unit/layer-publisher.test.ts +++ b/layers/tests/unit/layer-publisher.test.ts @@ -1,7 +1,12 @@ +import { readdirSync } from 'node:fs'; +import { join } from 'node:path'; import { App } from 'aws-cdk-lib'; import { Template } from 'aws-cdk-lib/assertions'; -import { describe, it } from 'vitest'; -import { LayerPublisherStack } from '../../src/layer-publisher-stack.js'; +import { describe, expect, it } from 'vitest'; +import { + getLayerUtilities, + LayerPublisherStack, +} from '../../src/layer-publisher-stack.js'; describe('Class: LayerPublisherStack', () => { it('creates the stack with a layer in it', () => { @@ -40,5 +45,30 @@ describe('Class: LayerPublisherStack', () => { Name: '/layers/powertools-layer-arn', Type: 'String', }); + + // Synthesising the stack above runs the local bundling, which installs the + // utilities into the tmp folder. Assert every publishable utility landed in + // the layer so the derived list can't silently drop a package. + const packagesDir = join(import.meta.dirname, '..', '..', '..', 'packages'); + const expectedUtilities = getLayerUtilities(packagesDir).map( + (util) => util.workspace + ); + const bundledScopeDir = join( + import.meta.dirname, + '..', + '..', + 'tmp', + 'nodejs', + 'node_modules', + '@aws-lambda-powertools' + ); + const bundledUtilities = readdirSync(bundledScopeDir).filter( + (name) => !name.startsWith('.') + ); + for (const utility of expectedUtilities) { + expect(bundledUtilities).toContain(utility); + } + // The internal testing package is private and must never ship in the layer. + expect(bundledUtilities).not.toContain('testing-utils'); }, 120000); });