From c86373d7283deef18208184f7b60e73502336c75 Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:55:35 -0400 Subject: [PATCH 1/2] test(@angular/build): add performance benchmark suite for i18n inliner Introduces an automated macro performance benchmark suite for the i18n inlining subsystem in @angular/build, executable via the benchmark command. The suite generates synthetic in-memory bundles, source maps, and translation catalogs to evaluate realistic workloads without checking large test fixtures into the repository. Benchmark scenarios cover standard applications with and without source maps, enterprise multilingual applications across 32 locales, monolithic bundles to verify 2D task sharding, and warm persistent cache throughput. Each scenario is executed in an isolated child process, and persistent cache priming is performed out-of-process to avoid cross-scenario memory contamination from the OS memory allocator. The harness collects high-precision timing, peak heap, peak RSS, and memory deltas with explicit garbage collection support. The runner is integrated into devkit-admin with support for baseline comparison, machine-readable JSON output, stale build detection, and automatic rebuilds. --- package.json | 1 + scripts/benchmark.mts | 115 ++++++++ scripts/benchmarks/i18n/fixtures.mts | 165 +++++++++++ scripts/benchmarks/i18n/harness.mts | 155 ++++++++++ scripts/benchmarks/i18n/index.mts | 164 +++++++++++ scripts/benchmarks/i18n/init-env.mts | 24 ++ scripts/benchmarks/i18n/reporters.mts | 159 ++++++++++ scripts/benchmarks/i18n/scenarios.mts | 404 ++++++++++++++++++++++++++ 8 files changed, 1187 insertions(+) create mode 100644 scripts/benchmark.mts create mode 100644 scripts/benchmarks/i18n/fixtures.mts create mode 100644 scripts/benchmarks/i18n/harness.mts create mode 100644 scripts/benchmarks/i18n/index.mts create mode 100644 scripts/benchmarks/i18n/init-env.mts create mode 100644 scripts/benchmarks/i18n/reporters.mts create mode 100644 scripts/benchmarks/i18n/scenarios.mts diff --git a/package.json b/package.json index 94f00c497d12..0109b24fd909 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "bazel": "bazelisk", "test": "bazel test //packages/...", "build": "pnpm --silent admin build", + "benchmark": "node --no-warnings=ExperimentalWarning --experimental-transform-types --expose-gc ./scripts/devkit-admin.mts benchmark", "build-schema": "bazel build //... --build_tag_filters schema --symlink_prefix dist-schema/", "lint": "eslint --cache --max-warnings=0", "templates": "pnpm --silent admin templates", diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts new file mode 100644 index 000000000000..2f1d13b8f13b --- /dev/null +++ b/scripts/benchmark.mts @@ -0,0 +1,115 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import fs from 'node:fs'; +import { type BenchmarkCliOptions, runI18nBenchmarks } from './benchmarks/i18n/index.mts'; + +function checkBuildStatus(logger: Console): boolean { + const distFile = 'dist/@angular/build/src/tools/esbuild/i18n-inliner.js'; + const srcFile = 'packages/angular/build/src/tools/esbuild/i18n-inliner.ts'; + + if (!fs.existsSync(distFile)) { + logger.error( + 'Error: @angular/build has not been built yet.\nPlease run "pnpm build" before benchmarking.', + ); + + return false; + } + + if (fs.existsSync(srcFile)) { + const srcMtime = fs.statSync(srcFile).mtimeMs; + const distMtime = fs.statSync(distFile).mtimeMs; + if (srcMtime > distMtime) { + logger.warn( + 'Warning: Source files in packages/angular/build are newer than dist/.\n' + + 'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n', + ); + } + } + + return true; +} + +export default async function ( + options: { + _?: string[]; + scenario?: string; + iterations?: string | number; + warmup?: string | number; + concurrency?: string | number; + build?: boolean; + json?: boolean; + saveBaseline?: string; + 'save-baseline'?: string; + compareBaseline?: string; + 'compare-baseline'?: string; + help?: boolean; + [key: string]: unknown; + }, + _cwd: string, +): Promise { + const positionals = options._ ?? []; + const targetSubsystem = positionals[0] ?? 'i18n'; + + if (options.help || targetSubsystem === 'help') { + // eslint-disable-next-line no-console + console.log(` +Angular CLI Performance Benchmark Runner + +Usage: + pnpm admin benchmark [subsystem] [options] + +Subsystems: + i18n Run i18n inliner performance benchmarks (default) + +Options: + --scenario= Run a specific scenario (e.g. standard-app, enterprise-multilingual) + --iterations= Number of measured iterations (default: 5) + --warmup= Number of warmup iterations (default: 2) + --concurrency= Override worker thread pool concurrency + --build Automatically build packages before benchmarking + --json Output results in machine-readable JSON + --save-baseline= Save run results to a baseline JSON file + --compare-baseline= Compare run results against an existing baseline JSON file + --help Show this help message +`); + + return 0; + } + + if (targetSubsystem !== 'i18n') { + // eslint-disable-next-line no-console + console.error(`Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n`); + + return 1; + } + + if (options.build) { + const buildModule = await import('./build.mts'); + await buildModule.default({ local: true }); + } + + // eslint-disable-next-line no-console + if (!checkBuildStatus(console)) { + return 1; + } + + const cliOptions: BenchmarkCliOptions = { + scenario: options.scenario, + iterations: options.iterations !== undefined ? Number(options.iterations) : undefined, + warmup: options.warmup !== undefined ? Number(options.warmup) : undefined, + concurrency: options.concurrency !== undefined ? Number(options.concurrency) : undefined, + json: Boolean(options.json), + saveBaseline: options.saveBaseline ?? options['save-baseline'], + compareBaseline: options.compareBaseline ?? options['compare-baseline'], + }; + + const { exitCode } = await runI18nBenchmarks(cliOptions); + + return exitCode; +} diff --git a/scripts/benchmarks/i18n/fixtures.mts b/scripts/benchmarks/i18n/fixtures.mts new file mode 100644 index 000000000000..16305b766bec --- /dev/null +++ b/scripts/benchmarks/i18n/fixtures.mts @@ -0,0 +1,165 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; +import type { ɵParsedTranslation } from '@angular/localize'; +import { transformSync } from 'esbuild'; +import { createRequire } from 'node:module'; + +import path from 'node:path'; + +import type { BuildOutputFile } from '../../../dist/@angular/build/src/tools/esbuild/bundler-files.d.ts'; +import type { LocaleInlineOptions } from '../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.d.ts'; + +// Setup module paths to resolve dependencies from packages/angular/build +const requireFromBuild = createRequire( + path.resolve(import.meta.dirname, '../../../packages/angular/build/package.json'), +); + +const { BuildOutputFileType, createOutputFile } = requireFromBuild( + '../../../dist/@angular/build/src/tools/esbuild/bundler-files.js', +) as typeof import('../../../packages/angular/build/src/tools/esbuild/bundler-files.js'); + +const { calculateHash, initializeHash } = requireFromBuild( + '../../../dist/@angular/build/src/utils/hash.js', +) as typeof import('../../../packages/angular/build/src/utils/hash.js'); + +let isHashInitialized = false; + +export async function initializeFixtures(): Promise { + if (!isHashInitialized) { + await initializeHash(); + isHashInitialized = true; + } +} + +export function parsedTranslation( + parts: string[], + placeholderNames: string[] = [], + text?: string, +): ɵParsedTranslation { + return { + messageParts: Object.assign([...parts], { raw: [...parts] }), + placeholderNames, + text: text ?? parts.join(''), + }; +} + +export function generateTranslations( + locales: string[], + messageCount: number, +): LocaleInlineOptions[] { + return locales.map((locale) => { + const translation: Record = {}; + + for (let i = 0; i < messageCount; i++) { + const msgId = `msg_${i}`; + translation[msgId] = parsedTranslation( + [`[${locale}] Order #`, ` was confirmed for customer `, `. Thank you!`], + ['orderId', 'customerName'], + `[${locale}] Order #${i} was confirmed for customer Doe. Thank you!`, + ); + } + + const translationIntegrity = calculateHash(JSON.stringify(translation)); + + return { + locale, + translation, + translationIntegrity, + }; + }); +} + +export interface SyntheticBundleOptions { + filename: string; + targetByteSize: number; + messageCount: number; + withSourceMap?: boolean; + messageIdOffset?: number; +} + +export function generateSyntheticBundle(options: SyntheticBundleOptions): { + codeFile: BuildOutputFile; + mapFile?: BuildOutputFile; +} { + const { + filename, + targetByteSize, + messageCount, + withSourceMap = true, + messageIdOffset = 0, + } = options; + + const parts: string[] = [ + '// Synthetic test bundle generated for i18n-inliner benchmark\n', + 'export const BUNDLE_META = { generated: true, timestamp: Date.now() };\n', + ]; + + // Generate functions with $localize call sites + for (let i = 0; i < messageCount; i++) { + const msgId = `msg_${messageIdOffset + i}`; + parts.push( + `export function renderMessage_${i}(orderId, customerName) {\n`, + ` return $localize\`:@@${msgId}:Order #\${orderId}:orderId: ` + + `was confirmed for customer \${customerName}:customerName:. Thank you!\`;\n`, + `}\n`, + ); + } + + // Calculate current approximate size and pad with realistic JS functions if needed + let currentSize = parts.reduce((acc, str) => acc + str.length, 0); + let classIndex = 0; + + while (currentSize < targetByteSize) { + const filler = + `export class DataProcessor_${classIndex} {\n` + + ` constructor(id, options = {}) {\n` + + ` this.id = id;\n` + + ` this.options = Object.assign({ enabled: true, retries: 3 }, options);\n` + + ` this.history = [];\n` + + ` }\n` + + ` process(batch) {\n` + + ` if (!Array.isArray(batch)) return [];\n` + + ` const result = batch.map((item, idx) => ({\n` + + ` id: this.id + '_' + idx,\n` + + ` source: item,\n` + + ` timestamp: Date.now(),\n` + + ` active: true\n` + + ` }));\n` + + ` this.history.push(...result);\n` + + ` return result;\n` + + ` }\n` + + `}\n`; + + parts.push(filler); + currentSize += filler.length; + classIndex++; + } + + const rawCode = parts.join(''); + + let finalCode = rawCode; + let finalMap: string | undefined; + + if (withSourceMap) { + const result = transformSync(rawCode, { + sourcemap: true, + sourcefile: filename.replace(/\.js$/, '.ts'), + }); + finalCode = result.code; + finalMap = result.map; + } + + const codeFile = createOutputFile(filename, finalCode, BuildOutputFileType.Browser); + const mapFile = finalMap + ? createOutputFile(filename + '.map', finalMap, BuildOutputFileType.Browser) + : undefined; + + return { codeFile, mapFile }; +} diff --git a/scripts/benchmarks/i18n/harness.mts b/scripts/benchmarks/i18n/harness.mts new file mode 100644 index 000000000000..700199c0af0b --- /dev/null +++ b/scripts/benchmarks/i18n/harness.mts @@ -0,0 +1,155 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; + +export interface BenchmarkScenario { + name: string; + description: string; + inputSizeBytes: number; + localeCount: number; + run(iteration: number): Promise; + setup?(): Promise; + teardown?(): Promise; +} + +export interface ScenarioResult { + name: string; + description: string; + inputSizeBytes: number; + localeCount: number; + iterations: number; + warmup: number; + durationsMs: number[]; + minMs: number; + maxMs: number; + meanMs: number; + medianMs: number; + p95Ms: number; + stdDevMs: number; + throughputMBps: number; + peakRssBytes: number; + peakHeapBytes: number; + rssDeltaBytes: number; + heapUsedDeltaBytes: number; +} + +export interface BenchmarkRunOptions { + warmup?: number; + iterations?: number; +} + +function calculatePercentile(sortedValues: number[], percentile: number): number { + if (sortedValues.length === 0) { + return 0; + } + const index = (percentile / 100) * (sortedValues.length - 1); + const lower = Math.floor(index); + const upper = Math.ceil(index); + const weight = index - lower; + + return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; +} + +export async function runScenario( + scenario: BenchmarkScenario, + options: BenchmarkRunOptions = {}, +): Promise { + const warmup = options.warmup ?? 2; + const iterations = options.iterations ?? 5; + + await scenario.setup?.(); + + try { + // Warmup phase + for (let w = 0; w < warmup; w++) { + // Force GC if available between warmups + global.gc?.(); + await scenario.run(-(w + 1)); + } + + // Measurement phase + const durationsMs: number[] = []; + let peakRssBytes = 0; + let peakHeapBytes = 0; + let maxRssDeltaBytes = 0; + let initialHeap = 0; + let finalHeap = 0; + + for (let i = 0; i < iterations; i++) { + global.gc?.(); + + const memBefore = process.memoryUsage(); + if (i === 0) { + initialHeap = memBefore.heapUsed; + } + + const start = performance.now(); + await scenario.run(i); + const duration = performance.now() - start; + + durationsMs.push(duration); + + const memAfter = process.memoryUsage(); + if (memAfter.rss > peakRssBytes) { + peakRssBytes = memAfter.rss; + } + if (memAfter.heapUsed > peakHeapBytes) { + peakHeapBytes = memAfter.heapUsed; + } + const rssDelta = Math.max(0, memAfter.rss - memBefore.rss); + if (rssDelta > maxRssDeltaBytes) { + maxRssDeltaBytes = rssDelta; + } + finalHeap = memAfter.heapUsed; + + // Force GC immediately after iteration to clean main thread isolate + global.gc?.(); + } + + // Sort ascending for percentile computation + const sorted = [...durationsMs].sort((a, b) => a - b); + const minMs = sorted[0]; + const maxMs = sorted[sorted.length - 1]; + const meanMs = durationsMs.reduce((sum, d) => sum + d, 0) / durationsMs.length; + const medianMs = calculatePercentile(sorted, 50); + const p95Ms = calculatePercentile(sorted, 95); + + const variance = + durationsMs.reduce((sum, d) => sum + Math.pow(d - meanMs, 2), 0) / durationsMs.length; + const stdDevMs = Math.sqrt(variance); + + // Throughput: total effective processed code volume in MB / mean seconds + const totalProcessedMb = (scenario.inputSizeBytes * scenario.localeCount) / (1024 * 1024); + const meanSeconds = meanMs / 1000; + const throughputMBps = meanSeconds > 0 ? totalProcessedMb / meanSeconds : 0; + + return { + name: scenario.name, + description: scenario.description, + inputSizeBytes: scenario.inputSizeBytes, + localeCount: scenario.localeCount, + iterations, + warmup, + durationsMs, + minMs, + maxMs, + meanMs, + medianMs, + p95Ms, + stdDevMs, + throughputMBps, + peakRssBytes, + peakHeapBytes, + rssDeltaBytes: maxRssDeltaBytes, + heapUsedDeltaBytes: Math.max(0, finalHeap - initialHeap), + }; + } finally { + await scenario.teardown?.(); + } +} diff --git a/scripts/benchmarks/i18n/index.mts b/scripts/benchmarks/i18n/index.mts new file mode 100644 index 000000000000..8b6503d60e83 --- /dev/null +++ b/scripts/benchmarks/i18n/index.mts @@ -0,0 +1,164 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { type ScenarioResult, runScenario } from './harness.mts'; +import { + buildReportData, + formatComparisonTable, + formatConsoleTable, + formatJsonReport, +} from './reporters.mts'; +import { type ScenarioFactoryOptions, getAllScenarios, getScenarioByName } from './scenarios.mts'; + +export interface BenchmarkCliOptions extends ScenarioFactoryOptions { + scenario?: string; + iterations?: number; + warmup?: number; + verbose?: boolean; + json?: boolean; + inProcess?: boolean; + saveBaseline?: string; + compareBaseline?: string; +} + +export async function runI18nBenchmarks( + options: BenchmarkCliOptions = {}, +): Promise<{ results: ScenarioResult[]; exitCode: number }> { + const warmup = options.warmup ?? 2; + const iterations = options.iterations ?? 5; + + let scenariosToRun = getAllScenarios({ concurrency: options.concurrency }); + + if (options.scenario) { + const single = getScenarioByName(options.scenario, { concurrency: options.concurrency }); + if (!single) { + // eslint-disable-next-line no-console + console.error( + `Unknown scenario: "${options.scenario}".\nAvailable scenarios: ${scenariosToRun.map((s) => s.name).join(', ')}`, + ); + + return { results: [], exitCode: 1 }; + } + scenariosToRun = [single]; + } + + const results: ScenarioResult[] = []; + + // When running multiple scenarios in a suite, isolate each scenario into its own child process + // so operating system memory and thread caches are not accumulated across scenarios. + if (scenariosToRun.length > 1 && !options.inProcess) { + for (const scenario of scenariosToRun) { + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + `Running scenario: ${scenario.name} (${warmup} warmups, ${iterations} iterations)...`, + ); + } + + const args = [ + '--no-warnings=ExperimentalWarning', + '--experimental-transform-types', + '--expose-gc', + path.resolve(import.meta.dirname, '../../devkit-admin.mts'), + 'benchmark', + 'i18n', + `--scenario=${scenario.name}`, + `--warmup=${warmup}`, + `--iterations=${iterations}`, + '--json', + ]; + if (options.concurrency !== undefined) { + args.push(`--concurrency=${options.concurrency}`); + } + + const proc = spawnSync(process.execPath, args, { encoding: 'utf-8' }); + if (proc.status !== 0) { + // eslint-disable-next-line no-console + console.error(`Error running scenario ${scenario.name}:\n${proc.stderr || proc.stdout}`); + + return { results, exitCode: 1 }; + } + + try { + const parsed = JSON.parse(proc.stdout); + const scenarioResult: ScenarioResult | undefined = Array.isArray(parsed) + ? parsed[0] + : parsed.results?.[0]; + if (scenarioResult) { + results.push(scenarioResult); + } + } catch { + // eslint-disable-next-line no-console + console.error(`Failed to parse result for scenario ${scenario.name}:\n${proc.stdout}`); + + return { results, exitCode: 1 }; + } + } + } else { + for (const scenario of scenariosToRun) { + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + `Running scenario: ${scenario.name} (${warmup} warmups, ${iterations} iterations)...`, + ); + } + + try { + const result = await runScenario(scenario, { warmup, iterations }); + results.push(result); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`Error running scenario ${scenario.name}:`, error); + + return { results, exitCode: 1 }; + } + } + } + + if (options.json) { + // eslint-disable-next-line no-console + console.log(formatJsonReport(results)); + } else { + // eslint-disable-next-line no-console + console.log('\n' + formatConsoleTable(results)); + } + + if (options.saveBaseline) { + const reportData = buildReportData(results); + await fs.writeFile(options.saveBaseline, JSON.stringify(reportData, null, 2), 'utf-8'); + if (!options.json) { + // eslint-disable-next-line no-console + console.log(`Saved baseline to: ${options.saveBaseline}`); + } + } + + if (options.compareBaseline) { + try { + const baselineContent = await fs.readFile(options.compareBaseline, 'utf-8'); + const baselineData = JSON.parse(baselineContent); + const baselineResults: ScenarioResult[] = Array.isArray(baselineData) + ? baselineData + : (baselineData.results ?? []); + + if (!options.json) { + // eslint-disable-next-line no-console + console.log(formatComparisonTable(results, baselineResults)); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(`Failed to load baseline from ${options.compareBaseline}:`, error); + } + } + + return { results, exitCode: 0 }; +} diff --git a/scripts/benchmarks/i18n/init-env.mts b/scripts/benchmarks/i18n/init-env.mts new file mode 100644 index 000000000000..679281e4dc18 --- /dev/null +++ b/scripts/benchmarks/i18n/init-env.mts @@ -0,0 +1,24 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import Module from 'node:module'; +import path from 'node:path'; + +// Resolve dependencies from packages/angular/build/node_modules for runtime resolution in dist/ +const buildNodeModules = path.resolve( + import.meta.dirname, + '../../../packages/angular/build/node_modules', +); + +const currentPath = process.env.NODE_PATH ?? ''; +if (!currentPath.includes(buildNodeModules)) { + process.env.NODE_PATH = currentPath ? `${buildNodeModules}:${currentPath}` : buildNodeModules; + // Initialize internal search paths for Node CommonJS loader + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (Module as any)._initPaths?.(); +} diff --git a/scripts/benchmarks/i18n/reporters.mts b/scripts/benchmarks/i18n/reporters.mts new file mode 100644 index 000000000000..d5153552a2b3 --- /dev/null +++ b/scripts/benchmarks/i18n/reporters.mts @@ -0,0 +1,159 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import os from 'node:os'; +import type { ScenarioResult } from './harness.mts'; + +export interface BenchmarkReportData { + timestamp: string; + system: { + nodeVersion: string; + platform: string; + arch: string; + cpus: number; + cpuModel: string; + totalMemoryMb: number; + }; + results: ScenarioResult[]; +} + +export function buildReportData(results: ScenarioResult[]): BenchmarkReportData { + const cpus = os.cpus(); + + return { + timestamp: new Date().toISOString(), + system: { + nodeVersion: process.version, + platform: os.platform(), + arch: os.arch(), + cpus: cpus.length, + cpuModel: cpus[0]?.model ?? 'unknown', + totalMemoryMb: Math.round(os.totalmem() / (1024 * 1024)), + }, + results, + }; +} + +export function formatJsonReport(results: ScenarioResult[]): string { + return JSON.stringify(buildReportData(results), null, 2); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) { + return `${bytes} B`; + } + if (bytes < 1024 * 1024) { + return `${(bytes / 1024).toFixed(1)} KB`; + } + + return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; +} + +function padRight(str: string, len: number): string { + return str.length >= len ? str : str + ' '.repeat(len - str.length); +} + +function padLeft(str: string, len: number): string { + return str.length >= len ? str : ' '.repeat(len - str.length) + str; +} + +export function formatConsoleTable(results: ScenarioResult[]): string { + const cpus = os.cpus(); + const gcStatus = typeof global.gc === 'function' ? 'active' : 'inactive (run with --expose-gc)'; + const header = + `================================================================================================================\n` + + `i18n Inliner Performance Benchmarks (Node ${process.version}, ${cpus.length} CPUs: ${cpus[0]?.model ?? ''} | GC: ${gcStatus})\n` + + `================================================================================================================\n`; + + const columns = [ + { name: 'Scenario', width: 25 }, + { name: 'Input Size', width: 11 }, + { name: 'Locales', width: 8 }, + { name: 'Mean Latency', width: 13 }, + { name: 'p50 / p95', width: 18 }, + { name: 'Throughput', width: 12 }, + { name: 'Peak Heap', width: 11 }, + { name: 'Peak RSS', width: 11 }, + ]; + + const colHeader = columns.map((col) => padRight(col.name, col.width)).join(' '); + const separator = columns.map((col) => '-'.repeat(col.width)).join(' '); + + const rows = results.map((r) => { + const inputFormatted = formatBytes(r.inputSizeBytes); + const meanFormatted = `${r.meanMs.toFixed(1)} ms`; + const p50p95Formatted = `${r.medianMs.toFixed(0)} ms / ${r.p95Ms.toFixed(0)} ms`; + const throughputFormatted = `${r.throughputMBps.toFixed(1)} MB/s`; + const peakHeapFormatted = formatBytes(r.peakHeapBytes ?? 0); + const peakRssFormatted = formatBytes(r.peakRssBytes); + + return [ + padRight(r.name, columns[0].width), + padLeft(inputFormatted, columns[1].width), + padLeft(r.localeCount.toString(), columns[2].width), + padLeft(meanFormatted, columns[3].width), + padLeft(p50p95Formatted, columns[4].width), + padLeft(throughputFormatted, columns[5].width), + padLeft(peakHeapFormatted, columns[6].width), + padLeft(peakRssFormatted, columns[7].width), + ].join(' '); + }); + + return `${header}\n${colHeader}\n${separator}\n${rows.join('\n')}\n${'='.repeat(separator.length)}\n`; +} + +export function formatComparisonTable( + currentResults: ScenarioResult[], + baselineResults: ScenarioResult[], +): string { + const header = + `====================================================================================================\n` + + `i18n Inliner Benchmark Comparison (Current vs Baseline)\n` + + `====================================================================================================\n`; + + const columns = [ + { name: 'Scenario', width: 26 }, + { name: 'Baseline Mean', width: 14 }, + { name: 'Current Mean', width: 14 }, + { name: 'Latency Diff', width: 14 }, + { name: 'Baseline MB/s', width: 14 }, + { name: 'Current MB/s', width: 14 }, + ]; + + const colHeader = columns.map((col) => padRight(col.name, col.width)).join(' '); + const separator = columns.map((col) => '-'.repeat(col.width)).join(' '); + + const rows = currentResults.map((curr) => { + const base = baselineResults.find((b) => b.name === curr.name); + if (!base) { + return [ + padRight(curr.name, columns[0].width), + padLeft('N/A', columns[1].width), + padLeft(`${curr.meanMs.toFixed(1)} ms`, columns[2].width), + padLeft('NEW', columns[3].width), + padLeft('N/A', columns[4].width), + padLeft(`${curr.throughputMBps.toFixed(1)} MB/s`, columns[5].width), + ].join(' '); + } + + const diffPercent = ((curr.meanMs - base.meanMs) / base.meanMs) * 100; + const diffSign = diffPercent > 0 ? '+' : ''; + const diffText = `${diffSign}${diffPercent.toFixed(1)}% ${diffPercent > 1 ? '(slower)' : diffPercent < -1 ? '(faster)' : '(same)'}`; + + return [ + padRight(curr.name, columns[0].width), + padLeft(`${base.meanMs.toFixed(1)} ms`, columns[1].width), + padLeft(`${curr.meanMs.toFixed(1)} ms`, columns[2].width), + padLeft(diffText, columns[3].width), + padLeft(`${base.throughputMBps.toFixed(1)} MB/s`, columns[4].width), + padLeft(`${curr.throughputMBps.toFixed(1)} MB/s`, columns[5].width), + ].join(' '); + }); + + return `${header}\n${colHeader}\n${separator}\n${rows.join('\n')}\n${'='.repeat(separator.length)}\n`; +} diff --git a/scripts/benchmarks/i18n/scenarios.mts b/scripts/benchmarks/i18n/scenarios.mts new file mode 100644 index 000000000000..f862e28c04f6 --- /dev/null +++ b/scripts/benchmarks/i18n/scenarios.mts @@ -0,0 +1,404 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import os from 'node:os'; +import path from 'node:path'; + +import type { BuildOutputFile } from '../../../dist/@angular/build/src/tools/esbuild/bundler-files.d.ts'; +import type { LocaleInlineOptions } from '../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.d.ts'; +import { generateSyntheticBundle, generateTranslations, initializeFixtures } from './fixtures.mts'; +import type { BenchmarkScenario } from './harness.mts'; + +const requireFromBuild = createRequire( + path.resolve(import.meta.dirname, '../../../packages/angular/build/package.json'), +); + +const { I18nInliner } = requireFromBuild( + '../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.js', +) as typeof import('../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.d.ts'); + +export interface ScenarioFactoryOptions { + concurrency?: number; +} + +const DEFAULT_LOCALES_8 = ['fr', 'de', 'es', 'ja', 'zh', 'it', 'pt', 'ko']; +const DEFAULT_LOCALES_32 = [ + 'fr', + 'de', + 'es', + 'ja', + 'zh', + 'it', + 'pt', + 'ko', + 'ru', + 'pl', + 'nl', + 'tr', + 'ar', + 'hi', + 'sv', + 'da', + 'fi', + 'no', + 'cs', + 'el', + 'he', + 'hu', + 'id', + 'ms', + 'ro', + 'sk', + 'th', + 'uk', + 'vi', + 'bg', + 'hr', + 'sr', +]; + +interface GeneratedWorkload { + files: BuildOutputFile[]; + locales: LocaleInlineOptions[]; + totalInputSizeBytes: number; +} + +function createBundleSet( + mainSizeBytes: number, + chunkCount: number, + chunkSizeBytes: number, + messageCount: number, + withSourceMap: boolean, +): BuildOutputFile[] { + const files: BuildOutputFile[] = []; + + const mainMessages = Math.floor(messageCount * 0.4); + const { codeFile: mainCode, mapFile: mainMap } = generateSyntheticBundle({ + filename: 'main.js', + targetByteSize: mainSizeBytes, + messageCount: mainMessages, + withSourceMap, + messageIdOffset: 0, + }); + + files.push(mainCode); + if (mainMap) { + files.push(mainMap); + } + + const remainingMessages = messageCount - mainMessages; + const messagesPerChunk = Math.max(1, Math.floor(remainingMessages / chunkCount)); + + for (let i = 0; i < chunkCount; i++) { + const { codeFile, mapFile } = generateSyntheticBundle({ + filename: `chunk_${i}.js`, + targetByteSize: chunkSizeBytes, + messageCount: messagesPerChunk, + withSourceMap, + messageIdOffset: mainMessages + i * messagesPerChunk, + }); + + files.push(codeFile); + if (mapFile) { + files.push(mapFile); + } + } + + return files; +} + +function calculateInputSizeBytes(files: BuildOutputFile[]): number { + return files.reduce((total, f) => { + // Only count JS code size towards raw input volume (maps are auxiliary) + return f.path.endsWith('.js') ? total + f.size : total; + }, 0); +} + +/** + * 1. Standard App Scenario: + * 1 main bundle (1 MB) + 20 route chunks (50 KB) = ~2 MB input JS, 8 locales, maps enabled. + */ +export function createStandardAppScenario(options: ScenarioFactoryOptions = {}): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'standard-app', + description: 'Standard App: 1 main bundle (1 MB) + 20 chunks (50 KB), 8 locales, sourcemaps ON', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(1024 * 1024, 20, 50 * 1024, 1000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * 2. Standard App without Sourcemaps: + * Evaluates inlining without source map generation and remapping. + */ +export function createStandardAppNoMapsScenario( + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'standard-app-no-maps', + description: + 'Standard App (No Maps): 1 main bundle (1 MB) + 20 chunks (50 KB), 8 locales, sourcemaps OFF', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(1024 * 1024, 20, 50 * 1024, 1000, false); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * 3. Enterprise Multilingual Scenario: + * 1 main bundle (2 MB) + 40 chunks (60 KB), 32 locales, 3,000 translations, sourcemaps ON. + * Exercises sliding window batching (4 windows of 8 locales) and memory scaling. + */ +export function createEnterpriseScenario(options: ScenarioFactoryOptions = {}): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'enterprise-multilingual', + description: 'Enterprise: 1 main (2 MB) + 40 chunks (60 KB), 32 locales, sourcemaps ON', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_32.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(2 * 1024 * 1024, 40, 60 * 1024, 3000, true); + const locales = generateTranslations(DEFAULT_LOCALES_32, 3000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * 4. Monolithic Dominant Scenario: + * 1 dominant bundle (6 MB) + 5 small runtime chunks (30 KB), 8 locales, sourcemaps ON. + * Evaluates whether LPT + DOMINANT_FILE_RATIO sharding saturates workers efficiently. + */ +export function createMonolithicScenario(options: ScenarioFactoryOptions = {}): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'monolithic-dominant', + description: 'Monolithic: 1 dominant bundle (6 MB) + 5 small chunks (30 KB), 8 locales', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(6 * 1024 * 1024, 5, 30 * 1024, 2000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 2000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * Helper to prime the persistent cache in an isolated process. + */ +export async function primeCache(cacheDir: string, concurrency?: number): Promise { + await initializeFixtures(); + const files = createBundleSet(1024 * 1024, 15, 50 * 1024, 1000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + const primer = new I18nInliner({ + missingTranslation: 'warning', + persistentCachePath: cacheDir, + maxConcurrency: concurrency, + }); + await primer.inlineAll(files, locales); + await primer.close(); +} + +/** + * 5. Persistent Cache Warm Scenario: + * Evaluates throughput when 100% of transformed files and translations are pre-cached in LMDB. + */ +export function createPersistentCacheWarmScenario( + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + let cacheDir: string | undefined; + + return { + name: 'persistent-cache-warm', + description: + 'Persistent Cache (Warm): 100% LMDB cache hits for transformed files & translations', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + cacheDir = await fs.mkdtemp(path.join(os.tmpdir(), 'angular-i18n-bench-cache-')); + const files = createBundleSet(1024 * 1024, 15, 50 * 1024, 1000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + + // Prime the persistent cache out-of-process so cold worker thread allocations + // do not inflate this process's RSS metrics. + const primerCode = + `import { primeCache } from ${JSON.stringify(path.resolve(import.meta.dirname, './scenarios.mts'))};\n` + + `await primeCache(${JSON.stringify(cacheDir)}, ${options.concurrency ?? 'undefined'});\n`; + + spawnSync( + process.execPath, + ['--no-warnings=ExperimentalWarning', '--experimental-transform-types', '-e', primerCode], + { stdio: 'inherit' }, + ); + }, + + async run() { + if (!workload || !cacheDir) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + persistentCachePath: cacheDir, + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + async teardown() { + if (cacheDir) { + await fs.rm(cacheDir, { recursive: true, force: true }).catch(() => {}); + } + }, + }; +} + +export function getAllScenarios(options: ScenarioFactoryOptions = {}): BenchmarkScenario[] { + return [ + createStandardAppScenario(options), + createStandardAppNoMapsScenario(options), + createEnterpriseScenario(options), + createMonolithicScenario(options), + createPersistentCacheWarmScenario(options), + ]; +} + +export function getScenarioByName( + name: string, + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario | undefined { + const all = getAllScenarios(options); + + return all.find((s) => s.name.toLowerCase() === name.toLowerCase()); +} From ae6291b269e86a0d6004e7fa4f63ada25db69eba Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:15:17 -0400 Subject: [PATCH 2/2] test(@angular/build): add large-enterprise-10k benchmark scenario Adds a large-scale stress test scenario to the i18n inliner benchmark suite with 10,000 translation messages across 32 locales. The scenario generates a 3 MB main bundle and 100 route chunks with source maps, evaluating binary translation catalog encoding, small file batching, and memory scaling under maximum enterprise workloads. --- scripts/benchmark.mts | 45 ++++++++++++-- scripts/benchmarks/i18n/fixtures.mts | 4 +- scripts/benchmarks/i18n/harness.mts | 6 +- scripts/benchmarks/i18n/index.mts | 7 ++- scripts/benchmarks/i18n/init-env.mts | 4 +- scripts/benchmarks/i18n/scenarios.mts | 84 +++++++++++++++++++++++++-- 6 files changed, 131 insertions(+), 19 deletions(-) diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index 2f1d13b8f13b..bf35b97808e3 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -7,6 +7,7 @@ */ import fs from 'node:fs'; +import path from 'node:path'; import { type BenchmarkCliOptions, runI18nBenchmarks } from './benchmarks/i18n/index.mts'; function checkBuildStatus(logger: Console): boolean { @@ -44,6 +45,8 @@ export default async function ( concurrency?: string | number; build?: boolean; json?: boolean; + inProcess?: boolean; + 'in-process'?: boolean; saveBaseline?: string; 'save-baseline'?: string; compareBaseline?: string; @@ -72,6 +75,7 @@ Options: --iterations= Number of measured iterations (default: 5) --warmup= Number of warmup iterations (default: 2) --concurrency= Override worker thread pool concurrency + --in-process Run all scenarios in a single process (useful for debugging) --build Automatically build packages before benchmarking --json Output results in machine-readable JSON --save-baseline= Save run results to a baseline JSON file @@ -99,14 +103,45 @@ Options: return 1; } + const rawSaveBaseline = options.saveBaseline ?? options['save-baseline']; + const rawCompareBaseline = options.compareBaseline ?? options['compare-baseline']; + + const iterations = options.iterations !== undefined ? Number(options.iterations) : undefined; + const warmup = options.warmup !== undefined ? Number(options.warmup) : undefined; + const concurrency = options.concurrency !== undefined ? Number(options.concurrency) : undefined; + + if (iterations !== undefined && (!Number.isInteger(iterations) || iterations < 1)) { + // eslint-disable-next-line no-console + console.error('Error: --iterations must be a positive integer.'); + + return 1; + } + + if (warmup !== undefined && (!Number.isInteger(warmup) || warmup < 0)) { + // eslint-disable-next-line no-console + console.error('Error: --warmup must be a non-negative integer.'); + + return 1; + } + + if (concurrency !== undefined && (!Number.isInteger(concurrency) || concurrency < 1)) { + // eslint-disable-next-line no-console + console.error('Error: --concurrency must be a positive integer.'); + + return 1; + } + const cliOptions: BenchmarkCliOptions = { scenario: options.scenario, - iterations: options.iterations !== undefined ? Number(options.iterations) : undefined, - warmup: options.warmup !== undefined ? Number(options.warmup) : undefined, - concurrency: options.concurrency !== undefined ? Number(options.concurrency) : undefined, + iterations, + warmup, + concurrency, json: Boolean(options.json), - saveBaseline: options.saveBaseline ?? options['save-baseline'], - compareBaseline: options.compareBaseline ?? options['compare-baseline'], + inProcess: Boolean(options.inProcess ?? options['in-process']), + saveBaseline: rawSaveBaseline ? path.resolve(_cwd, String(rawSaveBaseline)) : undefined, + compareBaseline: rawCompareBaseline + ? path.resolve(_cwd, String(rawCompareBaseline)) + : undefined, }; const { exitCode } = await runI18nBenchmarks(cliOptions); diff --git a/scripts/benchmarks/i18n/fixtures.mts b/scripts/benchmarks/i18n/fixtures.mts index 16305b766bec..3d53c3dbff5b 100644 --- a/scripts/benchmarks/i18n/fixtures.mts +++ b/scripts/benchmarks/i18n/fixtures.mts @@ -13,8 +13,8 @@ import { createRequire } from 'node:module'; import path from 'node:path'; -import type { BuildOutputFile } from '../../../dist/@angular/build/src/tools/esbuild/bundler-files.d.ts'; -import type { LocaleInlineOptions } from '../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.d.ts'; +import type { BuildOutputFile } from '../../../packages/angular/build/src/tools/esbuild/bundler-files.js'; +import type { LocaleInlineOptions } from '../../../packages/angular/build/src/tools/esbuild/i18n-inliner.js'; // Setup module paths to resolve dependencies from packages/angular/build const requireFromBuild = createRequire( diff --git a/scripts/benchmarks/i18n/harness.mts b/scripts/benchmarks/i18n/harness.mts index 700199c0af0b..caf7f0753d24 100644 --- a/scripts/benchmarks/i18n/harness.mts +++ b/scripts/benchmarks/i18n/harness.mts @@ -63,9 +63,9 @@ export async function runScenario( const warmup = options.warmup ?? 2; const iterations = options.iterations ?? 5; - await scenario.setup?.(); - try { + await scenario.setup?.(); + // Warmup phase for (let w = 0; w < warmup; w++) { // Force GC if available between warmups @@ -106,10 +106,10 @@ export async function runScenario( if (rssDelta > maxRssDeltaBytes) { maxRssDeltaBytes = rssDelta; } - finalHeap = memAfter.heapUsed; // Force GC immediately after iteration to clean main thread isolate global.gc?.(); + finalHeap = process.memoryUsage().heapUsed; } // Sort ascending for percentile computation diff --git a/scripts/benchmarks/i18n/index.mts b/scripts/benchmarks/i18n/index.mts index 8b6503d60e83..276671c193e0 100644 --- a/scripts/benchmarks/i18n/index.mts +++ b/scripts/benchmarks/i18n/index.mts @@ -82,9 +82,12 @@ export async function runI18nBenchmarks( } const proc = spawnSync(process.execPath, args, { encoding: 'utf-8' }); - if (proc.status !== 0) { + if (proc.status !== 0 || proc.error) { // eslint-disable-next-line no-console - console.error(`Error running scenario ${scenario.name}:\n${proc.stderr || proc.stdout}`); + console.error( + `Error running scenario ${scenario.name}:\n` + + (proc.error?.message ?? (proc.stderr || proc.stdout)), + ); return { results, exitCode: 1 }; } diff --git a/scripts/benchmarks/i18n/init-env.mts b/scripts/benchmarks/i18n/init-env.mts index 679281e4dc18..a1690cf576c8 100644 --- a/scripts/benchmarks/i18n/init-env.mts +++ b/scripts/benchmarks/i18n/init-env.mts @@ -17,7 +17,9 @@ const buildNodeModules = path.resolve( const currentPath = process.env.NODE_PATH ?? ''; if (!currentPath.includes(buildNodeModules)) { - process.env.NODE_PATH = currentPath ? `${buildNodeModules}:${currentPath}` : buildNodeModules; + process.env.NODE_PATH = currentPath + ? `${buildNodeModules}${path.delimiter}${currentPath}` + : buildNodeModules; // Initialize internal search paths for Node CommonJS loader // eslint-disable-next-line @typescript-eslint/no-explicit-any (Module as any)._initPaths?.(); diff --git a/scripts/benchmarks/i18n/scenarios.mts b/scripts/benchmarks/i18n/scenarios.mts index f862e28c04f6..c00f34ded8a2 100644 --- a/scripts/benchmarks/i18n/scenarios.mts +++ b/scripts/benchmarks/i18n/scenarios.mts @@ -13,9 +13,10 @@ import fs from 'node:fs/promises'; import { createRequire } from 'node:module'; import os from 'node:os'; import path from 'node:path'; +import { pathToFileURL } from 'node:url'; -import type { BuildOutputFile } from '../../../dist/@angular/build/src/tools/esbuild/bundler-files.d.ts'; -import type { LocaleInlineOptions } from '../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.d.ts'; +import type { BuildOutputFile } from '../../../packages/angular/build/src/tools/esbuild/bundler-files.js'; +import type { LocaleInlineOptions } from '../../../packages/angular/build/src/tools/esbuild/i18n-inliner.js'; import { generateSyntheticBundle, generateTranslations, initializeFixtures } from './fixtures.mts'; import type { BenchmarkScenario } from './harness.mts'; @@ -25,7 +26,7 @@ const requireFromBuild = createRequire( const { I18nInliner } = requireFromBuild( '../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.js', -) as typeof import('../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.d.ts'); +) as typeof import('../../../packages/angular/build/src/tools/esbuild/i18n-inliner.js'); export interface ScenarioFactoryOptions { concurrency?: number; @@ -124,6 +125,15 @@ function calculateInputSizeBytes(files: BuildOutputFile[]): number { }, 0); } +/** + * Note on Worker Pool Lifecycle: + * Each scenario's run() method instantiates and closes an I18nInliner per iteration. + * This is designed as a macro benchmark to reflect the cold-start behavior of single-shot + * CLI build invocations (including worker thread pool initialization, task dispatch, + * inlining transformations, and thread pool shutdown). Warmup iterations warm up the + * main-thread V8 isolate and runtime paths, while worker threads are initialized per iteration. + */ + /** * 1. Standard App Scenario: * 1 main bundle (1 MB) + 20 route chunks (50 KB) = ~2 MB input JS, 8 locales, maps enabled. @@ -350,15 +360,28 @@ export function createPersistentCacheWarmScenario( // Prime the persistent cache out-of-process so cold worker thread allocations // do not inflate this process's RSS metrics. + const scenariosUrl = pathToFileURL(path.resolve(import.meta.dirname, './scenarios.mts')).href; const primerCode = - `import { primeCache } from ${JSON.stringify(path.resolve(import.meta.dirname, './scenarios.mts'))};\n` + + `import { primeCache } from ${JSON.stringify(scenariosUrl)};\n` + `await primeCache(${JSON.stringify(cacheDir)}, ${options.concurrency ?? 'undefined'});\n`; - spawnSync( + const primerProc = spawnSync( process.execPath, - ['--no-warnings=ExperimentalWarning', '--experimental-transform-types', '-e', primerCode], + [ + '--no-warnings=ExperimentalWarning', + '--experimental-transform-types', + '--input-type=module', + '-e', + primerCode, + ], { stdio: 'inherit' }, ); + + if (primerProc.status !== 0 || primerProc.error) { + throw new Error( + `Failed to prime cache for persistent-cache-warm scenario: ${primerProc.error?.message ?? primerProc.status}`, + ); + } }, async run() { @@ -384,11 +407,60 @@ export function createPersistentCacheWarmScenario( }; } +/** + * 6. Large Enterprise (10k translations) Scenario: + * 1 main bundle (3 MB) + 100 chunks (50 KB), 32 locales, 10,000 translations, sourcemaps ON. + * Maximum scale stress test for binary translation tables, memory retention, and multi-locale windows. + */ +export function createLargeEnterpriseScenario( + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'large-enterprise-10k', + description: + 'Large Enterprise (10k msgs): 1 main (3 MB) + 100 chunks (50 KB), 32 locales, 10,000 translations', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_32.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(3 * 1024 * 1024, 100, 50 * 1024, 10000, true); + const locales = generateTranslations(DEFAULT_LOCALES_32, 10000); + + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + export function getAllScenarios(options: ScenarioFactoryOptions = {}): BenchmarkScenario[] { return [ createStandardAppScenario(options), createStandardAppNoMapsScenario(options), createEnterpriseScenario(options), + createLargeEnterpriseScenario(options), createMonolithicScenario(options), createPersistentCacheWarmScenario(options), ];