diff --git a/package.json b/package.json index 94f00c497d12..0109b24fd909 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "bazel": "bazelisk", "test": "bazel test //packages/...", "build": "pnpm --silent admin build", + "benchmark": "node --no-warnings=ExperimentalWarning --experimental-transform-types --expose-gc ./scripts/devkit-admin.mts benchmark", "build-schema": "bazel build //... --build_tag_filters schema --symlink_prefix dist-schema/", "lint": "eslint --cache --max-warnings=0", "templates": "pnpm --silent admin templates", diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts new file mode 100644 index 000000000000..bf35b97808e3 --- /dev/null +++ b/scripts/benchmark.mts @@ -0,0 +1,150 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { type BenchmarkCliOptions, runI18nBenchmarks } from './benchmarks/i18n/index.mts'; + +function checkBuildStatus(logger: Console): boolean { + const distFile = 'dist/@angular/build/src/tools/esbuild/i18n-inliner.js'; + const srcFile = 'packages/angular/build/src/tools/esbuild/i18n-inliner.ts'; + + if (!fs.existsSync(distFile)) { + logger.error( + 'Error: @angular/build has not been built yet.\nPlease run "pnpm build" before benchmarking.', + ); + + return false; + } + + if (fs.existsSync(srcFile)) { + const srcMtime = fs.statSync(srcFile).mtimeMs; + const distMtime = fs.statSync(distFile).mtimeMs; + if (srcMtime > distMtime) { + logger.warn( + 'Warning: Source files in packages/angular/build are newer than dist/.\n' + + 'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n', + ); + } + } + + return true; +} + +export default async function ( + options: { + _?: string[]; + scenario?: string; + iterations?: string | number; + warmup?: string | number; + concurrency?: string | number; + build?: boolean; + json?: boolean; + inProcess?: boolean; + 'in-process'?: boolean; + saveBaseline?: string; + 'save-baseline'?: string; + compareBaseline?: string; + 'compare-baseline'?: string; + help?: boolean; + [key: string]: unknown; + }, + _cwd: string, +): Promise { + const positionals = options._ ?? []; + const targetSubsystem = positionals[0] ?? 'i18n'; + + if (options.help || targetSubsystem === 'help') { + // eslint-disable-next-line no-console + console.log(` +Angular CLI Performance Benchmark Runner + +Usage: + pnpm admin benchmark [subsystem] [options] + +Subsystems: + i18n Run i18n inliner performance benchmarks (default) + +Options: + --scenario= Run a specific scenario (e.g. standard-app, enterprise-multilingual) + --iterations= Number of measured iterations (default: 5) + --warmup= Number of warmup iterations (default: 2) + --concurrency= Override worker thread pool concurrency + --in-process Run all scenarios in a single process (useful for debugging) + --build Automatically build packages before benchmarking + --json Output results in machine-readable JSON + --save-baseline= Save run results to a baseline JSON file + --compare-baseline= Compare run results against an existing baseline JSON file + --help Show this help message +`); + + return 0; + } + + if (targetSubsystem !== 'i18n') { + // eslint-disable-next-line no-console + console.error(`Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n`); + + return 1; + } + + if (options.build) { + const buildModule = await import('./build.mts'); + await buildModule.default({ local: true }); + } + + // eslint-disable-next-line no-console + if (!checkBuildStatus(console)) { + return 1; + } + + const rawSaveBaseline = options.saveBaseline ?? options['save-baseline']; + const rawCompareBaseline = options.compareBaseline ?? options['compare-baseline']; + + const iterations = options.iterations !== undefined ? Number(options.iterations) : undefined; + const warmup = options.warmup !== undefined ? Number(options.warmup) : undefined; + const concurrency = options.concurrency !== undefined ? Number(options.concurrency) : undefined; + + if (iterations !== undefined && (!Number.isInteger(iterations) || iterations < 1)) { + // eslint-disable-next-line no-console + console.error('Error: --iterations must be a positive integer.'); + + return 1; + } + + if (warmup !== undefined && (!Number.isInteger(warmup) || warmup < 0)) { + // eslint-disable-next-line no-console + console.error('Error: --warmup must be a non-negative integer.'); + + return 1; + } + + if (concurrency !== undefined && (!Number.isInteger(concurrency) || concurrency < 1)) { + // eslint-disable-next-line no-console + console.error('Error: --concurrency must be a positive integer.'); + + return 1; + } + + const cliOptions: BenchmarkCliOptions = { + scenario: options.scenario, + iterations, + warmup, + concurrency, + json: Boolean(options.json), + inProcess: Boolean(options.inProcess ?? options['in-process']), + saveBaseline: rawSaveBaseline ? path.resolve(_cwd, String(rawSaveBaseline)) : undefined, + compareBaseline: rawCompareBaseline + ? path.resolve(_cwd, String(rawCompareBaseline)) + : undefined, + }; + + const { exitCode } = await runI18nBenchmarks(cliOptions); + + return exitCode; +} diff --git a/scripts/benchmarks/i18n/fixtures.mts b/scripts/benchmarks/i18n/fixtures.mts new file mode 100644 index 000000000000..3d53c3dbff5b --- /dev/null +++ b/scripts/benchmarks/i18n/fixtures.mts @@ -0,0 +1,165 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; +import type { ɵParsedTranslation } from '@angular/localize'; +import { transformSync } from 'esbuild'; +import { createRequire } from 'node:module'; + +import path from 'node:path'; + +import type { BuildOutputFile } from '../../../packages/angular/build/src/tools/esbuild/bundler-files.js'; +import type { LocaleInlineOptions } from '../../../packages/angular/build/src/tools/esbuild/i18n-inliner.js'; + +// Setup module paths to resolve dependencies from packages/angular/build +const requireFromBuild = createRequire( + path.resolve(import.meta.dirname, '../../../packages/angular/build/package.json'), +); + +const { BuildOutputFileType, createOutputFile } = requireFromBuild( + '../../../dist/@angular/build/src/tools/esbuild/bundler-files.js', +) as typeof import('../../../packages/angular/build/src/tools/esbuild/bundler-files.js'); + +const { calculateHash, initializeHash } = requireFromBuild( + '../../../dist/@angular/build/src/utils/hash.js', +) as typeof import('../../../packages/angular/build/src/utils/hash.js'); + +let isHashInitialized = false; + +export async function initializeFixtures(): Promise { + if (!isHashInitialized) { + await initializeHash(); + isHashInitialized = true; + } +} + +export function parsedTranslation( + parts: string[], + placeholderNames: string[] = [], + text?: string, +): ɵParsedTranslation { + return { + messageParts: Object.assign([...parts], { raw: [...parts] }), + placeholderNames, + text: text ?? parts.join(''), + }; +} + +export function generateTranslations( + locales: string[], + messageCount: number, +): LocaleInlineOptions[] { + return locales.map((locale) => { + const translation: Record = {}; + + for (let i = 0; i < messageCount; i++) { + const msgId = `msg_${i}`; + translation[msgId] = parsedTranslation( + [`[${locale}] Order #`, ` was confirmed for customer `, `. Thank you!`], + ['orderId', 'customerName'], + `[${locale}] Order #${i} was confirmed for customer Doe. Thank you!`, + ); + } + + const translationIntegrity = calculateHash(JSON.stringify(translation)); + + return { + locale, + translation, + translationIntegrity, + }; + }); +} + +export interface SyntheticBundleOptions { + filename: string; + targetByteSize: number; + messageCount: number; + withSourceMap?: boolean; + messageIdOffset?: number; +} + +export function generateSyntheticBundle(options: SyntheticBundleOptions): { + codeFile: BuildOutputFile; + mapFile?: BuildOutputFile; +} { + const { + filename, + targetByteSize, + messageCount, + withSourceMap = true, + messageIdOffset = 0, + } = options; + + const parts: string[] = [ + '// Synthetic test bundle generated for i18n-inliner benchmark\n', + 'export const BUNDLE_META = { generated: true, timestamp: Date.now() };\n', + ]; + + // Generate functions with $localize call sites + for (let i = 0; i < messageCount; i++) { + const msgId = `msg_${messageIdOffset + i}`; + parts.push( + `export function renderMessage_${i}(orderId, customerName) {\n`, + ` return $localize\`:@@${msgId}:Order #\${orderId}:orderId: ` + + `was confirmed for customer \${customerName}:customerName:. Thank you!\`;\n`, + `}\n`, + ); + } + + // Calculate current approximate size and pad with realistic JS functions if needed + let currentSize = parts.reduce((acc, str) => acc + str.length, 0); + let classIndex = 0; + + while (currentSize < targetByteSize) { + const filler = + `export class DataProcessor_${classIndex} {\n` + + ` constructor(id, options = {}) {\n` + + ` this.id = id;\n` + + ` this.options = Object.assign({ enabled: true, retries: 3 }, options);\n` + + ` this.history = [];\n` + + ` }\n` + + ` process(batch) {\n` + + ` if (!Array.isArray(batch)) return [];\n` + + ` const result = batch.map((item, idx) => ({\n` + + ` id: this.id + '_' + idx,\n` + + ` source: item,\n` + + ` timestamp: Date.now(),\n` + + ` active: true\n` + + ` }));\n` + + ` this.history.push(...result);\n` + + ` return result;\n` + + ` }\n` + + `}\n`; + + parts.push(filler); + currentSize += filler.length; + classIndex++; + } + + const rawCode = parts.join(''); + + let finalCode = rawCode; + let finalMap: string | undefined; + + if (withSourceMap) { + const result = transformSync(rawCode, { + sourcemap: true, + sourcefile: filename.replace(/\.js$/, '.ts'), + }); + finalCode = result.code; + finalMap = result.map; + } + + const codeFile = createOutputFile(filename, finalCode, BuildOutputFileType.Browser); + const mapFile = finalMap + ? createOutputFile(filename + '.map', finalMap, BuildOutputFileType.Browser) + : undefined; + + return { codeFile, mapFile }; +} diff --git a/scripts/benchmarks/i18n/harness.mts b/scripts/benchmarks/i18n/harness.mts new file mode 100644 index 000000000000..caf7f0753d24 --- /dev/null +++ b/scripts/benchmarks/i18n/harness.mts @@ -0,0 +1,155 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; + +export interface BenchmarkScenario { + name: string; + description: string; + inputSizeBytes: number; + localeCount: number; + run(iteration: number): Promise; + setup?(): Promise; + teardown?(): Promise; +} + +export interface ScenarioResult { + name: string; + description: string; + inputSizeBytes: number; + localeCount: number; + iterations: number; + warmup: number; + durationsMs: number[]; + minMs: number; + maxMs: number; + meanMs: number; + medianMs: number; + p95Ms: number; + stdDevMs: number; + throughputMBps: number; + peakRssBytes: number; + peakHeapBytes: number; + rssDeltaBytes: number; + heapUsedDeltaBytes: number; +} + +export interface BenchmarkRunOptions { + warmup?: number; + iterations?: number; +} + +function calculatePercentile(sortedValues: number[], percentile: number): number { + if (sortedValues.length === 0) { + return 0; + } + const index = (percentile / 100) * (sortedValues.length - 1); + const lower = Math.floor(index); + const upper = Math.ceil(index); + const weight = index - lower; + + return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; +} + +export async function runScenario( + scenario: BenchmarkScenario, + options: BenchmarkRunOptions = {}, +): Promise { + const warmup = options.warmup ?? 2; + const iterations = options.iterations ?? 5; + + try { + await scenario.setup?.(); + + // Warmup phase + for (let w = 0; w < warmup; w++) { + // Force GC if available between warmups + global.gc?.(); + await scenario.run(-(w + 1)); + } + + // Measurement phase + const durationsMs: number[] = []; + let peakRssBytes = 0; + let peakHeapBytes = 0; + let maxRssDeltaBytes = 0; + let initialHeap = 0; + let finalHeap = 0; + + for (let i = 0; i < iterations; i++) { + global.gc?.(); + + const memBefore = process.memoryUsage(); + if (i === 0) { + initialHeap = memBefore.heapUsed; + } + + const start = performance.now(); + await scenario.run(i); + const duration = performance.now() - start; + + durationsMs.push(duration); + + const memAfter = process.memoryUsage(); + if (memAfter.rss > peakRssBytes) { + peakRssBytes = memAfter.rss; + } + if (memAfter.heapUsed > peakHeapBytes) { + peakHeapBytes = memAfter.heapUsed; + } + const rssDelta = Math.max(0, memAfter.rss - memBefore.rss); + if (rssDelta > maxRssDeltaBytes) { + maxRssDeltaBytes = rssDelta; + } + + // Force GC immediately after iteration to clean main thread isolate + global.gc?.(); + finalHeap = process.memoryUsage().heapUsed; + } + + // Sort ascending for percentile computation + const sorted = [...durationsMs].sort((a, b) => a - b); + const minMs = sorted[0]; + const maxMs = sorted[sorted.length - 1]; + const meanMs = durationsMs.reduce((sum, d) => sum + d, 0) / durationsMs.length; + const medianMs = calculatePercentile(sorted, 50); + const p95Ms = calculatePercentile(sorted, 95); + + const variance = + durationsMs.reduce((sum, d) => sum + Math.pow(d - meanMs, 2), 0) / durationsMs.length; + const stdDevMs = Math.sqrt(variance); + + // Throughput: total effective processed code volume in MB / mean seconds + const totalProcessedMb = (scenario.inputSizeBytes * scenario.localeCount) / (1024 * 1024); + const meanSeconds = meanMs / 1000; + const throughputMBps = meanSeconds > 0 ? totalProcessedMb / meanSeconds : 0; + + return { + name: scenario.name, + description: scenario.description, + inputSizeBytes: scenario.inputSizeBytes, + localeCount: scenario.localeCount, + iterations, + warmup, + durationsMs, + minMs, + maxMs, + meanMs, + medianMs, + p95Ms, + stdDevMs, + throughputMBps, + peakRssBytes, + peakHeapBytes, + rssDeltaBytes: maxRssDeltaBytes, + heapUsedDeltaBytes: Math.max(0, finalHeap - initialHeap), + }; + } finally { + await scenario.teardown?.(); + } +} diff --git a/scripts/benchmarks/i18n/index.mts b/scripts/benchmarks/i18n/index.mts new file mode 100644 index 000000000000..276671c193e0 --- /dev/null +++ b/scripts/benchmarks/i18n/index.mts @@ -0,0 +1,167 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { type ScenarioResult, runScenario } from './harness.mts'; +import { + buildReportData, + formatComparisonTable, + formatConsoleTable, + formatJsonReport, +} from './reporters.mts'; +import { type ScenarioFactoryOptions, getAllScenarios, getScenarioByName } from './scenarios.mts'; + +export interface BenchmarkCliOptions extends ScenarioFactoryOptions { + scenario?: string; + iterations?: number; + warmup?: number; + verbose?: boolean; + json?: boolean; + inProcess?: boolean; + saveBaseline?: string; + compareBaseline?: string; +} + +export async function runI18nBenchmarks( + options: BenchmarkCliOptions = {}, +): Promise<{ results: ScenarioResult[]; exitCode: number }> { + const warmup = options.warmup ?? 2; + const iterations = options.iterations ?? 5; + + let scenariosToRun = getAllScenarios({ concurrency: options.concurrency }); + + if (options.scenario) { + const single = getScenarioByName(options.scenario, { concurrency: options.concurrency }); + if (!single) { + // eslint-disable-next-line no-console + console.error( + `Unknown scenario: "${options.scenario}".\nAvailable scenarios: ${scenariosToRun.map((s) => s.name).join(', ')}`, + ); + + return { results: [], exitCode: 1 }; + } + scenariosToRun = [single]; + } + + const results: ScenarioResult[] = []; + + // When running multiple scenarios in a suite, isolate each scenario into its own child process + // so operating system memory and thread caches are not accumulated across scenarios. + if (scenariosToRun.length > 1 && !options.inProcess) { + for (const scenario of scenariosToRun) { + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + `Running scenario: ${scenario.name} (${warmup} warmups, ${iterations} iterations)...`, + ); + } + + const args = [ + '--no-warnings=ExperimentalWarning', + '--experimental-transform-types', + '--expose-gc', + path.resolve(import.meta.dirname, '../../devkit-admin.mts'), + 'benchmark', + 'i18n', + `--scenario=${scenario.name}`, + `--warmup=${warmup}`, + `--iterations=${iterations}`, + '--json', + ]; + if (options.concurrency !== undefined) { + args.push(`--concurrency=${options.concurrency}`); + } + + const proc = spawnSync(process.execPath, args, { encoding: 'utf-8' }); + if (proc.status !== 0 || proc.error) { + // eslint-disable-next-line no-console + console.error( + `Error running scenario ${scenario.name}:\n` + + (proc.error?.message ?? (proc.stderr || proc.stdout)), + ); + + return { results, exitCode: 1 }; + } + + try { + const parsed = JSON.parse(proc.stdout); + const scenarioResult: ScenarioResult | undefined = Array.isArray(parsed) + ? parsed[0] + : parsed.results?.[0]; + if (scenarioResult) { + results.push(scenarioResult); + } + } catch { + // eslint-disable-next-line no-console + console.error(`Failed to parse result for scenario ${scenario.name}:\n${proc.stdout}`); + + return { results, exitCode: 1 }; + } + } + } else { + for (const scenario of scenariosToRun) { + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + `Running scenario: ${scenario.name} (${warmup} warmups, ${iterations} iterations)...`, + ); + } + + try { + const result = await runScenario(scenario, { warmup, iterations }); + results.push(result); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`Error running scenario ${scenario.name}:`, error); + + return { results, exitCode: 1 }; + } + } + } + + if (options.json) { + // eslint-disable-next-line no-console + console.log(formatJsonReport(results)); + } else { + // eslint-disable-next-line no-console + console.log('\n' + formatConsoleTable(results)); + } + + if (options.saveBaseline) { + const reportData = buildReportData(results); + await fs.writeFile(options.saveBaseline, JSON.stringify(reportData, null, 2), 'utf-8'); + if (!options.json) { + // eslint-disable-next-line no-console + console.log(`Saved baseline to: ${options.saveBaseline}`); + } + } + + if (options.compareBaseline) { + try { + const baselineContent = await fs.readFile(options.compareBaseline, 'utf-8'); + const baselineData = JSON.parse(baselineContent); + const baselineResults: ScenarioResult[] = Array.isArray(baselineData) + ? baselineData + : (baselineData.results ?? []); + + if (!options.json) { + // eslint-disable-next-line no-console + console.log(formatComparisonTable(results, baselineResults)); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(`Failed to load baseline from ${options.compareBaseline}:`, error); + } + } + + return { results, exitCode: 0 }; +} diff --git a/scripts/benchmarks/i18n/init-env.mts b/scripts/benchmarks/i18n/init-env.mts new file mode 100644 index 000000000000..a1690cf576c8 --- /dev/null +++ b/scripts/benchmarks/i18n/init-env.mts @@ -0,0 +1,26 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import Module from 'node:module'; +import path from 'node:path'; + +// Resolve dependencies from packages/angular/build/node_modules for runtime resolution in dist/ +const buildNodeModules = path.resolve( + import.meta.dirname, + '../../../packages/angular/build/node_modules', +); + +const currentPath = process.env.NODE_PATH ?? ''; +if (!currentPath.includes(buildNodeModules)) { + process.env.NODE_PATH = currentPath + ? `${buildNodeModules}${path.delimiter}${currentPath}` + : buildNodeModules; + // Initialize internal search paths for Node CommonJS loader + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (Module as any)._initPaths?.(); +} diff --git a/scripts/benchmarks/i18n/reporters.mts b/scripts/benchmarks/i18n/reporters.mts new file mode 100644 index 000000000000..d5153552a2b3 --- /dev/null +++ b/scripts/benchmarks/i18n/reporters.mts @@ -0,0 +1,159 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import os from 'node:os'; +import type { ScenarioResult } from './harness.mts'; + +export interface BenchmarkReportData { + timestamp: string; + system: { + nodeVersion: string; + platform: string; + arch: string; + cpus: number; + cpuModel: string; + totalMemoryMb: number; + }; + results: ScenarioResult[]; +} + +export function buildReportData(results: ScenarioResult[]): BenchmarkReportData { + const cpus = os.cpus(); + + return { + timestamp: new Date().toISOString(), + system: { + nodeVersion: process.version, + platform: os.platform(), + arch: os.arch(), + cpus: cpus.length, + cpuModel: cpus[0]?.model ?? 'unknown', + totalMemoryMb: Math.round(os.totalmem() / (1024 * 1024)), + }, + results, + }; +} + +export function formatJsonReport(results: ScenarioResult[]): string { + return JSON.stringify(buildReportData(results), null, 2); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) { + return `${bytes} B`; + } + if (bytes < 1024 * 1024) { + return `${(bytes / 1024).toFixed(1)} KB`; + } + + return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; +} + +function padRight(str: string, len: number): string { + return str.length >= len ? str : str + ' '.repeat(len - str.length); +} + +function padLeft(str: string, len: number): string { + return str.length >= len ? str : ' '.repeat(len - str.length) + str; +} + +export function formatConsoleTable(results: ScenarioResult[]): string { + const cpus = os.cpus(); + const gcStatus = typeof global.gc === 'function' ? 'active' : 'inactive (run with --expose-gc)'; + const header = + `================================================================================================================\n` + + `i18n Inliner Performance Benchmarks (Node ${process.version}, ${cpus.length} CPUs: ${cpus[0]?.model ?? ''} | GC: ${gcStatus})\n` + + `================================================================================================================\n`; + + const columns = [ + { name: 'Scenario', width: 25 }, + { name: 'Input Size', width: 11 }, + { name: 'Locales', width: 8 }, + { name: 'Mean Latency', width: 13 }, + { name: 'p50 / p95', width: 18 }, + { name: 'Throughput', width: 12 }, + { name: 'Peak Heap', width: 11 }, + { name: 'Peak RSS', width: 11 }, + ]; + + const colHeader = columns.map((col) => padRight(col.name, col.width)).join(' '); + const separator = columns.map((col) => '-'.repeat(col.width)).join(' '); + + const rows = results.map((r) => { + const inputFormatted = formatBytes(r.inputSizeBytes); + const meanFormatted = `${r.meanMs.toFixed(1)} ms`; + const p50p95Formatted = `${r.medianMs.toFixed(0)} ms / ${r.p95Ms.toFixed(0)} ms`; + const throughputFormatted = `${r.throughputMBps.toFixed(1)} MB/s`; + const peakHeapFormatted = formatBytes(r.peakHeapBytes ?? 0); + const peakRssFormatted = formatBytes(r.peakRssBytes); + + return [ + padRight(r.name, columns[0].width), + padLeft(inputFormatted, columns[1].width), + padLeft(r.localeCount.toString(), columns[2].width), + padLeft(meanFormatted, columns[3].width), + padLeft(p50p95Formatted, columns[4].width), + padLeft(throughputFormatted, columns[5].width), + padLeft(peakHeapFormatted, columns[6].width), + padLeft(peakRssFormatted, columns[7].width), + ].join(' '); + }); + + return `${header}\n${colHeader}\n${separator}\n${rows.join('\n')}\n${'='.repeat(separator.length)}\n`; +} + +export function formatComparisonTable( + currentResults: ScenarioResult[], + baselineResults: ScenarioResult[], +): string { + const header = + `====================================================================================================\n` + + `i18n Inliner Benchmark Comparison (Current vs Baseline)\n` + + `====================================================================================================\n`; + + const columns = [ + { name: 'Scenario', width: 26 }, + { name: 'Baseline Mean', width: 14 }, + { name: 'Current Mean', width: 14 }, + { name: 'Latency Diff', width: 14 }, + { name: 'Baseline MB/s', width: 14 }, + { name: 'Current MB/s', width: 14 }, + ]; + + const colHeader = columns.map((col) => padRight(col.name, col.width)).join(' '); + const separator = columns.map((col) => '-'.repeat(col.width)).join(' '); + + const rows = currentResults.map((curr) => { + const base = baselineResults.find((b) => b.name === curr.name); + if (!base) { + return [ + padRight(curr.name, columns[0].width), + padLeft('N/A', columns[1].width), + padLeft(`${curr.meanMs.toFixed(1)} ms`, columns[2].width), + padLeft('NEW', columns[3].width), + padLeft('N/A', columns[4].width), + padLeft(`${curr.throughputMBps.toFixed(1)} MB/s`, columns[5].width), + ].join(' '); + } + + const diffPercent = ((curr.meanMs - base.meanMs) / base.meanMs) * 100; + const diffSign = diffPercent > 0 ? '+' : ''; + const diffText = `${diffSign}${diffPercent.toFixed(1)}% ${diffPercent > 1 ? '(slower)' : diffPercent < -1 ? '(faster)' : '(same)'}`; + + return [ + padRight(curr.name, columns[0].width), + padLeft(`${base.meanMs.toFixed(1)} ms`, columns[1].width), + padLeft(`${curr.meanMs.toFixed(1)} ms`, columns[2].width), + padLeft(diffText, columns[3].width), + padLeft(`${base.throughputMBps.toFixed(1)} MB/s`, columns[4].width), + padLeft(`${curr.throughputMBps.toFixed(1)} MB/s`, columns[5].width), + ].join(' '); + }); + + return `${header}\n${colHeader}\n${separator}\n${rows.join('\n')}\n${'='.repeat(separator.length)}\n`; +} diff --git a/scripts/benchmarks/i18n/scenarios.mts b/scripts/benchmarks/i18n/scenarios.mts new file mode 100644 index 000000000000..c00f34ded8a2 --- /dev/null +++ b/scripts/benchmarks/i18n/scenarios.mts @@ -0,0 +1,476 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import './init-env.mts'; + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import type { BuildOutputFile } from '../../../packages/angular/build/src/tools/esbuild/bundler-files.js'; +import type { LocaleInlineOptions } from '../../../packages/angular/build/src/tools/esbuild/i18n-inliner.js'; +import { generateSyntheticBundle, generateTranslations, initializeFixtures } from './fixtures.mts'; +import type { BenchmarkScenario } from './harness.mts'; + +const requireFromBuild = createRequire( + path.resolve(import.meta.dirname, '../../../packages/angular/build/package.json'), +); + +const { I18nInliner } = requireFromBuild( + '../../../dist/@angular/build/src/tools/esbuild/i18n-inliner.js', +) as typeof import('../../../packages/angular/build/src/tools/esbuild/i18n-inliner.js'); + +export interface ScenarioFactoryOptions { + concurrency?: number; +} + +const DEFAULT_LOCALES_8 = ['fr', 'de', 'es', 'ja', 'zh', 'it', 'pt', 'ko']; +const DEFAULT_LOCALES_32 = [ + 'fr', + 'de', + 'es', + 'ja', + 'zh', + 'it', + 'pt', + 'ko', + 'ru', + 'pl', + 'nl', + 'tr', + 'ar', + 'hi', + 'sv', + 'da', + 'fi', + 'no', + 'cs', + 'el', + 'he', + 'hu', + 'id', + 'ms', + 'ro', + 'sk', + 'th', + 'uk', + 'vi', + 'bg', + 'hr', + 'sr', +]; + +interface GeneratedWorkload { + files: BuildOutputFile[]; + locales: LocaleInlineOptions[]; + totalInputSizeBytes: number; +} + +function createBundleSet( + mainSizeBytes: number, + chunkCount: number, + chunkSizeBytes: number, + messageCount: number, + withSourceMap: boolean, +): BuildOutputFile[] { + const files: BuildOutputFile[] = []; + + const mainMessages = Math.floor(messageCount * 0.4); + const { codeFile: mainCode, mapFile: mainMap } = generateSyntheticBundle({ + filename: 'main.js', + targetByteSize: mainSizeBytes, + messageCount: mainMessages, + withSourceMap, + messageIdOffset: 0, + }); + + files.push(mainCode); + if (mainMap) { + files.push(mainMap); + } + + const remainingMessages = messageCount - mainMessages; + const messagesPerChunk = Math.max(1, Math.floor(remainingMessages / chunkCount)); + + for (let i = 0; i < chunkCount; i++) { + const { codeFile, mapFile } = generateSyntheticBundle({ + filename: `chunk_${i}.js`, + targetByteSize: chunkSizeBytes, + messageCount: messagesPerChunk, + withSourceMap, + messageIdOffset: mainMessages + i * messagesPerChunk, + }); + + files.push(codeFile); + if (mapFile) { + files.push(mapFile); + } + } + + return files; +} + +function calculateInputSizeBytes(files: BuildOutputFile[]): number { + return files.reduce((total, f) => { + // Only count JS code size towards raw input volume (maps are auxiliary) + return f.path.endsWith('.js') ? total + f.size : total; + }, 0); +} + +/** + * Note on Worker Pool Lifecycle: + * Each scenario's run() method instantiates and closes an I18nInliner per iteration. + * This is designed as a macro benchmark to reflect the cold-start behavior of single-shot + * CLI build invocations (including worker thread pool initialization, task dispatch, + * inlining transformations, and thread pool shutdown). Warmup iterations warm up the + * main-thread V8 isolate and runtime paths, while worker threads are initialized per iteration. + */ + +/** + * 1. Standard App Scenario: + * 1 main bundle (1 MB) + 20 route chunks (50 KB) = ~2 MB input JS, 8 locales, maps enabled. + */ +export function createStandardAppScenario(options: ScenarioFactoryOptions = {}): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'standard-app', + description: 'Standard App: 1 main bundle (1 MB) + 20 chunks (50 KB), 8 locales, sourcemaps ON', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(1024 * 1024, 20, 50 * 1024, 1000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * 2. Standard App without Sourcemaps: + * Evaluates inlining without source map generation and remapping. + */ +export function createStandardAppNoMapsScenario( + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'standard-app-no-maps', + description: + 'Standard App (No Maps): 1 main bundle (1 MB) + 20 chunks (50 KB), 8 locales, sourcemaps OFF', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(1024 * 1024, 20, 50 * 1024, 1000, false); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * 3. Enterprise Multilingual Scenario: + * 1 main bundle (2 MB) + 40 chunks (60 KB), 32 locales, 3,000 translations, sourcemaps ON. + * Exercises sliding window batching (4 windows of 8 locales) and memory scaling. + */ +export function createEnterpriseScenario(options: ScenarioFactoryOptions = {}): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'enterprise-multilingual', + description: 'Enterprise: 1 main (2 MB) + 40 chunks (60 KB), 32 locales, sourcemaps ON', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_32.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(2 * 1024 * 1024, 40, 60 * 1024, 3000, true); + const locales = generateTranslations(DEFAULT_LOCALES_32, 3000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * 4. Monolithic Dominant Scenario: + * 1 dominant bundle (6 MB) + 5 small runtime chunks (30 KB), 8 locales, sourcemaps ON. + * Evaluates whether LPT + DOMINANT_FILE_RATIO sharding saturates workers efficiently. + */ +export function createMonolithicScenario(options: ScenarioFactoryOptions = {}): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'monolithic-dominant', + description: 'Monolithic: 1 dominant bundle (6 MB) + 5 small chunks (30 KB), 8 locales', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(6 * 1024 * 1024, 5, 30 * 1024, 2000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 2000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +/** + * Helper to prime the persistent cache in an isolated process. + */ +export async function primeCache(cacheDir: string, concurrency?: number): Promise { + await initializeFixtures(); + const files = createBundleSet(1024 * 1024, 15, 50 * 1024, 1000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + const primer = new I18nInliner({ + missingTranslation: 'warning', + persistentCachePath: cacheDir, + maxConcurrency: concurrency, + }); + await primer.inlineAll(files, locales); + await primer.close(); +} + +/** + * 5. Persistent Cache Warm Scenario: + * Evaluates throughput when 100% of transformed files and translations are pre-cached in LMDB. + */ +export function createPersistentCacheWarmScenario( + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + let cacheDir: string | undefined; + + return { + name: 'persistent-cache-warm', + description: + 'Persistent Cache (Warm): 100% LMDB cache hits for transformed files & translations', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_8.length; + }, + async setup() { + await initializeFixtures(); + cacheDir = await fs.mkdtemp(path.join(os.tmpdir(), 'angular-i18n-bench-cache-')); + const files = createBundleSet(1024 * 1024, 15, 50 * 1024, 1000, true); + const locales = generateTranslations(DEFAULT_LOCALES_8, 1000); + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + + // Prime the persistent cache out-of-process so cold worker thread allocations + // do not inflate this process's RSS metrics. + const scenariosUrl = pathToFileURL(path.resolve(import.meta.dirname, './scenarios.mts')).href; + const primerCode = + `import { primeCache } from ${JSON.stringify(scenariosUrl)};\n` + + `await primeCache(${JSON.stringify(cacheDir)}, ${options.concurrency ?? 'undefined'});\n`; + + const primerProc = spawnSync( + process.execPath, + [ + '--no-warnings=ExperimentalWarning', + '--experimental-transform-types', + '--input-type=module', + '-e', + primerCode, + ], + { stdio: 'inherit' }, + ); + + if (primerProc.status !== 0 || primerProc.error) { + throw new Error( + `Failed to prime cache for persistent-cache-warm scenario: ${primerProc.error?.message ?? primerProc.status}`, + ); + } + }, + + async run() { + if (!workload || !cacheDir) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + persistentCachePath: cacheDir, + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + async teardown() { + if (cacheDir) { + await fs.rm(cacheDir, { recursive: true, force: true }).catch(() => {}); + } + }, + }; +} + +/** + * 6. Large Enterprise (10k translations) Scenario: + * 1 main bundle (3 MB) + 100 chunks (50 KB), 32 locales, 10,000 translations, sourcemaps ON. + * Maximum scale stress test for binary translation tables, memory retention, and multi-locale windows. + */ +export function createLargeEnterpriseScenario( + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario { + let workload: GeneratedWorkload | undefined; + + return { + name: 'large-enterprise-10k', + description: + 'Large Enterprise (10k msgs): 1 main (3 MB) + 100 chunks (50 KB), 32 locales, 10,000 translations', + get inputSizeBytes() { + return workload?.totalInputSizeBytes ?? 0; + }, + get localeCount() { + return DEFAULT_LOCALES_32.length; + }, + async setup() { + await initializeFixtures(); + const files = createBundleSet(3 * 1024 * 1024, 100, 50 * 1024, 10000, true); + const locales = generateTranslations(DEFAULT_LOCALES_32, 10000); + + workload = { + files, + locales, + totalInputSizeBytes: calculateInputSizeBytes(files), + }; + }, + async run() { + if (!workload) { + return; + } + const inliner = new I18nInliner({ + missingTranslation: 'warning', + maxConcurrency: options.concurrency, + }); + try { + await inliner.inlineAll(workload.files, workload.locales); + } finally { + await inliner.close(); + } + }, + }; +} + +export function getAllScenarios(options: ScenarioFactoryOptions = {}): BenchmarkScenario[] { + return [ + createStandardAppScenario(options), + createStandardAppNoMapsScenario(options), + createEnterpriseScenario(options), + createLargeEnterpriseScenario(options), + createMonolithicScenario(options), + createPersistentCacheWarmScenario(options), + ]; +} + +export function getScenarioByName( + name: string, + options: ScenarioFactoryOptions = {}, +): BenchmarkScenario | undefined { + const all = getAllScenarios(options); + + return all.find((s) => s.name.toLowerCase() === name.toLowerCase()); +}