+ Shared with {count} people +
Loading projects
Use {'{}'} & it's done…
${body}
{LABEL} for {name}
No layers yet
{i18n._({ id: 'editor.sidebar.empty.noLayersYet', message: 'No layers yet' })}
{LABEL} now
+ {/* i18n-ignore: product name */} + Made with Three.js +
+ {/* i18n-ignore: library name */} + +
') + expect(() => + scanSource('packages/editor/src/components/ui/sidebar/empty.tsx', output, allowlist), + ).not.toThrow() + expect(convert('packages/editor/src/components/ui/sidebar/empty.tsx', output).output).toBe(output) +}) + +test('the audit allowlist removes whole words only', () => { + expect(untranslatedWords('Find · 2 in', ['in'])).toEqual(['Find']) + expect(untranslatedWords('Pin', ['in'])).toEqual(['Pin']) + expect(untranslatedWords('[Ƒîñð] 2 in', ['in'])).toEqual([]) +}) + +test('pseudo-localization accents every letter and keeps ICU structure', () => { + const message = 'Delete {count, plural, one {# wall} other {# walls}} from <0>{level}?' + const pseudo = pseudoLocalize(message) + + expect(pseudo).toBe( + '[Ðéļéţé {count, plural, one {# ŵáļļ} other {# ŵáļļš}} ƒŕöɱ <0>{level}? ~~~~~~]', + ) + expect(() => compileMessageOrThrow(pseudo)).not.toThrow() + expect(pseudoLocalize("It''s '{'literal'}'")).toBe("[Îţ''š '{'ļîţéŕáļ'}' ~~~]") +}) + +const advanced: SourceMessage = { + id: 'editor.buildPanel.advanced', + message: 'Advanced', + context: [], +} +const entry = (overrides: Partial) => ({ + translation: 'Avancé', + sourceHash: hashSource('Advanced'), + review: 'human' as const, + fuzzy: false, + ...overrides, +}) +const everyLocale = (value: ReturnType): LocaleEntries => + Object.fromEntries( + ['zh-CN', 'fr', 'es', 'ja'].map((locale) => [locale, { [advanced.id]: value }]), + ) + +test('AI drafts pass ordinary checks but block the reviewed release gate', () => { + const drafts = everyLocale(entry({ review: 'required', fuzzy: true })) + expect(checkTranslationEntries([advanced], drafts, {}, {})).toEqual([]) + expect(checkTranslationEntries([advanced], drafts, {}, { requireReviewed: true })).toContain( + 'fr: "editor.buildPanel.advanced" still requires human review', + ) + expect( + checkTranslationEntries([advanced], everyLocale(entry({ review: 'missing' })), {}, {}), + ).toContain( + 'fr: "editor.buildPanel.advanced" is unreviewed but not marked fuzzy + Review: required', + ) +}) + +test('after a wording change, extract leaves a passing draft and a reviewed stale entry fails', () => { + const reworded = { ...advanced, message: 'Advanced tools' } + const staleDraft = everyLocale(entry({ review: 'required', fuzzy: true })) + expect(checkTranslationEntries([reworded], staleDraft, {}, {})).toEqual([]) + expect(checkTranslationEntries([reworded], staleDraft, {}, { requireReviewed: true })).toContain( + 'fr: stale "editor.buildPanel.advanced"', + ) + expect(checkTranslationEntries([reworded], everyLocale(entry({})), {}, {})).toContain( + 'fr: stale "editor.buildPanel.advanced"', + ) +}) + +test('a reviewed translation cannot fall back to draft while its English is unchanged', () => { + const base = everyLocale(entry({})) + expect( + reviewRegressions([advanced], everyLocale(entry({ review: 'required', fuzzy: true })), base), + ).toContain( + 'fr: "editor.buildPanel.advanced" is reviewed on the base branch with the same English and must stay reviewed', + ) + expect( + reviewRegressions([advanced], everyLocale(entry({ translation: 'Avancés' })), base), + ).toEqual([]) + const reworded = { ...advanced, message: 'Advanced tools' } + expect( + reviewRegressions([reworded], everyLocale(entry({ review: 'required', fuzzy: true })), base), + ).toEqual([]) +}) + +test('an unrelated addition sharing English with a removed reviewed ID only warns', () => { + const off = { id: 'viewer.common.off', message: 'Off', context: [] } + const added = { id: 'viewer.audio.off', message: 'Off', context: [] } + const reviewed = (translation: string) => ({ + translation, + sourceHash: hashSource('Off'), + review: 'human' as const, + fuzzy: false, + }) + const base: LocaleEntries = Object.fromEntries( + ['zh-CN', 'fr', 'es', 'ja'].map((locale) => [locale, { [off.id]: reviewed('Désactivé') }]), + ) + const current: LocaleEntries = Object.fromEntries( + ['zh-CN', 'fr', 'es', 'ja'].map((locale) => [ + locale, + { [added.id]: { ...reviewed('Désactivé'), review: 'required' as const, fuzzy: true } }, + ]), + ) + expect(reviewRegressions([added], current, base)).toEqual([]) + expect(possibleRenames([added], current, base)).toEqual([ + `possible rename "viewer.common.off" → "viewer.audio.off" (same English): carry over the translation if intended with comment: 'renamed-from: ' and bun run i18n:extract`, + ]) +}) + +test('a declared rename must carry the reviewed translation over', () => { + const renamed = { ...advanced, id: 'editor.buildPanel.advancedTools' } + const base = everyLocale(entry({})) + const declared = (overrides: Partial): LocaleEntries => + Object.fromEntries( + ['zh-CN', 'fr', 'es', 'ja'].map((locale) => [ + locale, + { [renamed.id]: entry({ renamedFrom: advanced.id, ...overrides }) }, + ]), + ) + expect( + reviewRegressions([renamed], declared({ review: 'required', fuzzy: true }), base), + ).toContain( + 'fr: "editor.buildPanel.advancedTools" is declared renamed-from "editor.buildPanel.advanced", whose reviewed translation was not carried over; run bun run i18n:extract', + ) + expect(reviewRegressions([renamed], declared({}), base)).toEqual([]) + expect(possibleRenames([renamed], declared({ review: 'required', fuzzy: true }), base)).toEqual( + [], + ) +}) + +test('an explicit base that cannot be resolved fails the release job', () => { + const result = Bun.spawnSync( + ['bun', 'scripts/i18n.ts', 'release-check', '--base', 'nonexistent-ref'], + { env: { ...process.env, GITHUB_STEP_SUMMARY: '' }, stdout: 'pipe', stderr: 'pipe' }, + ) + expect(result.exitCode).toBe(1) + expect(`${result.stdout}${result.stderr}`).toContain('--base nonexistent-ref is not a commit') +}) + +test('extraction never approves a newly filled translation by itself', () => { + const hash = hashSource('Advanced') + const pasted = { translation: 'Avancé', review: 'missing' as const, fuzzy: false } + expect(annotatedState(pasted, 'Avancé', hash)).toEqual({ + sourceHash: hash, + review: 'required', + fuzzy: true, + }) + expect(annotatedState({ ...pasted, review: 'human' }, 'Avancé', hash)).toEqual({ + sourceHash: hash, + review: 'human', + fuzzy: false, + }) + expect(annotatedState({ ...pasted, review: 'human', sourceHash: hash }, 'Avancé', hash)).toBe( + undefined, + ) +}) + +const LOCALES_DIR = 'packages/editor/src/i18n/locales' + +/** A copy of the real catalogs to run the release job against, edited per case. */ +function catalogFixture(edit: (root: string) => void): { current: string; base: string } { + const root = mkdtempSync(join(tmpdir(), 'pascal-i18n-')) + const base = join(root, 'base') + const current = join(root, 'current') + cpSync(LOCALES_DIR, base, { recursive: true }) + cpSync(LOCALES_DIR, current, { recursive: true }) + edit(current) + return { current, base } +} + +function releaseCheckJob(fixture: { current: string; base: string }) { + const result = Bun.spawnSync( + ['bun', 'scripts/i18n.ts', 'release-check', '--base-dir', fixture.base], + { + env: { ...process.env, I18N_CATALOG_ROOT: fixture.current, GITHUB_STEP_SUMMARY: '' }, + stdout: 'pipe', + stderr: 'pipe', + }, + ) + return { exitCode: result.exitCode, output: `${result.stdout}${result.stderr}` } +} + +function markDraft(root: string, id: string): void { + const path = join(root, 'fr/messages.po') + const po = parsePo(readFileSync(path, 'utf8')) + const item = po.items.find((entry) => entry.msgid === id) + if (!item) throw new Error(`fixture entry ${id} moved`) + item.comments = item.comments.map((comment) => + comment.startsWith('Review: ') ? 'Review: required' : comment, + ) + item.flags.fuzzy = true + writeFileSync(path, stringifyPo(po, { foldLength: 0 })) +} + +test('the release job passes on unchanged catalogs', () => { + expect(releaseCheckJob(catalogFixture(() => {})).exitCode).toBe(0) +}) + +test('the release job fails when a reviewed entry becomes a draft with the same English', () => { + const { exitCode, output } = releaseCheckJob( + catalogFixture((root) => markDraft(root, 'editor.buildPanel.advanced')), + ) + expect(exitCode).toBe(1) + expect(output).toContain( + 'fr: "editor.buildPanel.advanced" is reviewed on the base branch with the same English and must stay reviewed', + ) +}) + +test('the release job fails when a compiled pack ships a draft', () => { + const { exitCode, output } = releaseCheckJob( + catalogFixture((root) => { + const path = join(root, 'fr/messages.ts') + const pack = readFileSync(path, 'utf8') + const draftId = parsePo(readFileSync(join(root, 'fr/messages.po'), 'utf8')).items.find( + (item) => item.flags.fuzzy, + )?.msgid + if (!draftId) throw new Error('fixture has no draft') + writeFileSync( + path, + pack.replace('JSON.parse("{', `JSON.parse("{\\"${draftId}\\":[\\"Brouillon\\"],`), + ) + }), + ) + expect(exitCode).toBe(1) + expect(output).toMatch(/fr: compiled pack ships unreviewed "[^"]+"/) +}) + +/** A throwaway repository with the real catalogs, for the default base lookup. */ +function releaseRepo(steps: (git: (...args: string[]) => void, root: string) => void): string { + const root = mkdtempSync(join(tmpdir(), 'pascal-release-')) + const git = (...args: string[]) => { + const result = Bun.spawnSync( + ['git', '-c', 'user.name=Test', '-c', 'user.email=test@example.com', ...args], + { cwd: root, stdout: 'ignore', stderr: 'pipe' }, + ) + if (result.exitCode !== 0) throw new Error(`git ${args.join(' ')}: ${result.stderr}`) + } + git('init', '-q', '-b', 'main') + steps(git, root) + return root +} + +const addCatalogs = (root: string) => + cpSync(LOCALES_DIR, join(root, LOCALES_DIR), { recursive: true }) + +function defaultReleaseCheck(root: string) { + const result = Bun.spawnSync(['bun', 'scripts/i18n.ts', 'release-check'], { + env: { ...process.env, I18N_ROOT: root, GITHUB_STEP_SUMMARY: '' }, + stdout: 'pipe', + stderr: 'pipe', + }) + return { exitCode: result.exitCode, output: `${result.stdout}${result.stderr}` } +} + +test('the release job compares with the latest editor release tag and fails on a reviewed-to-draft change', () => { + const root = releaseRepo((git, root) => { + addCatalogs(root) + git('add', '.') + git('commit', '-q', '-m', 'release') + git('tag', '@pascal-app/editor@9.9.9') + git('tag', 'v0.0.1') + markDraft(join(root, LOCALES_DIR), 'editor.buildPanel.advanced') + }) + const { exitCode, output } = defaultReleaseCheck(root) + expect(output).toContain('Comparing with @pascal-app/editor@9.9.9') + expect(exitCode).toBe(1) + expect(output).toContain( + 'fr: "editor.buildPanel.advanced" is reviewed on the base branch with the same English and must stay reviewed', + ) +}) + +test('a release tag without catalogs falls back to the commit that introduced them', () => { + const root = releaseRepo((git, root) => { + writeFileSync(join(root, 'README.md'), 'before i18n\n') + git('add', '.') + git('commit', '-q', '-m', 'old release') + git('tag', '@pascal-app/editor@1.0.0') + addCatalogs(root) + git('add', '.') + git('commit', '-q', '-m', 'catalogs') + writeFileSync(join(root, 'README.md'), 'after i18n\n') + git('add', '.') + git('commit', '-q', '-m', 'later work') + markDraft(join(root, LOCALES_DIR), 'editor.buildPanel.advanced') + }) + const { exitCode, output } = defaultReleaseCheck(root) + expect(output).toContain( + 'the commit that introduced the catalogs (@pascal-app/editor@1.0.0 has no catalogs)', + ) + expect(exitCode).toBe(1) + expect(output).toContain('fr: "editor.buildPanel.advanced" is reviewed on the base branch') +}) + +test('with no base that has catalogs the release job fails loudly', () => { + const root = releaseRepo((git, root) => { + addCatalogs(root) + git('add', '.') + git('commit', '-q', '-m', 'catalogs at HEAD only') + }) + const { exitCode, output } = defaultReleaseCheck(root) + expect(exitCode).toBe(1) + expect(output).toContain('release-check: no base with bundled PO catalogs') +}) diff --git a/scripts/i18n-literals.ts b/scripts/i18n-literals.ts new file mode 100644 index 0000000000..96531c5c8a --- /dev/null +++ b/scripts/i18n-literals.ts @@ -0,0 +1,931 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { decodeHTMLStrict } from 'entities' +import ts from 'typescript' +import { baseRevision, readAtRevision } from './i18n-git' + +/** + * User-visible literal scanner shared by the one-time codemod (`convert`) and + * the CI ratchet (`check`). Both see exactly the same candidates, so a file + * the codemod finished is a file the guard keeps finished. + */ + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const BASELINE_PATH = join(ROOT, 'i18n/literal-baseline.json') +const ALLOWLIST_PATH = join(ROOT, 'i18n/literal-allowlist.json') + +export const SCANNED_GLOBS = [ + 'apps/editor/**/*.tsx', + 'packages/editor/src/**/*.tsx', + 'packages/viewer/src/**/*.tsx', + 'packages/nodes/src/**/*.tsx', + 'packages/core/src/**/*.tsx', + 'packages/ui/src/**/*.tsx', + // Plain modules only contribute notification payloads (toast/alert/confirm). + 'apps/editor/**/*.ts', + 'packages/editor/src/**/*.ts', +] + +const SKIPPED_PATH = + /(?:\.test|\.spec|\.stories|\.bench|\.d)\.tsx?$|\/(?:__fixtures__|__tests__|__bench__|test-utils|node_modules|vendor)\// + +/** JSX props whose string value is read by people or assistive technology. */ +const UI_ATTRIBUTES = new Set([ + 'alt', + 'aria-description', + 'aria-label', + 'aria-placeholder', + 'aria-roledescription', + 'aria-valuetext', + 'cancelLabel', + 'caption', + 'confirmLabel', + 'description', + 'emptyLabel', + 'emptyMessage', + 'emptyText', + 'heading', + 'helperText', + 'hint', + 'label', + 'placeholder', + 'subtitle', + 'title', + 'tooltip', +]) + +/** Object keys that carry display copy when built inside a component. */ +const UI_PROPERTIES = new Set([ + 'ariaLabel', + 'cancelLabel', + 'confirmLabel', + 'description', + 'detail', + 'emptyMessage', + 'heading', + 'hint', + 'label', + 'placeholder', + 'subtitle', + 'title', + 'tooltip', +]) + +const NOTIFY_CALLEES = new Set([ + 'alert', + 'confirm', + 'prompt', + 'setError', + 'setErrorMessage', + 'setNotice', + 'setStatusMessage', + 'toast', +]) +const NOTIFY_METHODS = new Set(['success', 'error', 'info', 'warning', 'message', 'loading']) + +export type CandidateKind = + | 'jsx-text' + | 'jsx-sentence' + | 'attribute' + | 'expression' + | 'property' + | 'notify' + +export type Candidate = { + kind: CandidateKind + /** English copy as rendered (JSX whitespace collapsed, placeholders as `{name}`). */ + text: string + line: number + /** Enclosing React component or hook that can own `usePascalI18n()`. */ + component?: string + /** Why the codemod leaves this one for a person. */ + manual?: string +} + +export type Allowlist = { + /** Exact strings that are not translatable copy anywhere (brands, units, symbols). */ + texts: string[] + /** Path prefixes whose literals are not product UI, each with a reason. */ + paths: Record +} + +/** Legacy candidates per file, by text (a multiset: repeated copy appears once per use). */ +export type Baseline = { version: 3; files: Record } + +function readAllowlist(): Allowlist { + return JSON.parse(readFileSync(ALLOWLIST_PATH, 'utf8')) as Allowlist +} + +/** Mirrors React's JSX text whitespace rules (`jsx-runtime` collapses per line). */ +export function normalizeJsxText(raw: string): string { + const lines = raw.split(/\r\n|\n|\r/) + if (lines.length === 1) return raw + const parts: string[] = [] + lines.forEach((line, index) => { + let value = line.replace(/\t/g, ' ') + if (index > 0) value = value.replace(/^ +/, '') + if (index < lines.length - 1) value = value.replace(/ +$/, '') + if (value) parts.push(value) + }) + return parts.join(' ') +} + +/** Leading/trailing ASCII whitespace; a no-break space (` `) is content. */ +export const ASCII_EDGE_WHITESPACE = /^[ \t\r\n]+|[ \t\r\n]+$/g + +/** What one JSX text node renders: React's whitespace rules, then HTML entities. */ +export function renderJsxText(raw: string): string { + return decodeHTMLStrict(normalizeJsxText(raw)) +} + +/** A JSX attribute's string renders with HTML entities decoded; other literals are as written. */ +export function attributeText(node: ts.StringLiteral | ts.NoSubstitutionTemplateLiteral): string { + return ts.isJsxAttribute(node.parent) ? decodeHTMLStrict(node.text) : node.text +} + +function looksTranslatable(text: string, allow: Set): boolean { + const trimmed = text.trim() + if (!/\p{L}{2}/u.test(trimmed)) return false + if (allow.has(trimmed)) return false + // Icon names, snake_case keys, paths and URLs. Hyphenated words ("Read-only", "zoom-in") are copy. + if (/^[a-z0-9-]+(?:[_:./#][a-z0-9-]+)+$/i.test(trimmed)) return false + if (/^(?:https?:|mailto:|\/|#|\.)/.test(trimmed)) return false + if (/^[a-z][a-zA-Z0-9]*$/.test(trimmed) && /[A-Z]/.test(trimmed)) return false + return true +} + +function lineOf(source: ts.SourceFile, node: ts.Node): number { + return source.getLineAndCharacterOfPosition(node.getStart(source)).line + 1 +} + +const IGNORE = /i18n-ignore:\s*\S/ +// A line holding only a comment: a line comment, a block comment, or a JSX {/* comment */}. +const COMMENT_ONLY_LINE = /^\s*(?:\/\/|\/\*|\{\s*\/\*)/ + +/** + * `i18n-ignore: ` exempts a literal on its own line, or on the line + * after a comment-only line. A trailing ignore that exempts one line's literal + * does not reach the next line. + */ +function isIgnored(source: ts.SourceFile, node: ts.Node): boolean { + const text = source.text + const start = node.getStart(source) + const lineStart = text.lastIndexOf('\n', start - 1) + 1 + const lineEnd = text.indexOf('\n', start) === -1 ? text.length : text.indexOf('\n', start) + if (IGNORE.test(text.slice(lineStart, lineEnd))) return true + if (lineStart === 0) return false + const previousLineStart = text.lastIndexOf('\n', lineStart - 2) + 1 + const previousLine = text.slice(previousLineStart, lineStart - 1) + return COMMENT_ONLY_LINE.test(previousLine) && IGNORE.test(previousLine) +} + +function isTranslationCall(node: ts.Node): boolean { + return ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + node.expression.name.text === '_' + ) +} + +function insideTranslationCall(node: ts.Node): boolean { + for (let current = node.parent; current; current = current.parent) { + if (isTranslationCall(current)) return true + } + return false +} + +export function functionName(node: ts.Node): string | undefined { + if (ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node)) { + if (node.name) return node.name.text + } + if (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) { + let parent = node.parent + // memo(...) / forwardRef(...) wrappers + while (parent && ts.isCallExpression(parent)) parent = parent.parent + if (parent && ts.isVariableDeclaration(parent) && ts.isIdentifier(parent.name)) { + return parent.name.text + } + } + return undefined +} + +export function isComponentName(name: string | undefined): boolean { + return Boolean(name && (/^[A-Z]/.test(name) || /^use[A-Z]/.test(name))) +} + +const COMPONENT_WRAPPERS = new Set(['memo', 'forwardRef']) + +/** A function passed as an argument, other than to `memo` / `forwardRef`. */ +function isCallbackArgument(node: ts.Node): boolean { + let argument: ts.Node = node + while (argument.parent && ts.isParenthesizedExpression(argument.parent)) + argument = argument.parent + const call = argument.parent + if (!(call && ts.isCallExpression(call) && call.arguments.includes(argument as ts.Expression))) { + return false + } + const callee = call.expression + const name = ts.isIdentifier(callee) + ? callee.text + : ts.isPropertyAccessExpression(callee) + ? callee.name.text + : undefined + return !(name && COMPONENT_WRAPPERS.has(name)) +} + +function isAsync(node: ts.FunctionLikeDeclaration): boolean { + return Boolean( + ts.canHaveModifiers(node) && + ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword), + ) +} + +/** + * The nearest enclosing function named like a component or hook, and why it + * cannot own a hook: an async function, or a named callback such as + * `.map(function Row() …)`, is not rendered by React as a component. + */ +export function componentOwner(node: ts.Node): { + owner?: ts.FunctionLikeDeclaration + problem?: string +} { + for (let current = node.parent; current; current = current.parent) { + if ( + (ts.isFunctionDeclaration(current) || + ts.isArrowFunction(current) || + ts.isFunctionExpression(current)) && + isComponentName(functionName(current)) + ) { + if (isAsync(current)) return { problem: 'inside an async function: hooks cannot run there' } + if (isCallbackArgument(current)) { + return { problem: 'inside a callback function: hooks cannot run there' } + } + return { owner: current } + } + } + return { problem: 'outside a component or hook' } +} + +/** The nearest enclosing component or hook that can own `usePascalI18n()`. */ +export function enclosingComponent(node: ts.Node): ts.FunctionLikeDeclaration | undefined { + return componentOwner(node).owner +} + +function stringValue(node: ts.Expression | undefined): string | undefined { + if (!node) return undefined + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) return node.text + return undefined +} + +/** Placeholder name for a simple interpolated expression, or undefined when it needs a person. */ +export function placeholderName(expression: ts.Expression): string | undefined { + if (ts.isIdentifier(expression)) return expression.text + if (ts.isPropertyAccessExpression(expression) && !expression.questionDotToken) { + return expression.name.text + } + if (ts.isNonNullExpression(expression)) return placeholderName(expression.expression) + return undefined +} + +export type SentenceChild = + /** `text` is what the node renders. */ + | { type: 'text'; node: ts.JsxText; text: string } + /** A string literal or `const` string rendered inline, e.g. `{' '}` or `{LABEL}`. */ + | { type: 'static'; node: ts.JsxExpression; text: string } + | { type: 'value'; node: ts.JsxExpression; expression: ts.Expression; name: string } + +/** + * Children of an element that render as one sentence: text, static strings and + * simple interpolations, no nested elements. Static parts are folded into the + * rendered text, so `Share{' '}project` and `Share project` read the same and a + * changed constant changes the sentence. + */ +export function sentenceChildren(element: ts.JsxElement): SentenceChild[] | undefined { + const children = element.children.filter( + (child) => !(ts.isJsxText(child) && child.containsOnlyTriviaWhiteSpaces), + ) + if (children.length < 2) return undefined + const result: SentenceChild[] = [] + let hasText = false + let hasJoin = false + for (const child of children) { + if (ts.isJsxText(child)) { + const text = renderJsxText(child.text) + hasText ||= /\p{L}{2}/u.test(text) + result.push({ type: 'text', node: child, text }) + continue + } + if (ts.isJsxExpression(child) && child.expression && !child.dotDotDotToken) { + const expression = child.expression + const literal = + ts.isStringLiteral(expression) || ts.isNoSubstitutionTemplateLiteral(expression) + ? expression + : ts.isIdentifier(expression) + ? constantValue(expression) + : undefined + if (literal) { + hasText ||= /\p{L}{2}/u.test(literal.text) + hasJoin = true + result.push({ type: 'static', node: child, text: literal.text }) + continue + } + const name = placeholderName(expression) + if (!name) return undefined + hasJoin = true + result.push({ type: 'value', node: child, expression, name }) + continue + } + return undefined + } + return hasText && hasJoin ? result : undefined +} + +export function sentenceMessage(children: SentenceChild[]): string { + return children + .map((child) => (child.type === 'value' ? `{${child.name}}` : child.text)) + .join('') + .replace(ASCII_EDGE_WHITESPACE, '') +} + +function notifyCallee(node: ts.CallExpression): string | undefined { + const callee = node.expression + if (ts.isIdentifier(callee) && NOTIFY_CALLEES.has(callee.text)) return callee.text + if ( + ts.isPropertyAccessExpression(callee) && + ts.isIdentifier(callee.expression) && + callee.expression.text === 'toast' && + NOTIFY_METHODS.has(callee.name.text) + ) { + return `toast.${callee.name.text}` + } + if ( + ts.isPropertyAccessExpression(callee) && + ts.isIdentifier(callee.expression) && + callee.expression.text === 'window' && + NOTIFY_CALLEES.has(callee.name.text) + ) { + return `window.${callee.name.text}` + } + return undefined +} + +export type Visitor = { + jsxText(node: ts.JsxText, text: string): void + sentence(element: ts.JsxElement, children: SentenceChild[], text: string): void + literal(node: ts.StringLiteral | ts.NoSubstitutionTemplateLiteral, kind: CandidateKind): void + template(node: ts.TemplateExpression, kind: CandidateKind): void + /** A `const` holding copy, rendered by name in JSX or a UI prop. */ + constant(node: ts.Identifier, text: string, kind: CandidateKind): void + /** Copy built with `+`, e.g. `'Delete ' + name`. */ + concatenation(node: ts.BinaryExpression, text: string, kind: CandidateKind): void +} + +function bindsName(name: ts.BindingName, wanted: string): boolean { + if (ts.isIdentifier(name)) return name.text === wanted + return name.elements.some( + (element) => !ts.isOmittedExpression(element) && bindsName(element.name, wanted), + ) +} + +type Binding = + | { found: false } + | { found: true; value?: ts.StringLiteral | ts.NoSubstitutionTemplateLiteral } + +/** The declaration of `name` directly owned by one scope node, if any. */ +function bindingIn(scope: ts.Node, name: string): Binding { + if (ts.isFunctionLike(scope)) { + if (scope.parameters.some((parameter) => bindsName(parameter.name, name))) + return { found: true } + return { found: false } + } + if ( + (ts.isForStatement(scope) || ts.isForOfStatement(scope) || ts.isForInStatement(scope)) && + scope.initializer && + ts.isVariableDeclarationList(scope.initializer) && + scope.initializer.declarations.some((declaration) => bindsName(declaration.name, name)) + ) { + return { found: true } + } + if ( + ts.isCatchClause(scope) && + scope.variableDeclaration && + bindsName(scope.variableDeclaration.name, name) + ) { + return { found: true } + } + if ( + !( + ts.isSourceFile(scope) || + ts.isBlock(scope) || + ts.isModuleBlock(scope) || + ts.isCaseClause(scope) + ) + ) { + return { found: false } + } + for (const statement of scope.statements) { + if (ts.isVariableStatement(statement)) { + const isConst = (statement.declarationList.flags & ts.NodeFlags.Const) !== 0 + for (const declaration of statement.declarationList.declarations) { + if (!bindsName(declaration.name, name)) continue + const value = declaration.initializer + return isConst && + ts.isIdentifier(declaration.name) && + value && + (ts.isStringLiteral(value) || ts.isNoSubstitutionTemplateLiteral(value)) + ? { found: true, value } + : { found: true } + } + } + if ( + (ts.isFunctionDeclaration(statement) || + ts.isClassDeclaration(statement) || + ts.isEnumDeclaration(statement)) && + statement.name?.text === name + ) { + return { found: true } + } + if (ts.isImportDeclaration(statement)) { + const clause = statement.importClause + const bindings = clause?.namedBindings + if ( + clause?.name?.text === name || + (bindings && ts.isNamespaceImport(bindings) && bindings.name.text === name) || + (bindings && + ts.isNamedImports(bindings) && + bindings.elements.some((element) => element.name.text === name)) + ) { + return { found: true } + } + } + } + return { found: false } +} + +/** The `const` string an identifier refers to, resolved through the nearest scope that declares it. */ +function constantValue( + identifier: ts.Identifier, +): ts.StringLiteral | ts.NoSubstitutionTemplateLiteral | undefined { + for (let scope = identifier.parent; scope; scope = scope.parent) { + const binding = bindingIn(scope, identifier.text) + if (binding.found) return binding.value + } + return undefined +} + +function isNotifyOptions(node: ts.Node): boolean { + const call = node.parent + return ( + ts.isObjectLiteralExpression(node) && + !!call && + ts.isCallExpression(call) && + call.arguments[1] === node && + notifyCallee(call) !== undefined + ) +} + +/** Walks every user-visible literal in one file. */ +export function visitUiLiterals( + source: ts.SourceFile, + visitor: Visitor, + allow: Set, + /** Plain modules are scanned for notifications only unless the codemod was pointed at them. */ + allKinds = source.fileName.endsWith('.tsx'), +): void { + const notifyOnly = !allKinds + const literalSink = (expression: ts.Expression | undefined, kind: CandidateKind): void => { + if (!expression || (notifyOnly && kind !== 'notify')) return + if (ts.isParenthesizedExpression(expression)) { + literalSink(expression.expression, kind) + return + } + if (ts.isConditionalExpression(expression)) { + literalSink(expression.whenTrue, kind) + literalSink(expression.whenFalse, kind) + return + } + if ( + ts.isBinaryExpression(expression) && + (expression.operatorToken.kind === ts.SyntaxKind.BarBarToken || + expression.operatorToken.kind === ts.SyntaxKind.QuestionQuestionToken || + expression.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken) + ) { + if (expression.operatorToken.kind !== ts.SyntaxKind.AmpersandAmpersandToken) { + literalSink(expression.left, kind) + } + literalSink(expression.right, kind) + return + } + if ( + ts.isBinaryExpression(expression) && + expression.operatorToken.kind === ts.SyntaxKind.PlusToken + ) { + const parts: ts.Expression[] = [] + const flatten = (part: ts.Expression): void => { + if (ts.isParenthesizedExpression(part)) flatten(part.expression) + else if ( + ts.isBinaryExpression(part) && + part.operatorToken.kind === ts.SyntaxKind.PlusToken + ) { + flatten(part.left) + flatten(part.right) + } else parts.push(part) + } + flatten(expression) + // A `const` string operand is copy like a literal one. + const literalOf = (part: ts.Expression) => + ts.isStringLiteral(part) || ts.isNoSubstitutionTemplateLiteral(part) + ? part + : ts.isIdentifier(part) + ? constantValue(part) + : undefined + const text = parts.map((part) => literalOf(part)?.text ?? '{?}').join('') + const hasCopy = parts.some((part) => /\p{L}{2}/u.test(literalOf(part)?.text ?? '')) + if (hasCopy && looksTranslatable(text, allow) && !isIgnored(source, expression)) { + visitor.concatenation(expression, text, kind) + } + return + } + if (ts.isIdentifier(expression)) { + const value = constantValue(expression) + if ( + value && + looksTranslatable(value.text, allow) && + !isIgnored(source, value) && + !isIgnored(source, expression) + ) { + visitor.constant(expression, value.text, kind) + } + return + } + if (ts.isStringLiteral(expression) || ts.isNoSubstitutionTemplateLiteral(expression)) { + if (looksTranslatable(expression.text, allow) && !isIgnored(source, expression)) { + visitor.literal(expression, kind) + } + return + } + if (ts.isTemplateExpression(expression)) { + const text = [ + expression.head.text, + ...expression.templateSpans.map((span) => span.literal.text), + ].join(' ') + if (looksTranslatable(text, allow) && !isIgnored(source, expression)) { + visitor.template(expression, kind) + } + } + } + + const visit = (node: ts.Node): void => { + if (isTranslationCall(node)) return + + if (ts.isJsxElement(node)) { + const sentence = sentenceChildren(node) + if (sentence) { + const text = sentenceMessage(sentence) + // The whole joined text is what renders, so exemptions apply to it. + if (looksTranslatable(text, allow) && !isIgnored(source, sentence[0]!.node)) { + visitor.sentence(node, sentence, text) + } + ts.forEachChild(node.openingElement, visit) + return + } + } + + if (ts.isJsxText(node) && !node.containsOnlyTriviaWhiteSpaces) { + const text = renderJsxText(node.text) + if (looksTranslatable(text, allow) && !isIgnored(source, node)) visitor.jsxText(node, text) + return + } + + if ( + ts.isJsxExpression(node) && + node.parent && + (ts.isJsxElement(node.parent) || ts.isJsxFragment(node.parent)) + ) { + literalSink(node.expression, 'expression') + } + + if ( + ts.isJsxAttribute(node) && + ts.isIdentifier(node.name) && + UI_ATTRIBUTES.has(node.name.text) + ) { + const initializer = node.initializer + if (initializer && ts.isStringLiteral(initializer)) literalSink(initializer, 'attribute') + else if (initializer && ts.isJsxExpression(initializer)) { + literalSink(initializer.expression, 'attribute') + } + } + + if ( + ts.isPropertyAssignment(node) && + (ts.isIdentifier(node.name) || ts.isStringLiteral(node.name)) && + UI_PROPERTIES.has(node.name.text) && + !insideTranslationCall(node) && + !isNotifyOptions(node.parent) + ) { + literalSink(node.initializer, 'property') + } + + if (ts.isCallExpression(node) && notifyCallee(node)) { + literalSink(node.arguments[0], 'notify') + const options = node.arguments[1] + if (options && ts.isObjectLiteralExpression(options)) { + for (const property of options.properties) { + if ( + ts.isPropertyAssignment(property) && + ts.isIdentifier(property.name) && + property.name.text === 'description' + ) { + literalSink(property.initializer, 'notify') + } + } + } + } + + ts.forEachChild(node, visit) + } + visit(source) +} + +export function parseSource(path: string, text: string): ts.SourceFile { + return ts.createSourceFile( + path, + text, + ts.ScriptTarget.Latest, + true, + path.endsWith('.tsx') ? ts.ScriptKind.TSX : ts.ScriptKind.TS, + ) +} + +export function scanSource( + path: string, + text: string, + allowlist: Allowlist = readAllowlist(), +): Candidate[] { + const source = parseSource(path, text) + const allow = new Set(allowlist.texts) + const candidates: Candidate[] = [] + const add = (node: ts.Node, kind: CandidateKind, value: string, manual?: string) => { + const { owner, problem } = componentOwner(node) + candidates.push({ + kind, + text: value, + line: lineOf(source, node), + component: owner ? functionName(owner) : undefined, + manual: manual ?? problem, + }) + } + visitUiLiterals( + source, + { + jsxText: (node, value) => add(node, 'jsx-text', value.replace(ASCII_EDGE_WHITESPACE, '')), + sentence: (element, _children, value) => add(element, 'jsx-sentence', value), + literal: (node, kind) => add(node, kind, attributeText(node)), + template: (node, kind) => { + const names = node.templateSpans.map((span) => placeholderName(span.expression)) + const value = [ + node.head.text, + ...node.templateSpans.map( + (span, index) => `{${names[index] ?? '?'}}${span.literal.text}`, + ), + ].join('') + add( + node, + kind, + value, + names.includes(undefined) ? 'complex template interpolation' : undefined, + ) + }, + constant: (node, value, kind) => + add(node, kind, value, 'string constant: make it a descriptor or translate at the use'), + concatenation: (node, value, kind) => + add(node, kind, value, 'concatenated copy: write one message with placeholders'), + }, + allow, + ) + return candidates +} + +export function scannedFiles(): string[] { + const allowlist = readAllowlist() + const files = new Set() + for (const pattern of SCANNED_GLOBS) { + for (const path of new Bun.Glob(pattern).scanSync({ cwd: ROOT, onlyFiles: true })) { + if (SKIPPED_PATH.test(`/${path}`)) continue + if (Object.keys(allowlist.paths).some((prefix) => path.startsWith(prefix))) continue + files.add(path) + } + } + return [...files].sort() +} + +export function scanRepository(): Map { + const allowlist = readAllowlist() + const result = new Map() + for (const path of scannedFiles()) { + const candidates = scanSource(path, readFileSync(join(ROOT, path), 'utf8'), allowlist) + if (candidates.length > 0) result.set(path, candidates) + } + return result +} + +export function parseBaseline(text: string | undefined): Baseline | undefined { + if (!text) return undefined + const parsed = JSON.parse(text) as { version?: number; files?: Record } + // Version 3 records rendered text (entities, joins); older baselines are not comparable. + return parsed.version === 3 ? (parsed as Baseline) : undefined +} + +function readBaseline(): Baseline { + return parseBaseline(readFileSync(BASELINE_PATH, 'utf8')) ?? { version: 3, files: {} } +} + +/** The base branch's baseline, when the base has one (see `baseRevision`). */ +function readBaseBaseline(): Baseline | undefined { + const revision = baseRevision() + return revision ? parseBaseline(readAtRevision(revision, BASELINE_PATH)) : undefined +} + +function writeBaseline(scan: Map): void { + const files = Object.fromEntries( + [...scan.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([path, list]) => [path, list.map((candidate) => candidate.text).sort()]), + ) + writeFileSync( + BASELINE_PATH, + `${JSON.stringify({ version: 3, files } satisfies Baseline, null, 2)}\n`, + ) +} + +/** Items of `values` left after removing one occurrence per item of `allowed`. */ +function beyond(values: readonly string[], allowed: readonly string[]): string[] { + const remaining = new Map() + for (const value of allowed) remaining.set(value, (remaining.get(value) ?? 0) + 1) + return values.filter((value) => { + const left = remaining.get(value) ?? 0 + if (left === 0) return true + remaining.set(value, left - 1) + return false + }) +} + +/** + * The ratchet: a file may only keep copy its baseline already lists, so new + * or swapped copy fails, and a file absent from the baseline must have none. + * The baseline itself may only shrink relative to the base branch; exempting + * copy goes through `i18n/literal-allowlist.json` or an `i18n-ignore: `. + */ +export function checkLiterals( + scan = scanRepository(), + baseline = readBaseline(), + base?: Baseline, +): { + errors: string[] + improvable: string[] + total: number +} { + const errors: string[] = [] + const improvable: string[] = [] + let total = 0 + for (const [path, candidates] of scan) { + total += candidates.length + const unknown = new Set( + beyond( + candidates.map((candidate) => candidate.text), + baseline.files[path] ?? [], + ), + ) + if (unknown.size === 0) continue + const listed = candidates + .filter((candidate) => unknown.has(candidate.text)) + .map((candidate) => ` ${path}:${candidate.line} ${JSON.stringify(candidate.text)}`) + errors.push( + `${path}: untranslated user-visible literals not in the baseline. Wrap them with i18n._({ id, message }) or add an "i18n-ignore: " comment.\n${listed.join('\n')}`, + ) + } + for (const [path, allowed] of Object.entries(baseline.files)) { + const left = beyond( + allowed, + (scan.get(path) ?? []).map((candidate) => candidate.text), + ) + if (left.length > 0) improvable.push(`${path}: ${left.length} fewer`) + } + if (base) { + // Growth is measured across the repository, so copy that moves with a + // renamed, split or extracted file passes; each added text must consume an + // equal text removed elsewhere, duplicates counted. + const all = (files: Baseline['files']) => Object.values(files).flat() + const added = beyond(all(baseline.files), all(base.files)) + if (added.length > 0) { + const where = (text: string) => + Object.entries(baseline.files) + .filter(([, texts]) => texts.includes(text)) + .map(([path]) => path) + errors.push( + `i18n/literal-baseline.json adds copy the base branch's baseline does not have; the baseline may only shrink:\n${[ + ...new Set(added), + ] + .map((text) => ` ${JSON.stringify(text)} (${where(text).join(', ')})`) + .join('\n')}`, + ) + } + } + return { errors, improvable, total } +} + +/** Exemptions are reviewed like the baseline: the allowlist may not gain entries relative to the base. */ +export function checkAllowlistGrowth(current: Allowlist, base: Allowlist | undefined): string[] { + if (!base) return [] + const texts = current.texts.filter((text) => !base.texts.includes(text)) + const paths = Object.keys(current.paths).filter((path) => !(path in base.paths)) + return [ + ...texts.map( + (text) => + `i18n/literal-allowlist.json adds the text ${JSON.stringify(text)}; exempt one use with i18n-ignore: instead`, + ), + ...paths.map( + (path) => + `i18n/literal-allowlist.json adds the path ${JSON.stringify(path)}; exempting a path needs a reviewed exception`, + ), + ] +} + +function readBaseAllowlist(): Allowlist | undefined { + const revision = baseRevision() + const text = revision ? readAtRevision(revision, ALLOWLIST_PATH) : undefined + return text ? (JSON.parse(text) as Allowlist) : undefined +} + +export function checkRepositoryLiterals() { + const result = checkLiterals(scanRepository(), readBaseline(), readBaseBaseline()) + result.errors.push(...checkAllowlistGrowth(readAllowlist(), readBaseAllowlist())) + return result +} + +function report(scan: Map): void { + const byArea = new Map() + for (const [path, candidates] of scan) { + const area = path + .split('/') + .slice(0, path.startsWith('packages/editor/src/components/ui/') ? 6 : 4) + .join('/') + byArea.set(area, (byArea.get(area) ?? 0) + candidates.length) + } + for (const [area, count] of [...byArea.entries()].sort((a, b) => b[1] - a[1])) { + console.log(`${String(count).padStart(6)} ${area}`) + } +} + +async function main(): Promise { + const [command, ...args] = process.argv.slice(2) + if (command === 'check') { + const { errors, improvable, total } = checkRepositoryLiterals() + if (improvable.length > 0) { + console.log( + `[i18n literals] ${improvable.length} files are below their baseline; run bun run i18n:literals -- update to lock in the progress`, + ) + } + if (errors.length > 0) { + console.error(errors.map((error) => `[i18n literals] ${error}`).join('\n')) + process.exitCode = 1 + return + } + console.log(`[i18n literals] no new untranslated literals (${total} legacy candidates remain)`) + return + } + if (command === 'update') { + writeBaseline(scanRepository()) + console.log(`Updated ${relative(ROOT, BASELINE_PATH)}`) + return + } + if (command === 'report') { + const scan = scanRepository() + const filter = args.filter((arg) => !arg.startsWith('--')) + if (args.includes('--files')) { + for (const [path, candidates] of scan) { + if (filter.length && !filter.some((prefix) => path.startsWith(prefix))) continue + for (const candidate of candidates) { + console.log( + `${path}:${candidate.line}\t${candidate.kind}\t${candidate.component ?? '-'}\t${JSON.stringify(candidate.text)}${candidate.manual ? `\tMANUAL: ${candidate.manual}` : ''}`, + ) + } + } + return + } + report(scan) + return + } + if (command === 'convert') { + const { convertFiles } = await import('./i18n-codemod') + await convertFiles(args) + return + } + throw new Error( + 'Usage: bun scripts/i18n-literals.ts >', + ) +} + +if (import.meta.main) await main() diff --git a/scripts/i18n-pseudo-audit.ts b/scripts/i18n-pseudo-audit.ts new file mode 100644 index 0000000000..f10dec99fa --- /dev/null +++ b/scripts/i18n-pseudo-audit.ts @@ -0,0 +1,180 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { chromium, type Page } from 'playwright-core' +import { type PseudoSurface, surfaces } from '../i18n/pseudo-surfaces' + +/** + * Pseudo-locale audit. Opens each surface of a running editor with + * `?lang=en-XA`, where every translated letter is accented and the text is + * ~30 % longer and bracketed, then reports any visible text or accessible + * name that still contains plain ASCII words: that copy bypassed the catalog. + * Visible controls without an accessible name are reported too, so narrow + * layouts that hide a button's text still name it. + * Screenshots of each surface go to the output directory for clipping review. + * + * bun run i18n:pseudo-audit -- --base http://localhost:3002 [--out dir] [surface...] + * + * Needs a Chrome/Chromium binary (`PLAYWRIGHT_CHROMIUM_EXECUTABLE`, defaulting + * to the system Google Chrome). It is a contributor and PR tool, not a CI job. + */ + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const ALLOWLIST = JSON.parse(readFileSync(join(ROOT, 'i18n/literal-allowlist.json'), 'utf8')) as { + texts: string[] +} + +export type Finding = { surface: string; where: string; text: string } + +function argument(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`) + return index === -1 ? undefined : process.argv[index + 1] +} + +/** ASCII words left after removing allowed data (authored names, brands, units). */ +export function untranslatedWords(text: string, allowed: readonly string[]): string[] { + let rest = text + for (const value of [...allowed].sort((a, b) => b.length - a.length)) { + if (!value) continue + // Whole words only: allowing the unit "in" must not hide "Find" or "Pin". + const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + rest = rest.replace(new RegExp(`(? { + const roots = selector ? [...document.querySelectorAll(selector)] : [document.body] + const results: Array<{ where: string; text: string }> = [] + const describe = (element: Element) => { + const parts: string[] = [] + for ( + let current: Element | null = element; + current && parts.length < 3; + current = current.parentElement + ) { + const id = current.id ? `#${current.id}` : '' + const testId = current.getAttribute('data-testid') + parts.unshift( + `${current.tagName.toLowerCase()}${id}${testId ? `[data-testid=${testId}]` : ''}`, + ) + } + return parts.join(' > ') + } + const visible = (element: Element) => + element.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }) && + !element.closest('[aria-hidden="true"], script, style, noscript, svg title') + for (const root of roots) { + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT) + for (let node = walker.nextNode(); node; node = walker.nextNode()) { + const text = node.textContent?.trim() + const parent = node.parentElement + if (text && parent && visible(parent)) results.push({ where: describe(parent), text }) + } + for (const element of root.querySelectorAll( + '[aria-label], [title], [placeholder], img[alt]', + )) { + if (!visible(element)) continue + for (const name of ['aria-label', 'title', 'placeholder', 'alt']) { + const value = element.getAttribute(name)?.trim() + if (value) results.push({ where: `${describe(element)} @${name}`, text: value }) + } + } + } + const unnamed: string[] = [] + for (const root of roots) { + for (const control of root.querySelectorAll( + 'button, a[href], [role="button"], input, select', + )) { + if (!visible(control)) continue + const labelledBy = control + .getAttribute('aria-labelledby') + ?.split(/\s+/) + .map((id) => document.getElementById(id)?.textContent ?? '') + .join(' ') + const name = [ + control.getAttribute('aria-label'), + labelledBy, + control.getAttribute('title'), + // biome-ignore lint/performance/useDomNodeTextContent: text hidden at this width must not count as a name + (control as HTMLElement).innerText, + // biome-ignore lint/performance/useDomNodeTextContent: text hidden at this width must not count as a name + (control as HTMLInputElement).labels?.[0]?.innerText, + ...[...control.querySelectorAll('img[alt]')].map((image) => image.getAttribute('alt')), + ] + .map((value) => value?.trim()) + .find(Boolean) + if (!name) unnamed.push(describe(control)) + } + } + return { results, unnamed } + }, scope) +} + +export async function auditSurface(page: Page, base: string, surface: PseudoSurface, out: string) { + const findings: Finding[] = [] + const path = typeof surface.path === 'string' ? surface.path : await surface.path(base) + const url = new URL(path, base) + url.searchParams.set('lang', 'en-XA') + await page.setViewportSize(surface.viewport ?? { width: 1440, height: 900 }) + await page.goto(url.toString(), { waitUntil: 'domcontentloaded' }) + await page.waitForLoadState('networkidle', { timeout: 60_000 }).catch(() => undefined) + await surface.steps?.(page) + await page.waitForTimeout(400) + const allowed = [...ALLOWLIST.texts, ...(surface.data ?? [])] + const { results, unnamed } = await collect(page, surface.scope) + for (const where of unnamed) { + findings.push({ surface: surface.name, where, text: 'control has no accessible name' }) + } + for (const entry of results) { + const words = untranslatedWords(entry.text, allowed) + if (words.length > 0) + findings.push({ surface: surface.name, where: entry.where, text: entry.text }) + } + const shot = join(out, `${surface.name}.png`) + if (surface.scope && (await page.locator(surface.scope).count()) === 0) { + findings.push({ surface: surface.name, where: surface.scope, text: 'surface did not open' }) + } + await page.screenshot({ path: shot }) + return { findings, shot } +} + +async function main(): Promise { + const base = argument('base') ?? 'http://localhost:3002' + const out = resolve(argument('out') ?? join(ROOT, '.i18n-audit')) + const wanted = process.argv + .slice(2) + .filter((arg, index, all) => !arg.startsWith('--') && !all[index - 1]?.startsWith('--')) + const selected = surfaces.filter( + (surface) => wanted.length === 0 || wanted.includes(surface.name), + ) + if (selected.length === 0) throw new Error(`No surface matches ${wanted.join(', ')}`) + mkdirSync(out, { recursive: true }) + + const browser = await chromium.launch({ + executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE ?? '/usr/local/bin/google-chrome', + args: ['--use-angle=swiftshader', '--enable-unsafe-swiftshader'], + }) + const findings: Finding[] = [] + try { + const context = await browser.newContext({ deviceScaleFactor: 1 }) + const page = await context.newPage() + for (const surface of selected) { + const result = await auditSurface(page, base, surface, out) + findings.push(...result.findings) + console.log( + `${result.findings.length === 0 ? 'PASS' : 'FAIL'} ${surface.name} (${result.findings.length} untranslated) → ${result.shot}`, + ) + } + } finally { + await browser.close() + } + writeFileSync(join(out, 'findings.json'), `${JSON.stringify(findings, null, 2)}\n`) + for (const finding of findings) { + console.log(` ${finding.surface}: ${JSON.stringify(finding.text)} (${finding.where})`) + } + if (findings.length > 0) process.exitCode = 1 +} + +if (import.meta.main) await main() diff --git a/scripts/i18n.ts b/scripts/i18n.ts index 5878f9ab5a..20b2274ddd 100644 --- a/scripts/i18n.ts +++ b/scripts/i18n.ts @@ -1,12 +1,20 @@ import { createHash } from 'node:crypto' -import { readFileSync, writeFileSync } from 'node:fs' +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' import { dirname, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { createCompiledCatalog, getCatalogs } from '@lingui/cli/api' import { getConfig } from '@lingui/conf' import { compileMessageOrThrow } from '@lingui/message-utils/compileMessage' import { type PoFile, type PoItem, parsePo, stringifyPo } from 'pofile-ts' -import ts from 'typescript' +import { + baseRevision, + firstCommitWith, + headRevision, + lastReleaseTag, + readAtRevision, + resolveRevision, +} from './i18n-git' +import { checkRepositoryLiterals } from './i18n-literals' export type SourceMessage = { id: string @@ -20,6 +28,8 @@ export type TranslationEntry = { review: 'human' | 'required' | 'missing' fuzzy: boolean reviewNotes?: string + /** Declared with a `renamed-from: ` descriptor comment (`#.` in the PO). */ + renamedFrom?: string } export type TranslationRequest = { @@ -46,23 +56,18 @@ type Glossary = { type CompiledToken = string | [name: string, type?: string, choices?: unknown] -const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') -const CATALOG_ROOT = join(ROOT, 'packages/editor/src/i18n/locales') +const ROOT = process.env.I18N_ROOT + ? resolve(process.env.I18N_ROOT) + : resolve(dirname(fileURLToPath(import.meta.url)), '..') +const CATALOG_ROOT = process.env.I18N_CATALOG_ROOT + ? resolve(process.env.I18N_CATALOG_ROOT) + : join(ROOT, 'packages/editor/src/i18n/locales') const GLOSSARY_PATH = join(ROOT, 'i18n/glossary.json') const LINGUI_BIN = join(ROOT, 'node_modules/.bin/lingui') const TARGET_LOCALES = ['zh-CN', 'fr', 'es', 'ja'] as const const ALL_LOCALES = ['en', ...TARGET_LOCALES] as const -const CONVERTED_SURFACES = [ - 'packages/editor/src/components/viewer', - 'packages/editor/src/components/ui/build-panel/build-panel.tsx', -] -const SOURCE_GLOBS = [ - 'packages/core/src/**/*.{ts,tsx}', - 'packages/viewer/src/**/*.{ts,tsx}', - 'packages/editor/src/**/*.{ts,tsx}', - 'packages/nodes/src/**/*.{ts,tsx}', - 'apps/editor/**/*.{ts,tsx}', -] +/** Generated from English for layout and coverage testing; never shipped as a language. */ +export const PSEUDO_LOCALE = 'en-XA' const SOURCE_HASH_PREFIX = 'Source-Hash: ' const REVIEW_PREFIX = 'Review: ' const ENGLISH_PREFIX = 'English: ' @@ -138,6 +143,13 @@ async function extractSourceMessages(): Promise { return [...messages.values()].sort((left, right) => left.id.localeCompare(right.id)) } +const RENAMED_FROM_PREFIX = 'renamed-from:' + +function renamedFrom(item: PoItem): string | undefined { + const comment = item.extractedComments.find((value) => value.startsWith(RENAMED_FROM_PREFIX)) + return comment?.slice(RENAMED_FROM_PREFIX.length).trim() || undefined +} + function readTranslationState(po: PoFile): Record { return Object.fromEntries( po.items @@ -152,6 +164,7 @@ function readTranslationState(po: PoFile): Record { review: review === 'human' || review === 'required' ? review : 'missing', fuzzy: Boolean(item.flags.fuzzy), reviewNotes: commentValue(item, REVIEW_NOTES_PREFIX), + renamedFrom: renamedFrom(item), } satisfies TranslationEntry, ] }), @@ -462,6 +475,26 @@ function writePo(path: string, po: PoFile): void { writeFileSync(path, stringifyPo(po, { foldLength: 0 })) } +/** + * Review state extraction writes for one target entry, or undefined to keep + * it. A newly filled translation is a draft unless a person already marked it + * `Review: human` without `fuzzy`; extraction never approves copy itself. + */ +export function annotatedState( + prior: TranslationEntry | undefined, + translation: string, + sourceHash: string, +): Pick | undefined { + if (!translation) return { sourceHash: undefined, review: 'missing', fuzzy: false } + if (prior?.sourceHash && prior.sourceHash !== sourceHash) { + return { sourceHash: prior.sourceHash, review: 'required', fuzzy: true } + } + if (prior?.sourceHash) return undefined + return prior?.review === 'human' && !prior.fuzzy + ? { sourceHash, review: 'human', fuzzy: false } + : { sourceHash, review: 'required', fuzzy: true } +} + function annotateCatalogs( previous: Map<(typeof TARGET_LOCALES)[number], Record>, ): void { @@ -477,18 +510,28 @@ function annotateCatalogs( replaceComment(item, ENGLISH_PREFIX, JSON.stringify(source.message)) const sourceHash = hashSource(source.message) const prior = previous.get(locale)?.[item.msgid] - if (!item.msgstr[0]) { - replaceComment(item, SOURCE_HASH_PREFIX) - replaceComment(item, REVIEW_PREFIX, 'missing') - delete item.flags.fuzzy - } else if (prior?.sourceHash && prior.sourceHash !== sourceHash) { - replaceComment(item, SOURCE_HASH_PREFIX, prior.sourceHash) - replaceComment(item, REVIEW_PREFIX, 'required') - item.flags.fuzzy = true - } else if (!prior?.sourceHash) { + const declared = renamedFrom(item) + const origin = declared ? previous.get(locale)?.[declared] : undefined + if ( + !item.msgstr[0] && + origin?.translation && + origin.review === 'human' && + !origin.fuzzy && + origin.sourceHash === sourceHash + ) { + // A declared rename with unchanged English keeps its reviewed translation. + item.msgstr = [origin.translation] replaceComment(item, SOURCE_HASH_PREFIX, sourceHash) replaceComment(item, REVIEW_PREFIX, 'human') delete item.flags.fuzzy + continue + } + const state = annotatedState(prior, item.msgstr[0] ?? '', sourceHash) + if (state) { + replaceComment(item, SOURCE_HASH_PREFIX, state.sourceHash) + replaceComment(item, REVIEW_PREFIX, state.review) + if (state.fuzzy) item.flags.fuzzy = true + else delete item.flags.fuzzy } } writePo(path, po) @@ -524,7 +567,12 @@ function compiledCatalog(locale: string, sources: SourceMessage[]) { function writeCompiledCatalogs(strict = false): void { const sources = readSourceMessages() if (strict) { - const errors = checkCatalogMessages(sources, readJson(GLOSSARY_PATH)) + const errors = checkTranslationEntries( + sources, + readLocaleEntries((path) => readFileSync(path, 'utf8')), + readJson(GLOSSARY_PATH), + {}, + ) if (errors.length > 0) throw new Error(errors.join('\n')) } const compiled = ALL_LOCALES.map((locale) => [locale, compiledCatalog(locale, sources)] as const) @@ -535,6 +583,147 @@ function writeCompiledCatalogs(strict = false): void { for (const [locale, result] of compiled) { writeFileSync(compiledCatalogPath(locale), result.source) } + writePseudoCatalog(sources) +} + +const PSEUDO_LETTERS: Record = Object.fromEntries( + [...'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'].map((letter, index) => [ + letter, + [...'áƀçðéƒĝĥîĵķļɱñöþǫŕšţûṽŵẋýžÅƁÇÐÉƑĜĤÎĴĶĻṀÑÖÞǪŔŠŢÛṼŴẊÝŽ'][index]!, + ]), +) + +/** + * en-XA: every letter accented and the text padded by about 30 %, wrapped in + * brackets so clipping shows. ICU arguments, plural/select keys, `#`, quoted + * literals and rich-text tags are copied unchanged. + */ +export function pseudoLocalize(message: string): string { + let index = 0 + let letters = 0 + + const text = (end: (character: string) => boolean, inPlural: boolean): string => { + let out = '' + while (index < message.length) { + const character = message[index]! + if (end(character)) return out + if (character === "'") { + const next = message[index + 1] + if (next === "'") { + out += "''" + index += 2 + continue + } + if (next === '{' || next === '}' || next === '#' || next === '<') { + const close = message.indexOf("'", index + 1) + const stop = close === -1 ? message.length : close + 1 + out += message.slice(index, stop) + index = stop + continue + } + } + if (character === '{') { + out += argument() + continue + } + if (character === '<') { + const close = message.indexOf('>', index) + const stop = close === -1 ? message.length : close + 1 + out += message.slice(index, stop) + index = stop + continue + } + if (character === '#' && inPlural) { + out += character + index += 1 + continue + } + const mapped = PSEUDO_LETTERS[character] + if (mapped) letters += 1 + out += mapped ?? character + index += 1 + } + return out + } + + const argument = (): string => { + const start = index + index += 1 + const header = message.slice(index).match(/^\s*([^,}\s]+)\s*(?:,\s*(\w+)\s*)?/) + const type = header?.[2] + if (!(type === 'plural' || type === 'select' || type === 'selectordinal')) { + let depth = 1 + while (index < message.length && depth > 0) { + if (message[index] === '{') depth += 1 + if (message[index] === '}') depth -= 1 + index += 1 + } + return message.slice(start, index) + } + index += header![0].length + let out = message.slice(start, index) + if (message[index] === ',') { + out += ',' + index += 1 + } + while (index < message.length && message[index] !== '}') { + const open = message.indexOf('{', index) + if (open === -1) break + out += message.slice(index, open + 1) + index = open + 1 + out += text((character) => character === '}', type !== 'select') + out += '}' + index += 1 + const spaces = message.slice(index).match(/^\s*/)?.[0] ?? '' + if (message[index + spaces.length] === '}') { + out += spaces + index += spaces.length + } + } + out += '}' + index += 1 + return out + } + + const body = text(() => false, false) + if (letters === 0) return body + return `[${body} ${'~'.repeat(Math.max(1, Math.ceil(letters * 0.3)))}]` +} + +function pseudoCatalog(sources: SourceMessage[]) { + const english = messagesForCompilation('en', sources) + return createCompiledCatalog( + PSEUDO_LOCALE, + Object.fromEntries( + Object.entries(english).map(([id, message]) => [id, pseudoLocalize(message)]), + ), + { namespace: 'ts', outputPrefix: '/*eslint-disable*/', strict: true }, + ) +} + +function writePseudoCatalog(sources: SourceMessage[]): void { + const compiled = pseudoCatalog(sources) + if (compiled.errors.length > 0) { + throw new Error( + compiled.errors + .map(({ id, error }) => `${PSEUDO_LOCALE}: "${id}": ${error.message}`) + .join('\n'), + ) + } + mkdirSync(dirname(compiledCatalogPath(PSEUDO_LOCALE)), { recursive: true }) + writeFileSync(compiledCatalogPath(PSEUDO_LOCALE), compiled.source) +} + +function checkPseudoCatalog(sources: SourceMessage[]): string[] { + const path = compiledCatalogPath(PSEUDO_LOCALE) + const compiled = pseudoCatalog(sources) + const errors = compiled.errors.map( + ({ id, error }) => `${PSEUDO_LOCALE}: failed to compile "${id}": ${error.message}`, + ) + if (!existsSync(path) || compiled.source !== readFileSync(path, 'utf8')) { + errors.push(`${relative(ROOT, path)} is not synchronized; run bun run i18n:compile`) + } + return errors } export function extractCatalogs(): void { @@ -581,64 +770,46 @@ function applyTranslations( } } -function sourceFiles(): string[] { - const files = new Set() - for (const pattern of SOURCE_GLOBS) { - const glob = new Bun.Glob(pattern) - for (const path of glob.scanSync({ cwd: ROOT, absolute: true, onlyFiles: true })) { - if ( - path.includes('/node_modules/') || - /\.test\.[tj]sx?$/.test(path) || - path.includes('/__fixtures__/') - ) { - continue - } - files.add(path) - } - } - return [...files].sort() +export type CatalogCheckOptions = { + /** + * Release sign-off: every bundled entry must be human-reviewed. Ordinary CI + * accepts AI drafts that are `fuzzy` + `Review: required`; they are kept out + * of the compiled packs, so the UI shows English until a person approves. + */ + requireReviewed?: boolean } -function bypassedLiterals(): string[] { - const warnings: string[] = [] - const paths = sourceFiles().filter((path) => - CONVERTED_SURFACES.some((surface) => { - const absolute = join(ROOT, surface) - return path === absolute || path.startsWith(`${absolute}/`) +export function pendingReview(sources: SourceMessage[]): Record { + return Object.fromEntries( + TARGET_LOCALES.map((locale) => { + const entries = readTranslationState(parsePo(readFileSync(catalogPath(locale), 'utf8'))) + const pending = sources.filter((source) => { + const entry = entries[source.id] + return entry && (entry.fuzzy || entry.review !== 'human') + }).length + return [locale, pending] }), ) +} - for (const path of paths) { - const source = ts.createSourceFile( - path, - readFileSync(path, 'utf8'), - ts.ScriptTarget.Latest, - true, - ts.ScriptKind.TSX, - ) - const warning = (node: ts.Node, text: string) => { - if (!/[A-Za-z]{2}/.test(text)) return - const line = source.getLineAndCharacterOfPosition(node.getStart()).line + 1 - warnings.push(`${relative(ROOT, path)}:${line}: "${text.trim()}"`) - } - const visit = (node: ts.Node): void => { - if (ts.isJsxText(node)) warning(node, node.text) - if ( - ts.isJsxAttribute(node) && - ['aria-label', 'title', 'placeholder', 'label', 'alt'].includes(node.name.text) && - node.initializer && - ts.isStringLiteral(node.initializer) - ) { - warning(node, node.initializer.text) - } - ts.forEachChild(node, visit) - } - visit(source) +export type LocaleEntries = Record> + +function readLocaleEntries(read: (path: string) => string | undefined): LocaleEntries { + const entries: LocaleEntries = {} + for (const locale of TARGET_LOCALES) { + const text = read(catalogPath(locale)) + if (text !== undefined) entries[locale] = readTranslationState(parsePo(text)) } - return warnings + return entries } -function checkCatalogMessages(sources: SourceMessage[], glossary: Glossary): string[] { +/** Pure completeness/validity/review rules for the bundled target catalogs. */ +export function checkTranslationEntries( + sources: SourceMessage[], + entriesByLocale: LocaleEntries, + glossary: Glossary, + options: CatalogCheckOptions = { requireReviewed: true }, +): string[] { const errors: string[] = [] const sourceById = new Map(sources.map((source) => [source.id, source])) for (const source of sources) { @@ -646,20 +817,28 @@ function checkCatalogMessages(sources: SourceMessage[], glossary: Glossary): str } for (const locale of TARGET_LOCALES) { - const po = parsePo(readFileSync(catalogPath(locale), 'utf8')) - const entries = readTranslationState(po) + const entries = entriesByLocale[locale] ?? {} for (const source of sources) { const entry = entries[source.id] if (!entry?.translation) { errors.push(`${locale}: missing "${source.id}"`) continue } - if (entry.sourceHash !== hashSource(source.message)) { + const awaitingReview = entry.fuzzy || entry.review !== 'human' + const stale = entry.sourceHash !== hashSource(source.message) + if (stale && !(awaitingReview && !options.requireReviewed)) { errors.push(`${locale}: stale "${source.id}"`) } - if (entry.fuzzy || entry.review !== 'human') { + if (awaitingReview && options.requireReviewed) { errors.push(`${locale}: "${source.id}" still requires human review`) + } else if (awaitingReview && !(entry.fuzzy && entry.review === 'required')) { + errors.push( + `${locale}: "${source.id}" is unreviewed but not marked fuzzy + Review: required`, + ) } + // A stale draft translates the previous English; it is never shipped and + // is compared with the current English when it is translated again. + if (stale && awaitingReview) continue for (const error of validateTranslation( source.message, entry.translation, @@ -676,6 +855,263 @@ function checkCatalogMessages(sources: SourceMessage[], glossary: Glossary): str return errors } +/** + * A translation a person approved on the base branch stays approved while its + * English is unchanged: a PR may edit it (still `Review: human`) but never + * send it back to draft, which would silently drop it from the shipped pack. + */ +export function reviewRegressions( + sources: SourceMessage[], + current: LocaleEntries, + base: LocaleEntries, +): string[] { + const errors: string[] = [] + for (const locale of TARGET_LOCALES) { + for (const source of sources) { + const before = base[locale]?.[source.id] + if ( + !before?.translation || + before.fuzzy || + before.review !== 'human' || + before.sourceHash !== hashSource(source.message) + ) { + continue + } + const after = current[locale]?.[source.id] + if (!after?.translation || after.fuzzy || after.review !== 'human') { + errors.push( + `${locale}: "${source.id}" is reviewed on the base branch with the same English and must stay reviewed`, + ) + } + } + // A declared rename must arrive with the reviewed translation; + // `i18n:extract` carries it over when the English is unchanged. + const sourceIds = new Set(sources.map((source) => source.id)) + for (const source of sources) { + const after = current[locale]?.[source.id] + const removedId = after?.renamedFrom + if (!removedId || sourceIds.has(removedId)) continue + const before = base[locale]?.[removedId] + if ( + !before?.translation || + before.fuzzy || + before.review !== 'human' || + before.sourceHash !== hashSource(source.message) + ) { + continue + } + const carried = + after.translation === before.translation && after.review === 'human' && !after.fuzzy + if (!carried) { + errors.push( + `${locale}: "${source.id}" is declared renamed-from "${removedId}", whose reviewed translation was not carried over; run bun run i18n:extract`, + ) + } + } + } + return errors +} + +/** + * Reviewed IDs removed while a new, undeclared ID has the same English. That + * may be a rename or an unrelated addition; a missed carry-over only falls back + * to English, so it is reported, not enforced. + */ +export function possibleRenames( + sources: SourceMessage[], + current: LocaleEntries, + base: LocaleEntries, +): string[] { + const sourceIds = new Set(sources.map((source) => source.id)) + const pairs = new Set() + for (const locale of TARGET_LOCALES) { + const baseEntries = base[locale] ?? {} + for (const [removedId, before] of Object.entries(baseEntries)) { + if (sourceIds.has(removedId) || before.fuzzy || before.review !== 'human') continue + for (const source of sources) { + if (source.id in baseEntries || hashSource(source.message) !== before.sourceHash) continue + const after = current[locale]?.[source.id] + if (after?.renamedFrom || (after?.review === 'human' && !after.fuzzy)) continue + pairs.add(`"${removedId}" → "${source.id}"`) + } + } + } + return [...pairs].map( + (pair) => + `possible rename ${pair} (same English): carry over the translation if intended with comment: 'renamed-from: ' and bun run i18n:extract`, + ) +} + +function isApproved(entry: TranslationEntry | undefined, source: SourceMessage): boolean { + return Boolean( + entry?.translation && + !entry.fuzzy && + entry.review === 'human' && + entry.sourceHash === hashSource(source.message), + ) +} + +/** The compiled module a browser loads for one locale, as message IDs. */ +export function readCompiledPack(locale: string): Record { + const text = readFileSync(compiledCatalogPath(locale), 'utf8') + const literal = text.match(/JSON\.parse\(("(?:[^"\\]|\\.)*")\)/)?.[1] + if (!literal) + throw new Error(`${relative(ROOT, compiledCatalogPath(locale))} has no compiled messages`) + return JSON.parse(JSON.parse(literal) as string) as Record +} + +/** + * Inspects the shipped packs themselves: a draft (fuzzy, `Review: required`, + * unreviewed or stale) must never be compiled in, whatever produced the file. + */ +export function packLeaks( + sources: SourceMessage[], + current: LocaleEntries, + packs: Record>, +): string[] { + const byId = new Map(sources.map((source) => [source.id, source])) + const errors: string[] = [] + for (const locale of TARGET_LOCALES) { + for (const id of Object.keys(packs[locale] ?? {})) { + const source = byId.get(id) + if (!source || !isApproved(current[locale]?.[id], source)) { + errors.push(`${locale}: compiled pack ships unreviewed "${id}"`) + } + } + } + return errors +} + +export function draftCounts( + sources: SourceMessage[], + current: LocaleEntries, +): Record { + return Object.fromEntries( + TARGET_LOCALES.map((locale) => [ + locale, + sources.filter((source) => !isApproved(current[locale]?.[source.id], source)).length, + ]), + ) +} + +function writeJobSummary( + sources: SourceMessage[], + current: LocaleEntries, + base?: string, + warnings: string[] = [], +): void { + const path = process.env.GITHUB_STEP_SUMMARY + if (!path) return + const counts = draftCounts(sources, current) + const rows = TARGET_LOCALES.map((locale) => `| ${locale} | ${counts[locale]} |`) + appendFileSync( + path, + [ + '### Translations awaiting review', + '', + `Drafts ship as English until reviewed. ${sources.length} messages${base ? `; compared with ${base}` : ''}.`, + '', + '| Locale | Drafts |', + '| --- | --- |', + ...rows, + '', + ...(warnings.length > 0 + ? ['**Warnings**', '', ...warnings.map((warning) => `- ${warning}`), ''] + : []), + ].join('\n'), + ) +} + +function currentEntries(): LocaleEntries { + return readLocaleEntries((path) => readFileSync(path, 'utf8')) +} + +function compiledPacks(): Record> { + return Object.fromEntries(TARGET_LOCALES.map((locale) => [locale, readCompiledPack(locale)])) +} + +/** Regression and leak checks shared by `check` (CI) and `release-check` (release). */ +export function releaseCheck( + sources: SourceMessage[], + base: LocaleEntries | undefined, +): { errors: string[]; warnings: string[] } { + const current = currentEntries() + return { + errors: [ + ...(base ? reviewRegressions(sources, current, base) : []), + ...packLeaks(sources, current, compiledPacks()), + ], + warnings: base ? possibleRenames(sources, current, base) : [], + } +} + +/** The package whose release carries the bundled catalogs. */ +const CATALOG_PACKAGE = 'editor' + +function hasCatalogs(revision: string): boolean { + return TARGET_LOCALES.every( + (locale) => readAtRevision(revision, catalogPath(locale)) !== undefined, + ) +} + +/** + * What a release is compared with: the latest `@pascal-app/editor@*` tag + * whose tree has the catalogs; else `origin/main` when it is another commit + * with catalogs; else the commit that introduced the catalogs, whose reviewed + * entries must survive. Never the working tree, and never a base without + * catalogs: with none available this throws. + */ +export function releaseBase(requested?: string): { revision: string; label: string } { + const head = headRevision() + if (requested) { + const revision = resolveRevision(requested) + if (!revision) throw new Error(`release-check: --base ${requested} is not a commit`) + if (!hasCatalogs(revision)) { + throw new Error( + `release-check: --base ${requested} has no bundled PO catalogs to compare with`, + ) + } + return { revision, label: requested } + } + const tag = lastReleaseTag(CATALOG_PACKAGE) + if (tag && hasCatalogs(tag.revision)) return { revision: tag.revision, label: tag.tag } + const reason = tag ? `${tag.tag} has no catalogs` : 'no release tag' + const main = resolveRevision('origin/main') + if (main && main !== head && hasCatalogs(main)) { + return { revision: main, label: `origin/main (${reason})` } + } + const introduced = firstCommitWith(catalogPath('fr')) + if (introduced && introduced !== head && hasCatalogs(introduced)) { + return { + revision: introduced, + label: `${introduced.slice(0, 12)}, the commit that introduced the catalogs (${reason})`, + } + } + throw new Error( + `release-check: no base with bundled PO catalogs (${reason}; origin/main and the catalogs' first commit are unavailable or HEAD); pass --base `, + ) +} + +function checkCatalogMessages( + sources: SourceMessage[], + glossary: Glossary, + options: CatalogCheckOptions = { requireReviewed: true }, +): string[] { + const current = currentEntries() + const base = baseRevision() + const release = releaseCheck( + sources, + base ? readLocaleEntries((path) => readAtRevision(base, path)) : undefined, + ) + reportWarnings(release.warnings) + writeJobSummary(sources, current, base?.slice(0, 12), release.warnings) + return [...checkTranslationEntries(sources, current, glossary, options), ...release.errors] +} + +function reportWarnings(warnings: string[]): void { + for (const warning of warnings) console.warn(`[i18n warning] ${warning}`) +} + function checkEnglishCatalog( extractedSources: SourceMessage[], catalogSources: SourceMessage[], @@ -714,15 +1150,16 @@ function checkCompiledCatalogs(sources: SourceMessage[]): string[] { }) } -export async function checkCatalogs(): Promise { +export async function checkCatalogs(options: CatalogCheckOptions = {}): Promise { const sources = await extractSourceMessages() const catalogSources = readSourceMessages() const glossary = readJson(GLOSSARY_PATH) return [ ...checkLinguiCatalogSync(), ...checkEnglishCatalog(sources, catalogSources), - ...checkCatalogMessages(sources, glossary), + ...checkCatalogMessages(sources, glossary, options), ...checkCompiledCatalogs(catalogSources), + ...checkPseudoCatalog(catalogSources), ] } @@ -740,16 +1177,66 @@ async function main(): Promise { return } - if (command === 'check') { - const errors = await checkCatalogs() - for (const literal of bypassedLiterals()) { - console.warn(`[i18n warning] user-facing literal bypasses Lingui: ${literal}`) + if (command === 'release-check') { + const flag = (name: string) => { + const index = process.argv.indexOf(`--${name}`) + return index === -1 ? undefined : process.argv[index + 1] + } + const baseDir = flag('base-dir') + let resolved: { revision: string; label: string } | undefined + try { + resolved = baseDir ? undefined : releaseBase(flag('base')) + } catch (error) { + console.error(`[i18n] ${error instanceof Error ? error.message : String(error)}`) + process.exitCode = 1 + return + } + const revision = resolved?.revision + if (resolved) console.log(`Comparing with ${resolved.label}`) + const base = baseDir + ? readLocaleEntries((path) => { + const file = join(resolve(baseDir), relative(CATALOG_ROOT, path)) + return existsSync(file) ? readFileSync(file, 'utf8') : undefined + }) + : readLocaleEntries((path) => readAtRevision(revision!, path)) + const sources = readSourceMessages() + const { errors, warnings } = releaseCheck(sources, base) + reportWarnings(warnings) + writeJobSummary(sources, currentEntries(), resolved?.label ?? baseDir, warnings) + if (errors.length > 0) { + console.error(errors.map((error) => `[i18n] ${error}`).join('\n')) + process.exitCode = 1 + return } + console.log( + `No reviewed translation regressed against ${baseDir ?? resolved?.label} and no draft is compiled in`, + ) + return + } + + if (command === 'check') { + const requireReviewed = process.argv.includes('--reviewed') + const errors = await checkCatalogs({ requireReviewed }) + const literals = checkRepositoryLiterals() + errors.push(...literals.errors.map((error) => `literal guard: ${error}`)) if (errors.length > 0) { console.error(errors.map((error) => `[i18n] ${error}`).join('\n')) process.exitCode = 1 - } else { - console.log(`Checked ${TARGET_LOCALES.length} complete bundled PO catalogs`) + return + } + const pending = Object.entries(pendingReview(readSourceMessages())) + .filter(([, count]) => count > 0) + .map(([locale, count]) => `${locale} ${count}`) + console.log( + `Checked ${TARGET_LOCALES.length} bundled PO catalogs and ${literals.total} legacy literal candidates` + + (pending.length > 0 + ? `; awaiting human review (shown in English until approved): ${pending.join(', ')}` + : ''), + ) + if (literals.improvable.length > 0) { + console.log( + `[i18n] ${literals.improvable.length} files are below their literal baseline; run bun run i18n:literals -- update`, + ) } return } @@ -781,7 +1268,9 @@ async function main(): Promise { return } - throw new Error('Usage: bun scripts/i18n.ts [locale...]') + throw new Error( + 'Usage: bun scripts/i18n.ts |--base-dir ]|translate> [locale...]', + ) } if (import.meta.main) await main()