From 9c1da6e2ad54cd3dde752edaedb9bfc22493c148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= <1005065+DeepDiver1975@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:02:27 +0200 Subject: [PATCH] ci: gate against duplicate keys in l10n catalogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both JSON.parse and PHP's json_decode silently collapse duplicate object keys (last value wins), so a catalog can carry the same key twice and still parse as valid JSON. That is how the duplicate de/de_DE empty-state keys reached master and needed the follow-up in #562. Add tests/l10n/check-duplicate-keys.py, which parses every l10n/*.{js,json} catalog with an object_pairs_hook that inspects all key/value pairs before deduplication and fails on any repeated key (covering the nested translations object of the .json format and the OC.L10N.register wrapper of the .js format). Wire it up as a 'make test-l10n' target and a dedicated l10n-lint GitHub Actions workflow so it runs on every push and pull request. Verified the checker flags the exact duplicates from the pre-#562 tree. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Thomas Müller <1005065+DeepDiver1975@users.noreply.github.com> --- .github/workflows/l10n-lint.yml | 28 ++++++ Makefile | 5 ++ tests/l10n/check-duplicate-keys.py | 132 +++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 .github/workflows/l10n-lint.yml create mode 100644 tests/l10n/check-duplicate-keys.py diff --git a/.github/workflows/l10n-lint.yml b/.github/workflows/l10n-lint.yml new file mode 100644 index 00000000..bea4f928 --- /dev/null +++ b/.github/workflows/l10n-lint.yml @@ -0,0 +1,28 @@ +name: l10n lint + +on: + push: + branches: + - master + pull_request: + types: + - opened + - synchronize + - reopened + +permissions: + contents: read + +concurrency: + group: l10n-lint-${{ github.ref }} + cancel-in-progress: true + +jobs: + duplicate-keys: + name: Duplicate translation keys + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check l10n catalogs for duplicate keys + run: make test-l10n diff --git a/Makefile b/Makefile index e19502a8..67a5ed3d 100644 --- a/Makefile +++ b/Makefile @@ -236,6 +236,11 @@ test-js: ## Test js files test-js: npm cd js && npm run test +.PHONY: test-l10n +test-l10n: ## Check l10n catalogs for duplicate translation keys +test-l10n: + python3 tests/l10n/check-duplicate-keys.py + .PHONY: test-acceptance-webui test-acceptance-webui: ## Run webUI acceptance tests test-acceptance-webui: $(acceptance_test_deps) diff --git a/tests/l10n/check-duplicate-keys.py b/tests/l10n/check-duplicate-keys.py new file mode 100644 index 00000000..0e67e5c7 --- /dev/null +++ b/tests/l10n/check-duplicate-keys.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Fail if any l10n catalog contains a duplicate translation key. + +Both ``JSON.parse`` (JS) and PHP ``json_decode`` silently collapse duplicate +object keys (last value wins), so a catalog can carry the same key twice and +still "validate" while being malformed. That is exactly how the duplicate +``de``/``de_DE`` empty-state keys reached master (see PR #562). This checker +inspects *all* key/value pairs before deduplication via ``object_pairs_hook`` +and reports any key that appears more than once. + +Covers both catalog formats: + * ``l10n/.json`` -- a plain JSON object. + * ``l10n/.js`` -- ``OC.L10N.register("notes", { ... }, "plural...");`` + where the ``{ ... }`` object literal is JSON. + +Exit code 0 when every catalog is clean, 1 when duplicates are found (or a +catalog cannot be parsed). +""" + +import json +import sys +from pathlib import Path + +# Repo root is two levels up from tests/l10n/. +REPO_ROOT = Path(__file__).resolve().parents[2] +L10N_DIR = REPO_ROOT / "l10n" + + +def make_pairs_hook(collector): + """Build an object_pairs_hook that appends duplicate keys to ``collector``. + + The hook runs for *every* object in the document (including nested ones such + as the ``translations`` wrapper in the .json catalogs) and fires before + Python collapses duplicate keys, so it sees all of them. + """ + def hook(pairs): + seen = set() + for key, _value in pairs: + if key in seen: + collector.append(key) + else: + seen.add(key) + return dict(pairs) + return hook + + +def extract_js_object(text): + """Return the JSON object literal embedded in an OC.L10N.register call. + + The catalog is ``OC.L10N.register("notes", { ... }, "plural...");``. We take + the first ``{`` and its matching ``}`` (tracking string literals so braces + inside translations are ignored) and return that substring. + """ + start = text.find("{") + if start == -1: + raise ValueError("no '{' found in .js catalog") + + depth = 0 + in_string = False + escaped = False + for i in range(start, len(text)): + ch = text[i] + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return text[start:i + 1] + raise ValueError("unbalanced braces in .js catalog") + + +def duplicates_in_file(path): + """Return the list of duplicate keys in one catalog file. + + Collects duplicates from every object in the document, so a duplicate inside + the nested ``translations`` object of a .json catalog is caught too. + """ + text = path.read_text(encoding="utf-8") + if path.suffix == ".js": + text = extract_js_object(text) + dups = [] + json.loads(text, object_pairs_hook=make_pairs_hook(dups)) + return dups + + +def main(): + catalogs = sorted( + p for p in L10N_DIR.iterdir() + if p.suffix in (".js", ".json") + ) + if not catalogs: + print(f"error: no catalogs found in {L10N_DIR}", file=sys.stderr) + return 1 + + failures = [] + for path in catalogs: + rel = path.relative_to(REPO_ROOT) + try: + dups = duplicates_in_file(path) + except (ValueError, json.JSONDecodeError) as exc: + failures.append(f"{rel}: could not parse ({exc})") + continue + if dups: + for key in dups: + failures.append(f"{rel}: duplicate key {json.dumps(key, ensure_ascii=False)}") + + if failures: + print("Duplicate translation keys detected:", file=sys.stderr) + for line in failures: + print(f" {line}", file=sys.stderr) + print( + f"\n{len(failures)} problem(s) across {len(catalogs)} catalog files.", + file=sys.stderr, + ) + return 1 + + print(f"OK: no duplicate keys in {len(catalogs)} l10n catalog files.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())