build_release_files: predict translation sizes instead of building them on PRs

On pull requests the translations other than en_US are built only to
prove that they still fit in flash. The compiled code is identical for
every translation; only three generated data files differ: the
compressed strings, the compression dictionary and the terminal font.

When the first language leaves less than 10 KB of headroom, run only
the generators (the make targets for translations-<lang>.c and
autogen_display_resources-<lang>.c, about 4 s, no compiler), count the
bytes of the generated tables and predict the flash usage. Only
translations predicted within LANGUAGE_MARGIN (1 KB) of the limit, or
whose build configuration differs (clean build), are really built.
LANGUAGE_PREDICT=dryrun builds everything and prints the prediction
error; LANGUAGE_PREDICT=off restores the old behaviour.

Measured on feather_m4_can, pybadge and metro_m4_express with every
language (41 predictions): error -5..+174 bytes, almost always an
over-estimate. On metro_m4_express the prediction flagged fr as not
fitting; the real build then overflowed by 56 bytes. feather_m4_can
board job: 443 s -> 228 s; the remainder is the two clean builds.
Push and release builds are unchanged.

Author: lynt-smitka

tools/build_release_files.py

@@ -6,6 +6,7 @@
 
 import os
 import multiprocessing
+import re
 import sys
 import subprocess
 import shutil
@@ -35,6 +36,91 @@
 LANGUAGE_FIRST = "en_US"
 LANGUAGE_THRESHOLD = 10 * 1024
 
+# On pull requests the translations other than en_US are built only to prove that they
+# still fit in flash. The compiled code is identical for every translation; only three
+# generated data files differ: the compressed strings, the compression dictionary and
+# the terminal font. So instead of relinking the firmware for each translation, generate
+# those files, count their bytes and predict the flash usage. Only translations predicted
+# within LANGUAGE_MARGIN of the region limit, or whose build configuration differs, are
+# really built. LANGUAGE_PREDICT=dryrun builds everything and prints the prediction
+# error; LANGUAGE_PREDICT=off restores the old behaviour.
+LANGUAGE_MARGIN = int(os.environ.get("LANGUAGE_MARGIN", 1024))
+LANGUAGE_PREDICT = os.environ.get("LANGUAGE_PREDICT", "skip")
+
+C_TYPE_SIZES = {
+    "char": 1,
+    "int8_t": 1,
+    "uint8_t": 1,
+    "int16_t": 2,
+    "uint16_t": 2,
+    "int32_t": 4,
+    "uint32_t": 4,
+}
+C_ARRAY_RE = re.compile(r"const\s+(\w+)\s+\w+\[\d*\]\s*=\s*\{([^}]*)\}")
+TRANSLATION_RE = re.compile(r"\.data = \d+, \.tail = \{([^}]*)\}")
+
+
+def flash_usage(port, build_dir):
+    """Return (used, region) bytes of the firmware flash region, or None if unknown."""
+    try:
+        with open(f"../ports/{port}/{build_dir}/firmware.size.json", "r") as f:
+            firmware = json.load(f)
+        return firmware["used_flash"], firmware["firmware_region"]
+    except FileNotFoundError:
+        return None
+
+
+def c_array_bytes(path):
+    """Sum the bytes of the const arrays initialised in a generated C file."""
+    if not path.exists():
+        return 0
+    text = path.read_text()
+    total = 0
+    for match in C_ARRAY_RE.finditer(text):
+        c_type, body = match.groups()
+        if c_type not in C_TYPE_SIZES:
+            typedef = re.search(r"typedef\s+(\w+)\s+" + c_type + ";", text)
+            c_type = typedef.group(1) if typedef else "uint8_t"
+        total += C_TYPE_SIZES[c_type] * len([x for x in body.split(",") if x.strip()])
+    return total
+
+
+def translation_bytes(port, build_dir, language):
+    """Bytes of flash that depend on the translation: strings, dictionary and font."""
+    build = pathlib.Path(f"../ports/{port}/{build_dir}")
+    strings = 0
+    for match in TRANSLATION_RE.finditer(
+        (build / "py" / f"translations-{language}.c").read_text()
+    ):
+        strings += 1 + len([x for x in match.group(1).split(",") if x.strip()])
+    dictionary = c_array_bytes(build / "genhdr" / "compressed_translations.generated.h")
+    font = c_array_bytes(build / f"autogen_display_resources-{language}.c")
+    return strings + dictionary + font
+
+
+def generate_translation(port, board, build_dir, language):
+    """Generate the translation data files of a language without compiling anything."""
+    targets = [f"{build_dir}/py/translations-{language}.c"]
+    if os.path.exists(f"../ports/{port}/{build_dir}/autogen_display_resources-{LANGUAGE_FIRST}.c"):
+        targets.append(f"{build_dir}/autogen_display_resources-{language}.c")
+    result = subprocess.run(
+        "make -C ../ports/{port} TRANSLATION={language} BOARD={board} BUILD={build} -j {cores} {targets}".format(
+            port=port,
+            language=language,
+            board=board,
+            build=build_dir,
+            cores=cores,
+            targets=" ".join(targets),
+        ),
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    if result.returncode != 0:
+        print(result.stdout.decode("utf-8"))
+    return result.returncode == 0
+
+
 languages = build_info.get_languages()
 
 all_languages = build_info.get_languages(list_all=True)
@@ -71,6 +157,11 @@
     languages.remove(LANGUAGE_FIRST)
     languages.insert(0, LANGUAGE_FIRST)
 
+    # Set after the first language when its flash usage is known and too tight to skip
+    # the other languages outright.
+    predict_flash = None
+    first_language_bytes = 0
+
     for language in languages:
         bin_directory = "../bin/{board}/{language}".format(board=board, language=language)
         os.makedirs(bin_directory, exist_ok=True)
@@ -99,6 +190,34 @@
         extensions = board_settings["CIRCUITPY_BUILD_EXTENSIONS"]
 
         artifacts = [os.path.join(build_dir, "firmware." + extension) for extension in extensions]
+
+        prediction = None
+        if predict_flash is not None and language != LANGUAGE_FIRST and not clean_build:
+            if generate_translation(board_info["port"], board, build_dir, language):
+                delta = (
+                    translation_bytes(board_info["port"], build_dir, language)
+                    - first_language_bytes
+                )
+                predicted = predict_flash[0] + delta
+                fits = predicted + LANGUAGE_MARGIN <= predict_flash[1]
+                skip = fits and LANGUAGE_PREDICT == "skip"
+                prediction = predicted
+                print(
+                    "Predict {board} for {language}: {predicted} of {region} bytes ({free} free, {delta:+d} vs {first}) -> {action}".format(
+                        board=board,
+                        language=language,
+                        predicted=predicted,
+                        region=predict_flash[1],
+                        free=predict_flash[1] - predicted,
+                        delta=delta,
+                        first=LANGUAGE_FIRST,
+                        action="skip" if skip else "build",
+                    ),
+                    flush=True,
+                )
+                if skip:
+                    continue
+
         make_result = subprocess.run(
             "make -C ../ports/{port} TRANSLATION={language} BOARD={board} BUILD={build} -j {cores} {artifacts}".format(
                 port=board_info["port"],
@@ -154,19 +273,33 @@
         print(make_result.stdout.decode("utf-8"))
         print(other_output)
 
+        if prediction is not None and make_result.returncode == 0:
+            usage = flash_usage(board_info["port"], build_dir)
+            if usage is not None:
+                print(
+                    "Predict check {board} for {language}: predicted {predicted}, actual {actual}, error {error:+d}".format(
+                        board=board,
+                        language=language,
+                        predicted=prediction,
+                        actual=usage[0],
+                        error=prediction - usage[0],
+                    )
+                )
+
         # Flush so we will see something before 10 minutes has passed.
         print(flush=True)
 
         if (not build_all) and (language == LANGUAGE_FIRST) and (exit_status == 0):
-            try:
-                with open(
-                    f"../ports/{board_info['port']}/{build_dir}/firmware.size.json", "r"
-                ) as f:
-                    firmware = json.load(f)
-                    if firmware["used_flash"] + LANGUAGE_THRESHOLD < firmware["firmware_region"]:
-                        print("Skipping languages")
-                        break
-            except FileNotFoundError:
-                pass
+            usage = flash_usage(board_info["port"], build_dir)
+            if usage is None:
+                print("Flash usage unknown, building all languages")
+            elif usage[0] + LANGUAGE_THRESHOLD < usage[1]:
+                print("Skipping languages")
+                break
+            elif LANGUAGE_PREDICT != "off" and board_info["port"] != "zephyr-cp":
+                predict_flash = usage
+                first_language_bytes = translation_bytes(
+                    board_info["port"], build_dir, LANGUAGE_FIRST
+                )
 
 sys.exit(exit_status)