Skip to content

Commit 121489f

Browse files
dhalbertdhalbert
authored
Merge pull request adafruit#11326 from MakerClassCZ/ci-language-predict
build_release_files: predict translation sizes instead of building them on PRs
2 parents 880082b + e88d2d6 commit 121489f

1 file changed

Lines changed: 146 additions & 10 deletions

File tree

‎tools/build_release_files.py‎

Lines changed: 146 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import os
88
import multiprocessing
9+
import re
910
import sys
1011
import subprocess
1112
import shutil
@@ -35,6 +36,91 @@
3536
LANGUAGE_FIRST = "en_US"
3637
LANGUAGE_THRESHOLD = 10 * 1024
3738

39+
# On pull requests the translations other than en_US are built only to prove that they
40+
# still fit in flash. The compiled code is identical for every translation; only three
41+
# generated data files differ: the compressed strings, the compression dictionary and
42+
# the terminal font. So instead of relinking the firmware for each translation, generate
43+
# those files, count their bytes and predict the flash usage. Only translations predicted
44+
# within LANGUAGE_MARGIN of the region limit, or whose build configuration differs, are
45+
# really built. LANGUAGE_PREDICT=dryrun builds everything and prints the prediction
46+
# error; LANGUAGE_PREDICT=off restores the old behaviour.
47+
LANGUAGE_MARGIN = int(os.environ.get("LANGUAGE_MARGIN", 1024))
48+
LANGUAGE_PREDICT = os.environ.get("LANGUAGE_PREDICT", "skip")
49+
50+
C_TYPE_SIZES = {
51+
"char": 1,
52+
"int8_t": 1,
53+
"uint8_t": 1,
54+
"int16_t": 2,
55+
"uint16_t": 2,
56+
"int32_t": 4,
57+
"uint32_t": 4,
58+
}
59+
C_ARRAY_RE = re.compile(r"const\s+(\w+)\s+\w+\[\d*\]\s*=\s*\{([^}]*)\}")
60+
TRANSLATION_RE = re.compile(r"\.data = \d+, \.tail = \{([^}]*)\}")
61+
62+
63+
def flash_usage(port, build_dir):
64+
"""Return (used, region) bytes of the firmware flash region, or (None, None) if unknown."""
65+
try:
66+
with open(f"../ports/{port}/{build_dir}/firmware.size.json", "r") as f:
67+
firmware = json.load(f)
68+
return firmware["used_flash"], firmware["firmware_region"]
69+
except FileNotFoundError:
70+
return None, None
71+
72+
73+
def c_array_bytes(path):
74+
"""Sum the bytes of the const arrays initialised in a generated C file."""
75+
if not path.exists():
76+
return 0
77+
text = path.read_text()
78+
total = 0
79+
for match in C_ARRAY_RE.finditer(text):
80+
c_type, body = match.groups()
81+
if c_type not in C_TYPE_SIZES:
82+
typedef = re.search(r"typedef\s+(\w+)\s+" + c_type + ";", text)
83+
c_type = typedef.group(1) if typedef else "uint8_t"
84+
total += C_TYPE_SIZES[c_type] * len([x for x in body.split(",") if x.strip()])
85+
return total
86+
87+
88+
def translation_bytes(port, build_dir, language):
89+
"""Bytes of flash that depend on the translation: strings, dictionary and font."""
90+
build = pathlib.Path(f"../ports/{port}/{build_dir}")
91+
strings = 0
92+
for match in TRANSLATION_RE.finditer(
93+
(build / "py" / f"translations-{language}.c").read_text()
94+
):
95+
strings += 1 + len([x for x in match.group(1).split(",") if x.strip()])
96+
dictionary = c_array_bytes(build / "genhdr" / "compressed_translations.generated.h")
97+
font = c_array_bytes(build / f"autogen_display_resources-{language}.c")
98+
return strings + dictionary + font
99+
100+
101+
def generate_translation(port, board, build_dir, language):
102+
"""Generate the translation data files of a language without compiling anything."""
103+
targets = [f"{build_dir}/py/translations-{language}.c"]
104+
if os.path.exists(f"../ports/{port}/{build_dir}/autogen_display_resources-{LANGUAGE_FIRST}.c"):
105+
targets.append(f"{build_dir}/autogen_display_resources-{language}.c")
106+
result = subprocess.run(
107+
"make -C ../ports/{port} TRANSLATION={language} BOARD={board} BUILD={build} -j {cores} {targets}".format(
108+
port=port,
109+
language=language,
110+
board=board,
111+
build=build_dir,
112+
cores=cores,
113+
targets=" ".join(targets),
114+
),
115+
shell=True,
116+
stdout=subprocess.PIPE,
117+
stderr=subprocess.STDOUT,
118+
)
119+
if result.returncode != 0:
120+
print(result.stdout.decode("utf-8"))
121+
return result.returncode == 0
122+
123+
38124
languages = build_info.get_languages()
39125

40126
all_languages = build_info.get_languages(list_all=True)
@@ -71,6 +157,13 @@
71157
languages.remove(LANGUAGE_FIRST)
72158
languages.insert(0, LANGUAGE_FIRST)
73159

160+
# Set after the first language's build when its flash usage is known and too tight to
161+
# skip the other languages outright: the flash that build used, the flash the region
162+
# holds, and how many of the used bytes are translation data.
163+
baseline_flash = None
164+
flash_region = 0
165+
baseline_translation_bytes = 0
166+
74167
for language in languages:
75168
bin_directory = "../bin/{board}/{language}".format(board=board, language=language)
76169
os.makedirs(bin_directory, exist_ok=True)
@@ -99,6 +192,34 @@
99192
extensions = board_settings["CIRCUITPY_BUILD_EXTENSIONS"]
100193

101194
artifacts = [os.path.join(build_dir, "firmware." + extension) for extension in extensions]
195+
196+
predicted_flash = None
197+
if baseline_flash is not None and language != LANGUAGE_FIRST and not clean_build:
198+
if generate_translation(board_info["port"], board, build_dir, language):
199+
translation_growth = (
200+
translation_bytes(board_info["port"], build_dir, language)
201+
- baseline_translation_bytes
202+
)
203+
predicted_flash = baseline_flash + translation_growth
204+
fits = predicted_flash + LANGUAGE_MARGIN <= flash_region
205+
skip = fits and LANGUAGE_PREDICT == "skip"
206+
print(
207+
"Predicted flash size for {board} {language}: {predicted} of {region} bytes"
208+
" ({free} free, {growth:+d} vs {first}) -> {action}".format(
209+
board=board,
210+
language=language,
211+
predicted=predicted_flash,
212+
region=flash_region,
213+
free=flash_region - predicted_flash,
214+
growth=translation_growth,
215+
first=LANGUAGE_FIRST,
216+
action="skip" if skip else "build",
217+
),
218+
flush=True,
219+
)
220+
if skip:
221+
continue
222+
102223
make_result = subprocess.run(
103224
"make -C ../ports/{port} TRANSLATION={language} BOARD={board} BUILD={build} -j {cores} {artifacts}".format(
104225
port=board_info["port"],
@@ -154,19 +275,34 @@
154275
print(make_result.stdout.decode("utf-8"))
155276
print(other_output)
156277

278+
if predicted_flash is not None and make_result.returncode == 0:
279+
actual_flash, _ = flash_usage(board_info["port"], build_dir)
280+
if actual_flash is not None:
281+
print(
282+
"Flash size check {board} {language}: predicted {predicted},"
283+
" actual {actual}, error {error:+d}".format(
284+
board=board,
285+
language=language,
286+
predicted=predicted_flash,
287+
actual=actual_flash,
288+
error=predicted_flash - actual_flash,
289+
)
290+
)
291+
157292
# Flush so we will see something before 10 minutes has passed.
158293
print(flush=True)
159294

160295
if (not build_all) and (language == LANGUAGE_FIRST) and (exit_status == 0):
161-
try:
162-
with open(
163-
f"../ports/{board_info['port']}/{build_dir}/firmware.size.json", "r"
164-
) as f:
165-
firmware = json.load(f)
166-
if firmware["used_flash"] + LANGUAGE_THRESHOLD < firmware["firmware_region"]:
167-
print("Skipping languages")
168-
break
169-
except FileNotFoundError:
170-
pass
296+
used_flash, flash_region = flash_usage(board_info["port"], build_dir)
297+
if used_flash is None:
298+
print("Flash usage unknown, building all languages")
299+
elif used_flash + LANGUAGE_THRESHOLD < flash_region:
300+
print("Skipping languages")
301+
break
302+
elif LANGUAGE_PREDICT != "off" and board_info["port"] != "zephyr-cp":
303+
baseline_flash = used_flash
304+
baseline_translation_bytes = translation_bytes(
305+
board_info["port"], build_dir, LANGUAGE_FIRST
306+
)
171307

172308
sys.exit(exit_status)

0 commit comments

Comments
 (0)