From 7b2a3fc55d102719d225747a06f1e6f137b5397a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 3 Jul 2026 16:37:55 +0200 Subject: [PATCH] Add cleanup_bconds.py and remove 31 unused bcond entries The script checks each bcond in config.toml against distgit commits and koji builds to determine if it was actually used during the Python 3.15 mass rebuild, then removes unused entries in place. Assisted-By: Claude Opus 4.6 --- cleanup_bconds.py | 237 ++++++++++++++++++++++++++++++++++++++++++++++ config.toml | 104 ++------------------ 2 files changed, 245 insertions(+), 96 deletions(-) create mode 100644 cleanup_bconds.py diff --git a/cleanup_bconds.py b/cleanup_bconds.py new file mode 100644 index 0000000..6b5904b --- /dev/null +++ b/cleanup_bconds.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +""" +Remove unused bcond entries from config.toml. + +For packages not in todo.pkgs: +- "just bootstrap" packages: check koji for ~bootstrap builds since 2026-06-03 +- Other packages: check distgit for Bootstrap/Rebuilt commits + +Unused bconds are removed from config.toml in place. +""" + +import argparse +import asyncio +import os +import shutil +import sys +import tempfile +import time + +import tomllib + +def log(*args, **kwargs): + kwargs.setdefault('file', sys.stderr) + kwargs.setdefault('flush', True) + return print(*args, **kwargs) + + +def load_todo_pkgs(config): + path = config['todo']['path'] + max_age_minutes = config['todo']['max_age_minutes'] + mtime = os.path.getmtime(path) + age_seconds = time.time() - mtime + if age_seconds > max_age_minutes * 60: + age_minutes = int(age_seconds / 60) + sys.exit(f'{path} is {age_minutes} minutes old (> {max_age_minutes} min). Refresh it first.') + + with open(path) as f: + return {line.strip() for line in f if line.strip()} + + +def is_just_bootstrap(entries): + for entry in entries: + if entry.get('withouts') or entry.get('replacements'): + return False + withs = entry.get('withs', []) + if list(withs) != ['bootstrap']: + return False + return True + + +def remove_bcond_blocks(content, packages_to_remove): + lines = content.splitlines(keepends=True) + result = [] + i = 0 + while i < len(lines): + stripped = lines[i].strip() + + if stripped.startswith('[[bconds.') and stripped.endswith(']]'): + key = stripped[len('[[bconds.'):-len(']]')].strip() + if key in packages_to_remove: + i += 1 + # skip key-value lines + while i < len(lines): + s = lines[i].strip() + if s and not s.startswith('#') and not s.startswith('['): + i += 1 + else: + break + # skip trailing blank lines + while i < len(lines) and lines[i].strip() == '': + i += 1 + continue + + result.append(lines[i]) + i += 1 + + return ''.join(result) + + +async def run_cmd(*cmd, cwd=None): + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + ) + stdout, stderr = await proc.communicate() + return proc.returncode, stdout.decode(), stderr.decode() + + +async def check_distgit(pkg_name, tmpdir, config): + clone_dir = os.path.join(tmpdir, pkg_name) + clone_depth = config['distgit']['clone_depth'] + clone_retries = config['distgit']['clone_retries'] + + for attempt in range(1, clone_retries + 1): + rc, stdout, stderr = await run_cmd( + 'fedpkg', 'clone', '-a', pkg_name, clone_dir, '--', f'--depth={clone_depth}' + ) + if rc == 0: + break + log(f' WARNING: fedpkg clone failed for {pkg_name} (attempt {attempt}/{clone_retries}): {stderr.strip()}') + if attempt < clone_retries: + await asyncio.sleep(2 ** attempt) + if os.path.isdir(clone_dir): + shutil.rmtree(clone_dir) + else: + return False + + rc, stdout, stderr = await run_cmd( + 'git', '-C', clone_dir, 'log', + f'--author={config["distgit"]["author"]}', '--format=%s', '--all', + ) + if rc != 0: + log(f' WARNING: git log failed for {pkg_name}: {stderr.strip()}') + return False + + bootstrap_msg = config['distgit']['bootstrap_commit_message'] + rebuild_msg = config['distgit']['commit_message'] + has_bootstrap = False + has_rebuild = False + for subject in stdout.splitlines(): + if subject == bootstrap_msg: + has_bootstrap = True + elif subject == rebuild_msg: + has_rebuild = True + + # Unused: no bootstrap commit but was rebuilt directly + return not has_bootstrap and has_rebuild + + +async def check_koji(pkg_name, config): + builds_after = config['koji']['builds_after'] + bootstrap_marker = config['koji']['bootstrap_marker'] + rc, stdout, stderr = await run_cmd( + 'koji', 'list-builds', f'--package={pkg_name}', f'--after={builds_after}' + ) + if rc != 0: + log(f' WARNING: koji list-builds failed for {pkg_name}: {stderr.strip()}') + return False + + for line in stdout.splitlines(): + if bootstrap_marker in line: + return False # bootstrap build exists, was used + + return True # no bootstrap builds found, unused + + +async def check_package(pkg_name, entries, tmpdir, config, sem): + async with sem: + just_bootstrap = is_just_bootstrap(entries) + + if just_bootstrap: + log(f' {pkg_name}: checking koji for ~bootstrap builds...') + unused = await check_koji(pkg_name, config) + else: + log(f' {pkg_name}: checking distgit for Bootstrap/Rebuilt commits...') + unused = await check_distgit(pkg_name, tmpdir, config) + + status = 'UNUSED' if unused else 'used' + log(f' {pkg_name}: {status}') + return pkg_name, unused + + +async def main(): + parser = argparse.ArgumentParser(description='Remove unused bconds from config.toml') + parser.add_argument( + '-j', '--jobs', + type=int, + default=os.cpu_count(), + help=f'Max parallel tasks (default: {os.cpu_count()})', + ) + args = parser.parse_args() + + with open('config.toml', 'rb') as f: + config = tomllib.load(f) + + todo_pkgs = load_todo_pkgs(config) + log(f'Loaded {len(todo_pkgs)} packages from {config["todo"]["path"]}') + + bconds = config['bconds'] + + to_check = {} + skipped_todo = 0 + for pkg_name in bconds: + if pkg_name.strip() in todo_pkgs: + skipped_todo += 1 + else: + to_check[pkg_name] = bconds[pkg_name] + + log(f'Skipping {skipped_todo} packages in todo.pkgs') + log(f'Checking {len(to_check)} packages with {args.jobs} parallel jobs...') + + sem = asyncio.Semaphore(args.jobs) + to_remove = set() + + def collect_result(task): + try: + pkg_name, unused = task.result() + except (asyncio.CancelledError, Exception): + return + if unused: + to_remove.add(pkg_name) + + try: + with tempfile.TemporaryDirectory(prefix='cleanup_bconds_') as tmpdir: + tasks = [] + for pkg_name, entries in to_check.items(): + task = asyncio.ensure_future(check_package( + pkg_name.strip(), entries, + tmpdir, config, sem, + )) + task.add_done_callback(collect_result) + tasks.append(task) + await asyncio.gather(*tasks) + except (KeyboardInterrupt, asyncio.CancelledError): + log(f'\nInterrupted. {len(to_remove)} unused bconds found so far.') + finally: + if not to_remove: + log('No unused bconds found.') + return + + log(f'\nRemoving {len(to_remove)} unused bcond entries:') + for pkg_name in sorted(to_remove): + log(f' - {pkg_name}') + + with open('config.toml') as f: + content = f.read() + content = remove_bcond_blocks(content, to_remove) + with open('config.toml', 'w') as f: + f.write(content) + + log(f'\nDone. Removed {len(to_remove)} entries from config.toml.') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/config.toml b/config.toml index db08449..a08de2d 100644 --- a/config.toml +++ b/config.toml @@ -35,12 +35,20 @@ target = 'rawhide' # target = 'f45-python' bootstrap_target = 'f45-python-bootstrap' +builds_after = '2026-06-03' +bootstrap_marker = '~bootstrap' [distgit] branch = "rawhide" commit_message = "Rebuilt for Python 3.15" bootstrap_commit_message = "Bootstrap for Python 3.15" author = "Python Maint " +clone_depth = 10 +clone_retries = 3 + +[todo] +path = "todo.pkgs" +max_age_minutes = 60 [architectures] repoquery = "x86_64" # used for repository querying @@ -96,12 +104,6 @@ withouts = ["check"] [[bconds.python-setuptools_scm]] withouts = ["tests"] -[[bconds.python-py]] -withouts = ["docs", "tests"] - -[[bconds.python-chardet]] -withouts = ["doc"] - [[bconds.python-pbr]] withs = ["bootstrap"] withouts = ["tests"] @@ -163,9 +165,6 @@ withs = ["bootstrap"] [[bconds.python-jinja2]] withouts = ["docs", "asyncio_tests"] -[[bconds.python-sphinx_rtd_theme]] -withs = ["bootstrap"] - [[bconds.python-sphinx-basic-ng]] withs = ["bootstrap"] @@ -202,21 +201,12 @@ withouts = ["tests"] [[bconds.python-flit-core]] withs = ["bootstrap"] -[[bconds.python-async-timeout]] -withouts = ["tests"] - -[[bconds.python-jupyter-client]] -withs = ["bootstrap"] - [[bconds.python-jupyter-server]] withs = ["bootstrap"] [[bconds.python-matplotlib]] withouts = ["check"] -[[bconds.ipython]] -withouts = ["check","doc"] - [[bconds.python-ipykernel]] withs = ["bootstrap"] @@ -226,24 +216,12 @@ withouts = ["optional_tests"] [[bconds.python-nbconvert]] withouts = ["check", "doc"] -[[bconds.python-nbclient]] -withs = ["bootstrap"] - -[[bconds.python-pyquery]] -withouts = ["tests"] - -[[bconds.python-cherrypy]] -withouts = ["tests"] - [[bconds.freeipa-healthcheck]] withouts = ["tests"] [[bconds.python-Traits]] withs = ["bootstrap"] -[[bconds.python-pcodedmp]] -withs = ["bootstrap"] - [[bconds.python-libcst]] withs = ["bootstrap"] @@ -278,15 +256,6 @@ withs = ["bootstrap"] [[bconds.python-astropy]] withouts = ["check"] -[[bconds.python-pyerfa]] -withouts = ["tests"] - -[[bconds.python-aiohttp]] -withouts = ["tests"] - -[[bconds.python-azure-core]] -withouts = ["tests"] - # this bcond is disabled in spec on purpose # it might be enabled in the future, so leaving this here, commend out #[[bconds.python-azure-common]] @@ -295,9 +264,6 @@ withouts = ["tests"] [[bconds.python-subprocess-tee]] withouts = ["tests"] -[[bconds.python-typeguard]] -withs = ["bootstrap"] - [[bconds.python-tox]] withs = ["bootstrap"] @@ -310,24 +276,15 @@ withs = ["bootstrap"] [[bconds.python-contourpy]] withs = ["bootstrap"] -[[bconds.python-msrest]] -withouts = ["tests"] - [[bconds.python-oletools]] withs = ["bootstrap"] -[[bconds.python-google-api-core]] -withouts = ["tests"] - [[bconds.fonttools]] withouts = ["tests", "plot_extra", "symfont_extra", "ufo_extra", "woff_extra", "graphite_extra", "interpolatable_extra"] [[bconds.conda]] withs = ["bootstrap"] -[[bconds.python-conda-libmamba-solver]] -withouts = ["tests"] - [[bconds.python-conda-index]] withs = ["bootstrap"] @@ -361,18 +318,12 @@ withs = ["bootstrap"] [[bconds.python-pysocks]] replacements = {with_python3_tests = "0"} -[[bconds.ara]] -replacements = {with_docs = "0"} - [[bconds.python-httpretty]] replacements = {run_tests = "0"} [[bconds.python-ruamel-yaml]] withs = ["bootstrap"] -[[bconds.python-rq]] -withouts = ["tests"] - [[bconds.python-markdown-it-py]] withouts = ["plugins"] @@ -388,9 +339,6 @@ withouts = ["docs", "tests"] [[bconds.python-executing]] withs = ["bootstrap"] -[[bconds.python-editables]] -withouts = ["doc"] - [[bconds.python-jsonschema-specifications]] replacements = {with_doc = "0"} @@ -400,12 +348,6 @@ withs = ["bootstrap"] [[bconds.python-oslo-config]] withs = ["bootstrap"] -[[bconds.python-neutronclient]] -replacements = {with_doc = "0"} - -[[bconds.python-glanceclient]] -replacements = {with_doc = "0"} - [[bconds.python-domdf-python-tools]] withouts = ["tests"] @@ -460,9 +402,6 @@ withouts = ["check"] [[bconds.python-tabulate]] withs = ["bootstrap"] -[[bconds.python-repoze-sphinx-autointerface]] -withouts = ["tests"] - [[bconds.python-zope-exceptions]] withouts = ["tests"] @@ -487,15 +426,6 @@ withouts = ["tests"] [[bconds.python-humanfriendly]] withs = ["bootstrap"] -[[bconds.python-cascadio]] -withs = ["bootstrap"] - -[[bconds.python-pint]] -withs = ["bootstrap"] - -[[bconds.python-qcengine]] -withouts = ["tests"] - [[bconds.python-astropy-iers-data]] withouts = ["tests"] @@ -511,9 +441,6 @@ withs = ["bootstrap"] [[bconds.python-pydantic-core]] withouts = ["inline_snapshot_tests"] -[[bconds.python-optking]] -withouts = ["tests"] - [[bconds.python-django5]] withouts = ["tests"] @@ -544,9 +471,6 @@ withs = ["bootstrap"] [[bconds.python-mkdocs-material-extensions]] withouts = ["tests"] -[[bconds.python-zope-testing]] -withouts = ["tests"] - [[bconds.python-xnat]] withs = ["bootstrap"] @@ -556,9 +480,6 @@ withs = ["bootstrap"] [[bconds.llvm]] withouts = ["pgo"] -[[bconds.llvm21 ]] -withouts = ["pgo"] - [[bconds.python-pathspec]] withouts = ["tests"] @@ -622,9 +543,6 @@ withouts = ["tests"] [[bconds.pyproj]] withouts = ["xarray"] -[[bconds.python-distlib]] -withouts = ["check"] - [[bconds.python-flit]] withouts = ["tests"] @@ -655,9 +573,6 @@ withs = ["bootstrap"] [[bconds.python-versioningit]] withouts = ["pydantic_tests"] -[[bconds.python-scikit-image]] -withouts = ["check"] - [[bconds.python-semantic_version]] withs = ["bootstrap"] @@ -667,9 +582,6 @@ withs = ["bootstrap"] [[bconds.python-trimesh]] withs = ["bootstrap"] -[[bconds.python-scikit-image]] -withs = ["bootstrap"] - [[bconds.python-xarray]] withs = ["bootstrap"]