From 31c1f1f28741a513ae2c29b2942b49919dd1ad6f Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Tue, 29 Sep 2026 14:09:14 +1300 Subject: [PATCH] Prune GitHub Environments left behind by PR envs Each anchor repo's deploy job runs in a pr-- GitHub Environment, which GitHub creates on first use and never removes. An hourly job now deletes the ones whose PR is closed or no longer carries the deploy label, the same test the PR-env ApplicationSet applies. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/prune-pr-environments.yml | 72 +++++++++++++++ scripts/prune_pr_environments.py | 90 +++++++++++++++++++ tests/test_prune_pr_environments.py | 98 +++++++++++++++++++++ 3 files changed, 260 insertions(+) create mode 100644 .github/workflows/prune-pr-environments.yml create mode 100644 scripts/prune_pr_environments.py create mode 100644 tests/test_prune_pr_environments.py diff --git a/.github/workflows/prune-pr-environments.yml b/.github/workflows/prune-pr-environments.yml new file mode 100644 index 0000000..5a6c243 --- /dev/null +++ b/.github/workflows/prune-pr-environments.yml @@ -0,0 +1,72 @@ +name: Prune PR environments + +# Deletes each anchor repo's pr-- GitHub Environment once PR N is +# closed or loses the `deploy` label. See scripts/prune_pr_environments.py. + +on: + schedule: + - cron: "17 * * * *" + workflow_dispatch: + inputs: + dry_run: + type: boolean + description: "List what would be deleted without deleting it" + default: false + +permissions: + contents: read + +concurrency: + group: prune-pr-environments + cancel-in-progress: false + +jobs: + prune: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + # Must match the anchorRepo list in the PR-env ApplicationSet + # (Kubernetes-Foundational-Services pr-env-appset.yaml). + ANCHOR_REPOS: >- + auth,mindshub_frontend,mindshub_inference,cowork-server,cowork,mindshub_stafftools,mindshub_services + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Mint release-train App token + id: token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ env.ANCHOR_REPOS }} + permission-actions: read + permission-environments: write + permission-pull-requests: read + + - name: Delete stale PR environments + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + OWNER: ${{ github.repository_owner }} + DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }} + run: | + set -euo pipefail + args=() + IFS=',' read -ra repos <<< "${ANCHOR_REPOS}" + for repo in "${repos[@]}"; do args+=(--repo "${OWNER}/${repo}"); done + if [ "${DRY_RUN}" = true ]; then args+=(--dry-run); fi + python3 scripts/prune_pr_environments.py "${args[@]}" + + notify: + needs: [prune] + if: ${{ github.event_name == 'schedule' && !cancelled() && !contains(needs.*.result, 'cancelled') }} + permissions: + contents: read + actions: read + uses: mindsdb/github-actions/.github/workflows/notify-main-failure.yml@main + with: + env-name: "PR environment prune" + status: ${{ contains(needs.*.result, 'failure') && 'failed' || 'recovered' }} + secrets: inherit diff --git a/scripts/prune_pr_environments.py b/scripts/prune_pr_environments.py new file mode 100644 index 0000000..f97e04b --- /dev/null +++ b/scripts/prune_pr_environments.py @@ -0,0 +1,90 @@ +"""Delete the GitHub Environments left behind by closed PR environments. + +Each anchor repo's deploy job runs in a GitHub Environment named +``pr--``, which GitHub creates on first use and never +removes. The PR env itself lives exactly as long as the PR is open and carries +the ``deploy`` label (the PR-env ApplicationSet's filter), so an environment +whose PR fails that test is stale and is deleted here. + +Environments are listed before PRs, so an environment created mid-run is left +for the next run. A repo whose PR list cannot be read is skipped entirely. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from typing import Callable, Iterable, Sequence + +Runner = Callable[[Sequence[str]], "subprocess.CompletedProcess[str]"] + +DEPLOY_LABEL = "deploy" + + +def _run(argv: Sequence[str]) -> "subprocess.CompletedProcess[str]": + return subprocess.run(list(argv), capture_output=True, text=True, check=False) + + +def stale_environments(repo: str, names: Iterable[str], keep: set[int]) -> list[str]: + """PR-env environment names in ``repo`` whose PR number is not in ``keep``.""" + slug = repo.split("/", 1)[1].replace("_", "-") + pattern = re.compile(rf"^pr-{re.escape(slug)}-([0-9]+)$") + stale = [] + for name in names: + match = pattern.match(name) + if match and int(match.group(1)) not in keep: + stale.append(name) + return sorted(stale) + + +def _lines(result: "subprocess.CompletedProcess[str]") -> list[str]: + return [line for line in result.stdout.splitlines() if line.strip()] + + +def _error(result: "subprocess.CompletedProcess[str]") -> str: + return result.stderr.strip() or result.stdout.strip() + + +def prune(repos: Sequence[str], *, dry_run: bool, runner: Runner = _run) -> int: + failed = False + for repo in repos: + envs = runner(["gh", "api", "--paginate", f"repos/{repo}/environments?per_page=100", "--jq", ".environments[].name"]) + if envs.returncode != 0: + print(f"::error title={repo}::could not list environments: {_error(envs)}", flush=True) + failed = True + continue + prs = runner([ + "gh", "api", "--paginate", f"repos/{repo}/pulls?state=open&per_page=100", + "--jq", f'.[] | select(any(.labels[]; .name == "{DEPLOY_LABEL}")) | .number', + ]) + if prs.returncode != 0: + print(f"::error title={repo}::could not list PRs, deleting nothing: {_error(prs)}", flush=True) + failed = True + continue + stale = stale_environments(repo, _lines(envs), {int(n) for n in _lines(prs)}) + print(f"{repo}: {len(stale)} stale", flush=True) + for name in stale: + if dry_run: + print(f" would delete {name}", flush=True) + continue + res = runner(["gh", "api", "-X", "DELETE", f"repos/{repo}/environments/{name}"]) + if res.returncode != 0: + print(f"::error title={repo}::could not delete {name}: {_error(res)}", flush=True) + failed = True + else: + print(f" deleted {name}", flush=True) + return 1 if failed else 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--repo", action="append", required=True, metavar="OWNER/NAME", help="anchor repo; repeat for each") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + return prune(args.repo, dry_run=args.dry_run) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_prune_pr_environments.py b/tests/test_prune_pr_environments.py new file mode 100644 index 0000000..7547108 --- /dev/null +++ b/tests/test_prune_pr_environments.py @@ -0,0 +1,98 @@ +"""Unit tests for ``scripts/prune_pr_environments.py``. + +The behaviour worth pinning is what is NOT deleted: environments of live PR +envs, environments that are not PR envs at all, and everything in a repo whose +PR list could not be read. +""" + +import importlib.util +import subprocess +from pathlib import Path + +_PATH = Path(__file__).resolve().parents[1] / "scripts" / "prune_pr_environments.py" +_spec = importlib.util.spec_from_file_location("prune_pr_environments", _PATH) +prune = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(prune) + + +def completed(stdout="", stderr="", returncode=0): + return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr) + + +class FakeGh: + """Answers `gh api` calls from per-repo tables and records every call.""" + + def __init__(self, envs, prs, fail_prs=(), fail_delete=()): + self.envs, self.prs = envs, prs + self.fail_prs, self.fail_delete = set(fail_prs), set(fail_delete) + self.calls = [] + + def __call__(self, argv): + argv = list(argv) + self.calls.append(argv) + if "-X" in argv: + path = argv[-1] + repo, name = path.removeprefix("repos/").split("/environments/") + if name in self.fail_delete: + return completed(stderr="HTTP 403", returncode=1) + return completed() + path = next(a for a in argv if a.startswith("repos/")) + repo = "/".join(path.split("/")[1:3]) + if "/environments" in path: + return completed("\n".join(self.envs.get(repo, [])) + "\n") + if repo in self.fail_prs: + return completed(stderr="HTTP 502", returncode=1) + return completed("\n".join(str(n) for n in self.prs.get(repo, [])) + "\n") + + def deleted(self): + return [a[-1].removeprefix("repos/") for a in self.calls if "-X" in a] + + +class TestStaleEnvironments: + def test_keeps_live_prs_and_ignores_non_pr_envs(self): + names = ["dev", "staging", "pr-auth-110", "pr-auth-111", "pr-auth-12x", "pr-cowork-5", "pr-auth-"] + assert prune.stale_environments("mindsdb/auth", names, keep={110}) == ["pr-auth-111"] + + def test_underscore_repo_uses_the_slug(self): + names = ["pr-mindshub-frontend-7", "pr-mindshub_frontend-8", "pr-mindshub-frontend-9"] + assert prune.stale_environments("mindsdb/mindshub_frontend", names, keep={9}) == ["pr-mindshub-frontend-7"] + + def test_anchor_that_prefixes_another_is_not_confused(self): + # pr-cowork-server-588 belongs to cowork-server, not to cowork PR "server-588". + assert prune.stale_environments("mindsdb/cowork", ["pr-cowork-server-588", "pr-cowork-3"], keep=set()) == ["pr-cowork-3"] + + +class TestPrune: + def test_deletes_only_stale_environments(self): + gh = FakeGh( + envs={"mindsdb/auth": ["dev", "pr-auth-1", "pr-auth-2"], "mindsdb/cowork": ["pr-cowork-9"]}, + prs={"mindsdb/auth": [2], "mindsdb/cowork": []}, + ) + assert prune.prune(["mindsdb/auth", "mindsdb/cowork"], dry_run=False, runner=gh) == 0 + assert gh.deleted() == ["mindsdb/auth/environments/pr-auth-1", "mindsdb/cowork/environments/pr-cowork-9"] + + def test_environments_are_listed_before_prs(self): + # An env created after the PR list was read must not be judged stale. + gh = FakeGh(envs={"mindsdb/auth": []}, prs={"mindsdb/auth": []}) + prune.prune(["mindsdb/auth"], dry_run=False, runner=gh) + paths = [next(a for a in c if a.startswith("repos/")) for c in gh.calls] + assert "/environments" in paths[0] and "/pulls" in paths[1] + + def test_dry_run_deletes_nothing(self): + gh = FakeGh(envs={"mindsdb/auth": ["pr-auth-1"]}, prs={"mindsdb/auth": []}) + assert prune.prune(["mindsdb/auth"], dry_run=True, runner=gh) == 0 + assert gh.deleted() == [] + + def test_unreadable_pr_list_skips_that_repo_only(self): + gh = FakeGh( + envs={"mindsdb/auth": ["pr-auth-1"], "mindsdb/cowork": ["pr-cowork-9"]}, + prs={"mindsdb/cowork": []}, + fail_prs={"mindsdb/auth"}, + ) + assert prune.prune(["mindsdb/auth", "mindsdb/cowork"], dry_run=False, runner=gh) == 1 + assert gh.deleted() == ["mindsdb/cowork/environments/pr-cowork-9"] + + def test_failed_delete_does_not_stop_the_rest(self): + gh = FakeGh(envs={"mindsdb/auth": ["pr-auth-1", "pr-auth-2"]}, prs={"mindsdb/auth": []}, fail_delete={"pr-auth-1"}) + assert prune.prune(["mindsdb/auth"], dry_run=False, runner=gh) == 1 + assert gh.deleted() == ["mindsdb/auth/environments/pr-auth-1", "mindsdb/auth/environments/pr-auth-2"]