Prune GitHub Environments left behind by PR envs

.github/workflows/prune-pr-environments.yml

@@ -0,0 +1,72 @@
+name: Prune PR environments
+
+# Deletes each anchor repo's pr-<repo>-<N> GitHub Environment once PR N is
+# closed or loses the `deploy` label. See scripts/prune_pr_environments.py.
+
+on:
+  schedule:
+    - cron: "17 * * * *"
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        type: boolean
+        description: "List what would be deleted without deleting it"
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: prune-pr-environments
+  cancel-in-progress: false
+
+jobs:
+  prune:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      # Must match the anchorRepo list in the PR-env ApplicationSet
+      # (Kubernetes-Foundational-Services pr-env-appset.yaml).
+      ANCHOR_REPOS: >-
+        auth,mindshub_frontend,mindshub_inference,cowork-server,cowork,mindshub_stafftools,mindshub_services
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+
+      - name: Mint release-train App token
+        id: token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          client-id: ${{ vars.RELEASE_APP_CLIENT_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: ${{ env.ANCHOR_REPOS }}
+          permission-actions: read
+          permission-environments: write
+          permission-pull-requests: read
+
+      - name: Delete stale PR environments
+        env:
+          GH_TOKEN: ${{ steps.token.outputs.token }}
+          OWNER: ${{ github.repository_owner }}
+          DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }}
+        run: |
+          set -euo pipefail
+          args=()
+          IFS=',' read -ra repos <<< "${ANCHOR_REPOS}"
+          for repo in "${repos[@]}"; do args+=(--repo "${OWNER}/${repo}"); done
+          if [ "${DRY_RUN}" = true ]; then args+=(--dry-run); fi
+          python3 scripts/prune_pr_environments.py "${args[@]}"
+
+  notify:
+    needs: [prune]
+    if: ${{ github.event_name == 'schedule' && !cancelled() && !contains(needs.*.result, 'cancelled') }}
+    permissions:
+      contents: read
+      actions: read
+    uses: mindsdb/github-actions/.github/workflows/notify-main-failure.yml@main
+    with:
+      env-name: "PR environment prune"
+      status: ${{ contains(needs.*.result, 'failure') && 'failed' || 'recovered' }}
+    secrets: inherit

scripts/prune_pr_environments.py

@@ -0,0 +1,90 @@
+"""Delete the GitHub Environments left behind by closed PR environments.
+
+Each anchor repo's deploy job runs in a GitHub Environment named
+``pr-<repo slug>-<PR number>``, which GitHub creates on first use and never
+removes. The PR env itself lives exactly as long as the PR is open and carries
+the ``deploy`` label (the PR-env ApplicationSet's filter), so an environment
+whose PR fails that test is stale and is deleted here.
+
+Environments are listed before PRs, so an environment created mid-run is left
+for the next run. A repo whose PR list cannot be read is skipped entirely.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+from typing import Callable, Iterable, Sequence
+
+Runner = Callable[[Sequence[str]], "subprocess.CompletedProcess[str]"]
+
+DEPLOY_LABEL = "deploy"
+
+
+def _run(argv: Sequence[str]) -> "subprocess.CompletedProcess[str]":
+    return subprocess.run(list(argv), capture_output=True, text=True, check=False)
+
+
+def stale_environments(repo: str, names: Iterable[str], keep: set[int]) -> list[str]:
+    """PR-env environment names in ``repo`` whose PR number is not in ``keep``."""
+    slug = repo.split("/", 1)[1].replace("_", "-")
+    pattern = re.compile(rf"^pr-{re.escape(slug)}-([0-9]+)$")
+    stale = []
+    for name in names:
+        match = pattern.match(name)
+        if match and int(match.group(1)) not in keep:
+            stale.append(name)
+    return sorted(stale)
+
+
+def _lines(result: "subprocess.CompletedProcess[str]") -> list[str]:
+    return [line for line in result.stdout.splitlines() if line.strip()]
+
+
+def _error(result: "subprocess.CompletedProcess[str]") -> str:
+    return result.stderr.strip() or result.stdout.strip()
+
+
+def prune(repos: Sequence[str], *, dry_run: bool, runner: Runner = _run) -> int:
+    failed = False
+    for repo in repos:
+        envs = runner(["gh", "api", "--paginate", f"repos/{repo}/environments?per_page=100", "--jq", ".environments[].name"])
+        if envs.returncode != 0:
+            print(f"::error title={repo}::could not list environments: {_error(envs)}", flush=True)
+            failed = True
+            continue
+        prs = runner([
+            "gh", "api", "--paginate", f"repos/{repo}/pulls?state=open&per_page=100",
+            "--jq", f'.[] | select(any(.labels[]; .name == "{DEPLOY_LABEL}")) | .number',
+        ])
+        if prs.returncode != 0:
+            print(f"::error title={repo}::could not list PRs, deleting nothing: {_error(prs)}", flush=True)
+            failed = True
+            continue
+        stale = stale_environments(repo, _lines(envs), {int(n) for n in _lines(prs)})
+        print(f"{repo}: {len(stale)} stale", flush=True)
+        for name in stale:
+            if dry_run:
+                print(f"  would delete {name}", flush=True)
+                continue
+            res = runner(["gh", "api", "-X", "DELETE", f"repos/{repo}/environments/{name}"])
+            if res.returncode != 0:
+                print(f"::error title={repo}::could not delete {name}: {_error(res)}", flush=True)
+                failed = True
+            else:
+                print(f"  deleted {name}", flush=True)
+    return 1 if failed else 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--repo", action="append", required=True, metavar="OWNER/NAME", help="anchor repo; repeat for each")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args(argv)
+    return prune(args.repo, dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_prune_pr_environments.py

@@ -0,0 +1,98 @@
+"""Unit tests for ``scripts/prune_pr_environments.py``.
+
+The behaviour worth pinning is what is NOT deleted: environments of live PR
+envs, environments that are not PR envs at all, and everything in a repo whose
+PR list could not be read.
+"""
+
+import importlib.util
+import subprocess
+from pathlib import Path
+
+_PATH = Path(__file__).resolve().parents[1] / "scripts" / "prune_pr_environments.py"
+_spec = importlib.util.spec_from_file_location("prune_pr_environments", _PATH)
+prune = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(prune)
+
+
+def completed(stdout="", stderr="", returncode=0):
+    return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr=stderr)
+
+
+class FakeGh:
+    """Answers `gh api` calls from per-repo tables and records every call."""
+
+    def __init__(self, envs, prs, fail_prs=(), fail_delete=()):
+        self.envs, self.prs = envs, prs
+        self.fail_prs, self.fail_delete = set(fail_prs), set(fail_delete)
+        self.calls = []
+
+    def __call__(self, argv):
+        argv = list(argv)
+        self.calls.append(argv)
+        if "-X" in argv:
+            path = argv[-1]
+            repo, name = path.removeprefix("repos/").split("/environments/")
+            if name in self.fail_delete:
+                return completed(stderr="HTTP 403", returncode=1)
+            return completed()
+        path = next(a for a in argv if a.startswith("repos/"))
+        repo = "/".join(path.split("/")[1:3])
+        if "/environments" in path:
+            return completed("\n".join(self.envs.get(repo, [])) + "\n")
+        if repo in self.fail_prs:
+            return completed(stderr="HTTP 502", returncode=1)
+        return completed("\n".join(str(n) for n in self.prs.get(repo, [])) + "\n")
+
+    def deleted(self):
+        return [a[-1].removeprefix("repos/") for a in self.calls if "-X" in a]
+
+
+class TestStaleEnvironments:
+    def test_keeps_live_prs_and_ignores_non_pr_envs(self):
+        names = ["dev", "staging", "pr-auth-110", "pr-auth-111", "pr-auth-12x", "pr-cowork-5", "pr-auth-"]
+        assert prune.stale_environments("mindsdb/auth", names, keep={110}) == ["pr-auth-111"]
+
+    def test_underscore_repo_uses_the_slug(self):
+        names = ["pr-mindshub-frontend-7", "pr-mindshub_frontend-8", "pr-mindshub-frontend-9"]
+        assert prune.stale_environments("mindsdb/mindshub_frontend", names, keep={9}) == ["pr-mindshub-frontend-7"]
+
+    def test_anchor_that_prefixes_another_is_not_confused(self):
+        # pr-cowork-server-588 belongs to cowork-server, not to cowork PR "server-588".
+        assert prune.stale_environments("mindsdb/cowork", ["pr-cowork-server-588", "pr-cowork-3"], keep=set()) == ["pr-cowork-3"]
+
+
+class TestPrune:
+    def test_deletes_only_stale_environments(self):
+        gh = FakeGh(
+            envs={"mindsdb/auth": ["dev", "pr-auth-1", "pr-auth-2"], "mindsdb/cowork": ["pr-cowork-9"]},
+            prs={"mindsdb/auth": [2], "mindsdb/cowork": []},
+        )
+        assert prune.prune(["mindsdb/auth", "mindsdb/cowork"], dry_run=False, runner=gh) == 0
+        assert gh.deleted() == ["mindsdb/auth/environments/pr-auth-1", "mindsdb/cowork/environments/pr-cowork-9"]
+
+    def test_environments_are_listed_before_prs(self):
+        # An env created after the PR list was read must not be judged stale.
+        gh = FakeGh(envs={"mindsdb/auth": []}, prs={"mindsdb/auth": []})
+        prune.prune(["mindsdb/auth"], dry_run=False, runner=gh)
+        paths = [next(a for a in c if a.startswith("repos/")) for c in gh.calls]
+        assert "/environments" in paths[0] and "/pulls" in paths[1]
+
+    def test_dry_run_deletes_nothing(self):
+        gh = FakeGh(envs={"mindsdb/auth": ["pr-auth-1"]}, prs={"mindsdb/auth": []})
+        assert prune.prune(["mindsdb/auth"], dry_run=True, runner=gh) == 0
+        assert gh.deleted() == []
+
+    def test_unreadable_pr_list_skips_that_repo_only(self):
+        gh = FakeGh(
+            envs={"mindsdb/auth": ["pr-auth-1"], "mindsdb/cowork": ["pr-cowork-9"]},
+            prs={"mindsdb/cowork": []},
+            fail_prs={"mindsdb/auth"},
+        )
+        assert prune.prune(["mindsdb/auth", "mindsdb/cowork"], dry_run=False, runner=gh) == 1
+        assert gh.deleted() == ["mindsdb/cowork/environments/pr-cowork-9"]
+
+    def test_failed_delete_does_not_stop_the_rest(self):
+        gh = FakeGh(envs={"mindsdb/auth": ["pr-auth-1", "pr-auth-2"]}, prs={"mindsdb/auth": []}, fail_delete={"pr-auth-1"})
+        assert prune.prune(["mindsdb/auth"], dry_run=False, runner=gh) == 1
+        assert gh.deleted() == ["mindsdb/auth/environments/pr-auth-1", "mindsdb/auth/environments/pr-auth-2"]