diff --git a/.github/workflows/artifact-cleanup.yml b/.github/workflows/artifact-cleanup.yml index a8fb393..63b2b26 100644 --- a/.github/workflows/artifact-cleanup.yml +++ b/.github/workflows/artifact-cleanup.yml @@ -1,12 +1,7 @@ name: Artifact Cleanup on: - workflow_run: - workflows: ["Build NixOS ISOs"] - types: - - completed - schedule: - - cron: "0 4 * * *" + workflow_call: permissions: contents: write @@ -14,104 +9,76 @@ permissions: jobs: cleanup: - name: Rotate ISO pre-release artifacts and tags + name: Rotate pre-release artifacts and tags runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} steps: - - name: Rotate releases and artifacts - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} + - name: Delete old pre-release releases and tags (keep 2 per branch) run: | set -euo pipefail - gh release list --repo "$REPO" --limit 200 \ - --json tagName,body,createdAt,isPrerelease \ - > /tmp/releases.json - - gh api "repos/${REPO}/actions/artifacts?per_page=100" \ - --paginate \ - | jq -r '.artifacts[] | select(.expired == false) | [.id, .name, .created_at, (.workflow_run.head_branch // "unknown")] | @tsv' \ - > /tmp/artifacts.tsv - - python3 - <<'EOF' - import json, subprocess, re, sys, os - from datetime import datetime, timezone - from collections import defaultdict + PROTECTED="iso-latest" - GRACE_SECS = 86400 - MAX_PER_BRANCH = 2 - NOW = datetime.now(timezone.utc) - REPO = os.environ["REPO"] - - ISO_PRERELEASE_RE = re.compile(r'^iso-v\d+\.\d+\.\d+-(alpha|beta|rc)\.(\d+)$') - STABLE_RE = re.compile(r'^v\d+\.\d+\.\d+$') - PROTECTED_TAGS = {"iso-latest"} + gh release list --repo "$REPO" --limit 200 \ + --json tagName,isPrerelease,createdAt \ + | jq -r '.[] | select(.isPrerelease == true) | [.tagName, .createdAt] | @tsv' \ + | sort -t$'\t' -k2,2r \ + | awk -F'\t' -v protected="$PROTECTED" ' + { + tag = $1 + if (tag == protected) next + if (match(tag, /--(.+)$/, m)) branch = m[1] + else branch = "unknown" + count[branch]++ + if (count[branch] > 2) print tag + } + ' \ + | while read -r tag; do + echo "[INFO] deleting release tag=${tag}" + gh release delete "$tag" --repo "$REPO" --yes --cleanup-tag || \ + echo "[WARN] delete failed tag=${tag}" + done - with open("/tmp/releases.json") as f: - releases = json.load(f) + - name: Delete orphaned tags without release + run: | + set -euo pipefail - branch_releases = {} - for r in releases: - tag = r["tagName"] - if tag in PROTECTED_TAGS or STABLE_RE.match(tag): - continue - if not r["isPrerelease"]: - continue - m = ISO_PRERELEASE_RE.match(tag) - if not m: - continue - body = r.get("body") or "" - branch = None - for line in body.splitlines(): - if line.startswith("branch: "): - branch = line[len("branch: "):].strip() - break - if not branch: - continue - count = int(m.group(2)) - created = datetime.fromisoformat(r["createdAt"].replace("Z", "+00:00")) - branch_releases.setdefault(branch, []).append((count, tag, created)) + PROTECTED="iso-latest" - for branch, entries in branch_releases.items(): - entries.sort(key=lambda x: x[0]) - for count, tag, created in entries[:-MAX_PER_BRANCH]: - age = (NOW - created).total_seconds() - if age < GRACE_SECS: - print(f"[INFO] skip tag={tag} age={int(age)}s below grace period") - continue - print(f"[INFO] deleting tag={tag} branch={branch} age={int(age)}s") - result = subprocess.run( - ["gh", "release", "delete", tag, "--repo", REPO, "--yes", "--cleanup-tag"], - capture_output=True, text=True - ) - if result.returncode != 0: - print(f"[WARN] delete failed tag={tag} err={result.stderr.strip()}", file=sys.stderr) + gh api "repos/${REPO}/git/refs/tags" --paginate \ + | jq -r '.[].ref | ltrimstr("refs/tags/")' \ + | while read -r tag; do + [ "$tag" = "$PROTECTED" ] && continue + if ! gh release view "$tag" --repo "$REPO" > /dev/null 2>&1; then + echo "[INFO] deleting orphaned tag=${tag}" + gh api -X DELETE "repos/${REPO}/git/refs/tags/${tag}" || \ + echo "[WARN] delete failed tag=${tag}" + fi + done - groups = defaultdict(list) - with open("/tmp/artifacts.tsv") as f: - for line in f: - line = line.strip() - if not line: - continue - parts = line.split("\t") - if len(parts) < 4: - continue - artifact_id, name, created_at, branch = parts[0], parts[1], parts[2], parts[3] - created = datetime.fromisoformat(created_at.replace("Z", "+00:00")) - groups[(name, branch)].append((created, artifact_id)) + - name: Delete old Actions artifacts (keep 2 per name+branch) + run: | + set -euo pipefail - for (name, branch), entries in groups.items(): - entries.sort(key=lambda x: x[0], reverse=True) - for created, artifact_id in entries[MAX_PER_BRANCH:]: - age = (NOW - created).total_seconds() - if age < GRACE_SECS: - print(f"[INFO] skip artifact_id={artifact_id} name={name} age={int(age)}s below grace period") - continue - print(f"[INFO] deleting artifact_id={artifact_id} name={name} branch={branch} age={int(age)}s") - result = subprocess.run( - ["gh", "api", "-X", "DELETE", f"repos/{REPO}/actions/artifacts/{artifact_id}"], - capture_output=True, text=True - ) - if result.returncode != 0: - print(f"[WARN] delete failed artifact_id={artifact_id} err={result.stderr.strip()}", file=sys.stderr) - EOF + gh api "repos/${REPO}/actions/artifacts?per_page=100" --paginate \ + | jq -r '.artifacts[] | select(.expired == false) | [ + (.id | tostring), + .name, + (.workflow_run.head_branch // "unknown"), + .created_at + ] | @tsv' \ + | sort -t$'\t' -k2,2 -k3,3 -k4,4r \ + | awk -F'\t' ' + { + key = $2 SUBSEP $3 + count[key]++ + if (count[key] > 2) print $1 + } + ' \ + | while read -r id; do + echo "[INFO] deleting artifact id=${id}" + gh api -X DELETE "repos/${REPO}/actions/artifacts/${id}" || \ + echo "[WARN] delete failed artifact_id=${id}" + done diff --git a/.github/workflows/build-iso.yml b/.github/workflows/build-iso.yml index 2484869..1ee99e8 100644 --- a/.github/workflows/build-iso.yml +++ b/.github/workflows/build-iso.yml @@ -20,8 +20,62 @@ permissions: contents: write jobs: + lint: + name: Format and lint + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + sha: ${{ steps.push.outputs.sha }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref || github.ref_name }} + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: Run nixpkgs-fmt + run: nix run nixpkgs#nixpkgs-fmt -- . + + - name: Run statix + run: nix run nixpkgs#statix -- check . + + - name: Commit formatting fixes + id: push + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if ! git diff --quiet; then + git add -A + git commit -m "style: apply nixpkgs-fmt" + git push + fi + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + check: + name: Nix flake check + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.lint.outputs.sha }} + + - uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: Check flake evaluation + run: nix flake check --no-build + build: name: ISO ${{ matrix.config }} (${{ matrix.arch }}) + needs: [lint, check] runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -36,6 +90,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + ref: ${{ needs.lint.outputs.sha }} - uses: cachix/install-nix-action@v27 with: @@ -82,28 +138,38 @@ jobs: - name: Resolve version id: ver run: | - VERSION=$(grep 'version' versions.nix | head -1 | sed 's/.*"\(.*\)"/\1/') + VERSION=$(grep -m1 'version' versions.nix | sed 's/[^"]*"\([^"]*\)".*/\1/') BRANCH="${{ github.ref_name }}" echo "version=${VERSION}" >> $GITHUB_OUTPUT echo "branch=${BRANCH}" >> $GITHUB_OUTPUT - name: Create versioned pre-release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | VERSION="${{ steps.ver.outputs.version }}" BRANCH="${{ steps.ver.outputs.branch }}" - TAG="iso-v${VERSION}" + BRANCH_SLUG=$(echo "$BRANCH" | sed 's|/|-|g') + TAG="iso-v${VERSION}--${BRANCH_SLUG}" REPO="${{ github.repository }}" BASE="https://github.com/${REPO}/releases/download/${TAG}" - gh release create "${TAG}" \ - --title "ISO v${VERSION}" \ - --notes "branch: ${BRANCH}" \ - --prerelease \ - *.iso *.sha256 + if gh release view "${TAG}" --repo "$REPO" > /dev/null 2>&1; then + gh release upload "${TAG}" --repo "$REPO" *.iso *.sha256 --clobber + else + gh release create "${TAG}" \ + --title "ISO v${VERSION} (${BRANCH})" \ + --notes "" \ + --prerelease \ + *.iso *.sha256 + fi if [ "${BRANCH}" = "develop" ]; then - git tag -f iso-latest - git push origin iso-latest --force + SHA="${{ github.sha }}" + gh api -X PATCH "repos/${REPO}/git/refs/tags/iso-latest" \ + -f sha="$SHA" -F force=true 2>/dev/null || \ + gh api -X POST "repos/${REPO}/git/refs" \ + -f ref="refs/tags/iso-latest" -f sha="$SHA" fi echo "## ISO Download URLs" >> $GITHUB_STEP_SUMMARY @@ -112,8 +178,6 @@ jobs: echo "|--------|------|-----|" >> $GITHUB_STEP_SUMMARY echo "| csfx-node-iso | amd64 | ${BASE}/csfx-node-iso-amd64.iso |" >> $GITHUB_STEP_SUMMARY echo "| csfx-node-iso-arm64 | arm64 | ${BASE}/csfx-node-iso-arm64-arm64.iso |" >> $GITHUB_STEP_SUMMARY - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} release: name: Publish ISOs to Release @@ -135,6 +199,8 @@ jobs: echo "version=${VERSION}" >> $GITHUB_OUTPUT - name: Upload to release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | VERSION="${{ steps.ver.outputs.version }}" TAG="v${VERSION}" @@ -156,5 +222,12 @@ jobs: echo "|--------|------|-----|" >> $GITHUB_STEP_SUMMARY echo "| csfx-node-iso | amd64 | ${BASE}/csfx-node-iso-amd64.iso |" >> $GITHUB_STEP_SUMMARY echo "| csfx-node-iso-arm64 | arm64 | ${BASE}/csfx-node-iso-arm64-arm64.iso |" >> $GITHUB_STEP_SUMMARY - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + cleanup: + name: Artifact Cleanup + needs: [release-dev, release] + if: always() + uses: ./.github/workflows/artifact-cleanup.yml + permissions: + contents: write + actions: write diff --git a/.github/workflows/pr-merge-cleanup.yml b/.github/workflows/pr-merge-cleanup.yml index 5e94970..c857567 100644 --- a/.github/workflows/pr-merge-cleanup.yml +++ b/.github/workflows/pr-merge-cleanup.yml @@ -14,54 +14,44 @@ jobs: name: Remove ISO pre-releases for merged branch if: github.event.pull_request.merged == true runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + BRANCH: ${{ github.event.pull_request.head.ref }} steps: - - name: Delete stale ISO pre-releases for branch - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - BRANCH: ${{ github.event.pull_request.head.ref }} + - name: Delete all ISO pre-releases for branch run: | set -euo pipefail - TAGS=$(gh release list --repo "$REPO" --limit 200 --json tagName,body,isPrerelease \ - | jq -r --arg branch "$BRANCH" ' + BRANCH_SLUG=$(echo "$BRANCH" | sed 's|/|-|g') + + gh release list --repo "$REPO" --limit 200 \ + --json tagName,isPrerelease \ + | jq -r --arg slug "$BRANCH_SLUG" ' .[] | select(.isPrerelease == true) - | select(.tagName | test("^iso-v[0-9]+\\.[0-9]+\\.[0-9]+-(alpha|beta|rc)\\.[0-9]+$")) - | select(.body | test("^branch: " + $branch + "$"; "m")) + | select(.tagName | endswith("--" + $slug)) | .tagName - ') - - if [ -z "$TAGS" ]; then - echo "[INFO] no ISO pre-releases found branch=${BRANCH}" - exit 0 - fi - - LATEST=$(echo "$TAGS" \ - | grep -oE '\.[0-9]+$' \ - | tr -d '.' \ - | sort -n \ - | tail -1) - - LATEST_TAG=$(echo "$TAGS" \ - | grep -E "\\.${LATEST}$" \ - | head -1) - - echo "[INFO] keeping tag=${LATEST_TAG} branch=${BRANCH}" - - echo "$TAGS" | while read -r TAG; do - if [ "$TAG" = "$LATEST_TAG" ]; then - continue - fi - echo "[INFO] deleting tag=${TAG} branch=${BRANCH}" - gh release delete "$TAG" --repo "$REPO" --yes --cleanup-tag || true - done - - RUN_IDS=$(gh run list --repo "$REPO" --workflow build-iso.yml \ - --branch "$BRANCH" --limit 100 --json databaseId \ - | jq -r '.[].databaseId') + ' \ + | while read -r tag; do + echo "[INFO] deleting tag=${tag} branch=${BRANCH}" + gh release delete "$tag" --repo "$REPO" --yes --cleanup-tag || \ + echo "[WARN] delete failed tag=${tag}" + done + + - name: Delete Actions artifacts for branch + run: | + set -euo pipefail - echo "$RUN_IDS" | tail -n +2 | while read -r RUN_ID; do - echo "[INFO] deleting run artifacts run_id=${RUN_ID}" - gh api -X DELETE "repos/${REPO}/actions/runs/${RUN_ID}/artifacts" 2>/dev/null || true - done + gh api "repos/${REPO}/actions/artifacts?per_page=100" --paginate \ + | jq -r --arg branch "$BRANCH" ' + .artifacts[] + | select(.expired == false) + | select((.workflow_run.head_branch // "") == $branch) + | .id | tostring + ' \ + | while read -r id; do + echo "[INFO] deleting artifact id=${id} branch=${BRANCH}" + gh api -X DELETE "repos/${REPO}/actions/artifacts/${id}" || \ + echo "[WARN] delete failed artifact_id=${id}" + done diff --git a/versions.nix b/versions.nix index c3664e5..02c8816 100644 --- a/versions.nix +++ b/versions.nix @@ -3,92 +3,92 @@ version = "0.2.2-alpha.474"; agent = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-agent-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-agent-amd64"; sha256 = "0129eac1b3ba27ec02ee3dc2a808e9c4f83711eb27d5f600c73bb4e18252f0e2"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-agent-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-agent-arm64"; sha256 = "4dae7a3607c958272ff63110e4158066d5feecfdf901fc87a5d3b81fc2f8c825"; }; }; controlPlane = { migrate = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-csfx-migrate-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-csfx-migrate-amd64"; sha256 = "1483f6807c9b3de27ebe5728d84910ad0c4bb54de3d895178f0b8a92aabbcd30"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-csfx-migrate-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-csfx-migrate-arm64"; sha256 = "35781460e5525533f49704f5e5b8cc423f6e5f1aaba199db5af8d23caa08e296"; }; }; api-gateway = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-api-gateway-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-api-gateway-amd64"; sha256 = "d2dcffaa498ac54a7b5629a6e828e20fbf65b928e85d6ddb147b5c1261acb973"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-api-gateway-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-api-gateway-arm64"; sha256 = "38fe718131e52a0e0d3b5a369d301a81a7c19b981d1a4a2c672b224807adfbc1"; }; }; registry = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-registry-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-registry-amd64"; sha256 = "bbfc6e22fe44867c95e2aad5f902da0f9eb2876f122e0b3b87ebd2dac8b67428"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-registry-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-registry-arm64"; sha256 = "bc16c332ec3dff9c38244e8275346bc1882007007600e4a93feb8fd95c0675ab"; }; }; scheduler = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-scheduler-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-scheduler-amd64"; sha256 = "3f8749a5ef0a9e08fe8f1a7c97130252392a09c95c056a193e909d28be5aac62"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-scheduler-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-scheduler-arm64"; sha256 = "11ba7946b4f8187d122efa0eb68f18ef8acd272377d99a07e1be858838893608"; }; }; volume-manager = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-volume-manager-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-volume-manager-amd64"; sha256 = "ca822c83e2f332fcd68f044b0e875db5db4dca575421f338ca014912f995d6ac"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-volume-manager-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-volume-manager-arm64"; sha256 = "7e38ca070d4c3872b32beb944e4f120ac1e80ca2a0c08eff74425719441b3592"; }; }; failover-controller = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-failover-controller-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-failover-controller-amd64"; sha256 = "505204add832f89084e16661a03d49bbb02c2ef1d000470c78adc02da32b7ec3"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-failover-controller-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-failover-controller-arm64"; sha256 = "dd8b1adf626c05af1a4172f6d56538976cabc3b30fbf0b6856aa77f61c0550d8"; }; }; sdn-controller = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-sdn-controller-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-sdn-controller-amd64"; sha256 = "556dd0f9e5c20737d821164a214b02191f9b20843e2a860c4f23d3a34c75b37f"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-sdn-controller-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-cp-sdn-controller-arm64"; sha256 = "81a011e8cad4513647c3137fd25293fe16f5b5b079f746a73ec9c6c63eb6b409"; }; }; csfx-updater = { amd64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-updater-amd64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-updater-amd64"; sha256 = "1e054b8f49f321fa7a2d06c83b65cf6031bfbdf910277546086a8954be8b67cb"; }; arm64 = { - url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-updater-arm64"; + url = "https://github.com/CSFX-cloud/CSFX-Core/releases/download/v0.2.2-alpha.474/csfx-updater-arm64"; sha256 = "e0a9bdfa73593751723dd75bcc096de863cc63b78408a2b583b1f5745d8c8695"; }; };