From cad9b25b4909f00fe36a3dc77c4c786dd6ffa382 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Wed, 22 Jul 2026 09:22:40 +0300 Subject: [PATCH] nightly: adopt gotestsum rerun config (from feat/nightly-selfhosted) + timeout headroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two combined fixes for the 3 consecutive red nightlies (Jul 19-21): 1. main was still running the OLD serialized config (-parallel 1, no reruns) — the validated gotestsum config from feat/nightly-selfhosted (parallel 6, --rerun-fails=3 to re-run load-starvation failures in a fresh process, nightly-beta release job) never landed. Adopt it. 2. Raise go-test -timeout 40/45m -> 60/65m and job timeout-minutes -> 80/85 so the first pass is never killed mid-run and the rerun pass has room. Evidence: first pass takes ~39m at -parallel 6 with ~45 load-dependent failures that pass individually (isolated 2026-07-15); -parallel 3 measured strictly worse (59 fails/60m), so parallelism tuning is a dead end — completion headroom is the fix. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/nightly.yml | 119 ++++++++++++++++++++++++++++------ 1 file changed, 98 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 01fe6959..167dadc8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -37,8 +37,14 @@ permissions: jobs: integration: name: Integration suite (default tags) - runs-on: ubuntu-latest - timeout-minutes: 55 + runs-on: [self-hosted, pilot-nightly] + timeout-minutes: 80 + env: + # Dedicated GCP e2-standard-4 self-hosted runner (pilot-nightly-runner). + # PILOT_REAL_NETWORK=1 disables the requireRealNetwork skip so the FULL + # meaningful suite runs — the multi-daemon UDP/NAT/WSS/HTTP integration + # tests that hosted runners starve on complete fine on this dedicated box. + PILOT_REAL_NETWORK: "1" steps: - uses: actions/checkout@v7 @@ -47,18 +53,31 @@ jobs: go-version-file: go.mod cache: true + - name: Install gotestsum + run: go install gotest.tools/gotestsum@v1.13.0 + - name: Integration tests - # The full ./tests/ default-tag suite (~300 tests). Moved out of - # PR CI in PR #121 — sustained 15m timeouts on public runners. + # Full ./tests/ default-tag suite (~300 tests) on the dedicated + # self-hosted runner with real networking. These multi-daemon + # tests are not perfectly hermetic — under the accumulated load of + # a 300-test serial run a few slow past their tight internal + # timeouts or collide on a port, yet pass cleanly in isolation. + # gotestsum --rerun-fails re-runs any failure in a fresh, nearly + # empty process (real isolation): a test that then passes was + # flaky-under-load, not broken; one that fails even alone is a + # genuine bug and still fails the job. run: | - go test -parallel 1 -count=1 -timeout 45m \ - -skip 'TestMultipleLargeWrites|TestConcurrentDialEncryptDecrypt|TestHTTPPrivateWithTrust|TestInviteInboxCapEnforced|TestConnectionLimits|TestRC6PeerRestartRecoveryEndToEnd|TestEndToEndRelay|TestNameserverOverwriteA|TestNameserverPersistence|TestNameserverRegisterN|TestNameserver$' \ - ./tests/ + "$(go env GOPATH)/bin/gotestsum" \ + --rerun-fails=3 --rerun-fails-max-failures=50 --packages=./tests -- \ + -parallel 6 -count=1 -timeout 60m \ + -skip 'TestMultipleLargeWrites|TestConcurrentDialEncryptDecrypt|TestHTTPPrivateWithTrust|TestInviteInboxCapEnforced|TestConnectionLimits|TestRC6PeerRestartRecoveryEndToEnd|TestEndToEndRelay|TestNameserverOverwriteA|TestNameserverPersistence|TestNameserverRegisterN|TestNameserver$' full: name: Full test suite (nightly tag) - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: [self-hosted, pilot-nightly] + timeout-minutes: 85 + env: + PILOT_REAL_NETWORK: "1" steps: - uses: actions/checkout@v7 @@ -67,20 +86,19 @@ jobs: go-version-file: go.mod cache: true + - name: Install gotestsum + run: go install gotest.tools/gotestsum@v1.13.0 + - name: Full test (default + nightly) - # 30m timeout: fuzz / scaling / stress files have wide variance. - # 200 extra tests beyond what PR CI sees, including: - # - 7 fuzz suites (daemon, protocol, config, crypto, - # fsutil_pool, ipc, secure, registry) - # - zz_scaling_test.go (16 funcs) - # - zz_stress_test.go (1 func) - # The skip list mirrors PR CI: those tests are known-slow - # and run sufficiently via the architecture-gates stress - # harness instead. + # default+nightly tags: fuzz / scaling / stress + the real-network + # integration suite. Same rerun-in-isolation policy as the + # integration job (see its comment): flaky-under-load tests pass on + # a fresh-process rerun; genuine failures still fail the job. run: | - go test -tags nightly -parallel 1 -count=1 -timeout 50m \ - -skip 'TestMultipleLargeWrites|TestConcurrentDialEncryptDecrypt|TestHTTPPrivateWithTrust|TestInviteInboxCapEnforced|TestConnectionLimits|TestRC6PeerRestartRecoveryEndToEnd|TestEndToEndRelay|TestNameserverOverwriteA|TestNameserverPersistence|TestNameserverRegisterN|TestNameserver$' \ - ./tests/ + "$(go env GOPATH)/bin/gotestsum" \ + --rerun-fails=3 --rerun-fails-max-failures=50 --packages=./tests -- \ + -tags nightly -parallel 6 -count=1 -timeout 65m \ + -skip 'TestMultipleLargeWrites|TestConcurrentDialEncryptDecrypt|TestHTTPPrivateWithTrust|TestInviteInboxCapEnforced|TestConnectionLimits|TestRC6PeerRestartRecoveryEndToEnd|TestEndToEndRelay|TestNameserverOverwriteA|TestNameserverPersistence|TestNameserverRegisterN|TestNameserver$' - name: Benchmark sanity (no upload) # Quick benchmark smoke — just confirms zz_bench_*_test.go still @@ -88,3 +106,62 @@ jobs: # benchstat against a fresh laptop run, not GitHub runners. run: | go test -run='^$' -bench='.' -benchtime=1x -count=1 -timeout 20m ./tests/ + + # --------------------------------------------------------------------------- + # Nightly beta release: if BOTH test jobs pass, tag a nightly prerelease. + # The tag push (as the pilot-release-bot App, so it actually triggers + # workflows) fires release.yml, which builds + publishes the cross-platform + # binaries as a GitHub prerelease. release-watch tracks -nightly tags without + # cascading (prereleases are not fanned out). One tag per UTC day; a second + # green run the same day is a no-op. + # --------------------------------------------------------------------------- + nightly-beta: + name: Nightly beta release + needs: [integration, full] + # Only cut a release from main (a branch test-run must not tag). + if: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + runs-on: ubuntu-latest + permissions: + contents: write + env: + HAS_APP: ${{ secrets.RELEASE_APP_ID != '' }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Mint App token + # Default GITHUB_TOKEN-pushed tags do NOT trigger workflows (recursion + # guard), so release.yml would never fire. The App token has no such + # restriction. Falls back to no-op when the secrets aren't present. + id: app + if: env.HAS_APP == 'true' + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - name: Tag nightly prerelease + env: + GH_TOKEN: ${{ steps.app.outputs.token || github.token }} + run: | + set -euo pipefail + # Next patch above the highest stable tag, suffixed -nightly.. + stable=$(git tag -l 'v*' --sort=-v:refname \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1) + stable=${stable:-v0.0.0} + IFS='.' read -r M m p <<<"${stable#v}" + date=$(date -u +%Y%m%d) + tag="v${M}.${m}.$((p + 1))-nightly.${date}" + + if git ls-remote --tags origin "refs/tags/${tag}" | grep -q "$tag"; then + echo "::notice::${tag} already exists — nightly beta already cut today" + exit 0 + fi + + git config user.name "pilot-release-bot" + git config user.email "release-bot@pilotprotocol.network" + git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git tag "$tag" + git push origin "$tag" + echo "::notice::pushed nightly prerelease tag ${tag} → release.yml will build + publish"