From 7be8bd4d32414576f4eb3d53cca87b5c42e3f261 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Tue, 27 Oct 2026 12:45:23 +0100 Subject: [PATCH] ci: fall back off a slow Azure apt mirror in smoke tests (#15203) A degraded node in the azure.archive.ubuntu.com pool serves package indexes at full speed but archives at tens of kB/s. The runner images set apt timeouts that only catch a full stall, so apt never moves to archive.ubuntu.com. The Playwright dependency step then hits its 15 minute limit and setup-r takes about 20 minutes. The Azure line is dropped from apt-mirrors.txt only when the mirror is measured slow, so healthy runs keep using it. - Sample one .deb from the Azure mirror before setup-r, whose apt calls cannot be wrapped, and drop the mirror when it stays under 1 MB/s for 10 seconds - Run the later apt steps through a helper that gives each attempt 3 minutes, drops the mirror after a failure, and retries - Run dpkg --configure -a before each retry, in case a timeout interrupted an install (not exercised by the probes) The logic is inline in test-smokes.yml rather than a local composite action, because release mode checks out older tags that lack the action directory. See #15186 (job that hit the 15 minute limit), actions/runner-images#14823, and actions/runner-images#14596. (cherry picked from commit 1d82b1219ba8566b14bca5d12ba74d6dde034558) --- .github/workflows/test-smokes.yml | 86 +++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index f014105f63..7eb414d2be 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -72,6 +72,70 @@ jobs: echo "TEMP=${{ runner.temp }}" >> $GITHUB_ENV shell: bash + # Drops a slow azure.archive.ubuntu.com from the apt mirror list so the job uses + # archive.ubuntu.com (actions/runner-images#14823): sampled here before setup-r, + # and after a failed attempt in later apt steps run via $APT_WITH_FALLBACK. + # Inline, not a local action, because release mode checks out older tags. + - name: Configure apt mirror fallback + if: runner.os == 'Linux' + shell: bash + env: + APT_MIRROR_SAMPLE_PACKAGE: fonts-wqy-zenhei + APT_MIRROR_MIN_SPEED: "1000000" + run: | + mirrors=/etc/apt/apt-mirrors.txt + helper="$RUNNER_TEMP/apt-with-mirror-fallback.sh" + cat > "$helper" <<'EOF' + #!/usr/bin/env bash + # Usage: apt-with-mirror-fallback.sh '' + # Runs the command as root with a per-attempt timeout; timeout runs as + # root so the kill reaches apt and dpkg children. + mirrors=/etc/apt/apt-mirrors.txt + log=$(mktemp) + for attempt in 1 2 3; do + echo "::group::Attempt $attempt of 3" + start=$SECONDS + sudo env "PATH=$PATH" timeout --kill-after=15 180 bash -c "$1" 2>&1 | tee "$log" + status=${PIPESTATUS[0]} + echo "::endgroup::" + echo "Attempt $attempt exited $status after $((SECONDS - start))s; hosts served:" + grep -oE '^(Get|Hit):[0-9]+ https?://[^/ ]+' "$log" | awk '{print $2}' | sort | uniq -c + [ "$status" -eq 0 ] && exit 0 + if grep -q 'azure\.archive\.ubuntu\.com' "$mirrors" 2>/dev/null; then + echo "::warning::apt attempt $attempt failed (exit $status); dropping azure.archive.ubuntu.com from $mirrors" + sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors" + fi + # Finishes any install the timeout interrupted; a no-op otherwise. + sudo dpkg --configure -a + done + echo "::error::apt command failed after 3 attempts" + exit 1 + EOF + chmod +x "$helper" + echo "APT_WITH_FALLBACK=$helper" >> "$GITHUB_ENV" + + if ! grep -q 'azure\.archive\.ubuntu\.com' "$mirrors" 2>/dev/null; then + echo "No Azure mirror configured; skipping the speed sample" + exit 0 + fi + set +e + sudo apt-get update -qq + # The printed URI may name the mirror+file: list rather than a host, so keep only the pool path. + uri=$(apt-get download --print-uris -qq "$APT_MIRROR_SAMPLE_PACKAGE" | head -n 1 | cut -d"'" -f2) + if [[ "$uri" != */pool/* ]]; then + echo "::warning::Could not resolve a pool path for $APT_MIRROR_SAMPLE_PACKAGE; skipping the speed sample" + exit 0 + fi + url="http://azure.archive.ubuntu.com/ubuntu/pool/${uri#*/pool/}" + speed=$(curl -fsS -o /dev/null --speed-limit "$APT_MIRROR_MIN_SPEED" --speed-time 10 --max-time 60 -w '%{speed_download}' "$url") + status=$? + echo "Sampled $url: exit $status, ${speed:-0} bytes/s" + if [ "$status" -ne 0 ]; then + echo "::warning::azure.archive.ubuntu.com is slow or unreachable (curl exit $status); dropping it from $mirrors" + sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors" + fi + exit 0 + - name: Set up R uses: r-lib/actions/setup-r@v2 with: @@ -140,7 +204,12 @@ jobs: && (format('{0}', inputs.buckets) == '' || contains(inputs.buckets, 'playwright-tests.test.ts')) timeout-minutes: 15 - run: npx playwright install-deps + run: | + if [ -n "${APT_WITH_FALLBACK:-}" ]; then + "$APT_WITH_FALLBACK" 'npx playwright install-deps' + else + npx playwright install-deps + fi working-directory: ./tests/integration/playwright - name: Install Playwright Browsers @@ -182,19 +251,18 @@ jobs: - name: Install missing system deps if: runner.os == 'Linux' run: | - sudo apt-get update -y - sudo apt-get install -y libcurl4-openssl-dev - sudo apt-get install -y libxml2-utils - sudo apt-get install -y libharfbuzz-dev libfribidi-dev - sudo apt-get install -y poppler-utils - sudo apt-get install -y libnode-dev + "$APT_WITH_FALLBACK" 'apt-get update -y && + apt-get install -y libcurl4-openssl-dev && + apt-get install -y libxml2-utils && + apt-get install -y libharfbuzz-dev libfribidi-dev && + apt-get install -y poppler-utils && + apt-get install -y libnode-dev' - name: Install rsvg-convert for SVG conversion tests # Only do it on linux runner, and only if we are running the relevant tests if: runner.os == 'Linux' && (format('{0}', inputs.buckets) == '' || contains(inputs.buckets, 'render-pdf-svg-conversion') || contains(inputs.buckets, 'pdf-standard')) run: | - sudo apt-get update -y - sudo apt-get install -y librsvg2-bin + "$APT_WITH_FALLBACK" 'apt-get update -y && apt-get install -y librsvg2-bin' - name: Restore R packages id: restore-r-packages