From c1346a06e5ee9e8adda46406cd537928fc240bda Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Tue, 27 Oct 2026 11:54:09 +0100 Subject: [PATCH 1/2] Fall back off a slow Azure apt mirror in the smoke test job A degraded node in the azure.archive.ubuntu.com pool serves package indexes at full speed but archives at tens of kB/s. The runner images set Acquire::Retries 1 and Acquire::http::Timeout 15, which only catch a total stall, so apt never moves to the next entry in apt-mirrors.txt. The Playwright install-deps step then runs into its 15 minute limit and setup-r takes about 20 minutes (actions/runner-images#14823). The Azure line is dropped from apt-mirrors.txt only when the mirror is measured slow, so healthy runs keep using it and load moves to archive.ubuntu.com only when needed. Two checks decide: - Before setup-r, whose apt calls cannot be wrapped, curl samples one multi-megabyte .deb from the Azure mirror and drops it when the speed stays under 1 MB/s for 10 seconds. With the mirror throttled to 50 kB/s, setup-r then downloaded everything from archive.ubuntu.com. - The later apt steps run through a helper that gives each attempt three minutes (healthy runs take 20 to 45 seconds) and drops the mirror before retrying, which covers a node that degrades partway through a download. Probe runs caught such a node live; the retry finished in about 50 seconds on archive.ubuntu.com. The logic lives inline in test-smokes.yml instead of a local composite action: release mode checks out older tags, which would not contain the action directory. --- .github/workflows/test-smokes.yml | 84 +++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index f21f925c9c..74a91ddb99 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -122,6 +122,68 @@ jobs: echo "TEMP=${{ runner.temp }}" >> $GITHUB_ENV shell: bash + # Drops a slow azure.archive.ubuntu.com from the apt mirror list so the job uses + # archive.ubuntu.com (actions/runner-images#14823): sampled here before setup-r, + # and after a failed attempt in later apt steps run via $APT_WITH_FALLBACK. + # Inline, not a local action, because release mode checks out older tags. + - name: Configure apt mirror fallback + if: runner.os == 'Linux' + shell: bash + env: + APT_MIRROR_SAMPLE_PACKAGE: fonts-wqy-zenhei + APT_MIRROR_MIN_SPEED: "1000000" + run: | + mirrors=/etc/apt/apt-mirrors.txt + helper="$RUNNER_TEMP/apt-with-mirror-fallback.sh" + cat > "$helper" <<'EOF' + #!/usr/bin/env bash + # Usage: apt-with-mirror-fallback.sh '' + # Runs the command as root with a per-attempt timeout; timeout runs as + # root so the kill reaches apt and dpkg children. + mirrors=/etc/apt/apt-mirrors.txt + log=$(mktemp) + for attempt in 1 2 3; do + echo "::group::Attempt $attempt of 3" + start=$SECONDS + sudo env "PATH=$PATH" timeout --kill-after=15 180 bash -c "$1" 2>&1 | tee "$log" + status=${PIPESTATUS[0]} + echo "::endgroup::" + echo "Attempt $attempt exited $status after $((SECONDS - start))s; hosts served:" + grep -oE '^(Get|Hit):[0-9]+ https?://[^/ ]+' "$log" | awk '{print $2}' | sort | uniq -c + [ "$status" -eq 0 ] && exit 0 + if grep -q 'azure\.archive\.ubuntu\.com' "$mirrors" 2>/dev/null; then + echo "::warning::apt attempt $attempt failed (exit $status); dropping azure.archive.ubuntu.com from $mirrors" + sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors" + fi + done + echo "::error::apt command failed after 3 attempts" + exit 1 + EOF + chmod +x "$helper" + echo "APT_WITH_FALLBACK=$helper" >> "$GITHUB_ENV" + + if ! grep -q 'azure\.archive\.ubuntu\.com' "$mirrors" 2>/dev/null; then + echo "No Azure mirror configured; skipping the speed sample" + exit 0 + fi + set +e + sudo apt-get update -qq + # The printed URI may name the mirror+file: list rather than a host, so keep only the pool path. + uri=$(apt-get download --print-uris -qq "$APT_MIRROR_SAMPLE_PACKAGE" | head -n 1 | cut -d"'" -f2) + if [[ "$uri" != */pool/* ]]; then + echo "::warning::Could not resolve a pool path for $APT_MIRROR_SAMPLE_PACKAGE; skipping the speed sample" + exit 0 + fi + url="http://azure.archive.ubuntu.com/ubuntu/pool/${uri#*/pool/}" + speed=$(curl -fsS -o /dev/null --speed-limit "$APT_MIRROR_MIN_SPEED" --speed-time 10 --max-time 60 -w '%{speed_download}' "$url") + status=$? + echo "Sampled $url: exit $status, ${speed:-0} bytes/s" + if [ "$status" -ne 0 ]; then + echo "::warning::azure.archive.ubuntu.com is slow or unreachable (curl exit $status); dropping it from $mirrors" + sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors" + fi + exit 0 + - name: Set up R uses: r-lib/actions/setup-r@v2 with: @@ -191,7 +253,12 @@ jobs: || (format('{0}', inputs.buckets) == '' && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) timeout-minutes: 15 - run: npx playwright install-deps + run: | + if [ -n "${APT_WITH_FALLBACK:-}" ]; then + "$APT_WITH_FALLBACK" 'npx playwright install-deps' + else + npx playwright install-deps + fi working-directory: ./tests/integration/playwright - name: Install Playwright Browsers @@ -234,19 +301,18 @@ jobs: - name: Install missing system deps if: runner.os == 'Linux' run: | - sudo apt-get update -y - sudo apt-get install -y libcurl4-openssl-dev - sudo apt-get install -y libxml2-utils - sudo apt-get install -y libharfbuzz-dev libfribidi-dev - sudo apt-get install -y poppler-utils - sudo apt-get install -y libnode-dev + "$APT_WITH_FALLBACK" 'apt-get update -y && + apt-get install -y libcurl4-openssl-dev && + apt-get install -y libxml2-utils && + apt-get install -y libharfbuzz-dev libfribidi-dev && + apt-get install -y poppler-utils && + apt-get install -y libnode-dev' - name: Install rsvg-convert for SVG conversion tests # Only do it on linux runner, and only if we are running the relevant tests if: runner.os == 'Linux' && (format('{0}', inputs.buckets) == '' || contains(inputs.buckets, 'render-pdf-svg-conversion') || contains(inputs.buckets, 'pdf-standard')) run: | - sudo apt-get update -y - sudo apt-get install -y librsvg2-bin + "$APT_WITH_FALLBACK" 'apt-get update -y && apt-get install -y librsvg2-bin' - name: Install missing system deps (macOS) if: runner.os == 'macOS' From f2e01d169fb66cf5e3cc3c93f9cc0ca6a38e5ed8 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Tue, 27 Oct 2026 12:00:30 +0100 Subject: [PATCH 2/2] Recover an interrupted dpkg run before retrying an apt attempt The per-attempt timeout is not tied to the download phase. A slow download that finishes just before the deadline lets the kill land while dpkg unpacks, and every retry then stops on "dpkg was interrupted". dpkg --configure -a finishes that transaction and does nothing on a clean system. Precaution, untested: the probes only killed attempts during the download. --- .github/workflows/test-smokes.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 74a91ddb99..b7a7636021 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -155,6 +155,8 @@ jobs: echo "::warning::apt attempt $attempt failed (exit $status); dropping azure.archive.ubuntu.com from $mirrors" sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors" fi + # Finishes any install the timeout interrupted; a no-op otherwise. + sudo dpkg --configure -a done echo "::error::apt command failed after 3 attempts" exit 1