Fix CI reliability: test compile OOM, separate dependency fetch, failure log tails

.github/workflows/server-parity.yml

@@ -363,6 +363,17 @@ jobs:
             -v "$PWD/mysql-tls:/tls" --entrypoint openssl "$image" req -x509 -newkey rsa:2048 \
             -nodes -sha256 -days 1 -keyout /tls/untrusted.key -out /tls/untrusted.crt -subj /CN=untrusted-development-ca \
             >> parity-evidence/tls-certificate-generation.txt 2>&1
+      - name: Fetch locked Rust dependencies
+        # A registry or network failure belongs to this step, not to the
+        # offline native builds and tests that follow.
+        run: |
+          set -euo pipefail
+          mkdir -p cargo-home
+          docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=2g --memory-swap=2g \
+            -e CARGO_HOME=/cargo -e CARGO_NET_RETRY=10 \
+            -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -w /source/rust \
+            mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \
+            cargo fetch --locked
       - name: Bootstrap independent published PHP and native MySQL databases
         run: |
           set -euo pipefail
@@ -396,14 +407,15 @@ jobs:
             -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \
             -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \
             "$image" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt
-          docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=2g --memory-swap=2g \
+          # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101).
+          docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=4g --memory-swap=4g \
             -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \
             -e DW_TEST_MYSQL_URL=mysql://root:parity-disposable-password@parity-mysql:3306/php_ref \
             -e DW_EXECUTION_MYSQL_URL=mysql://root:parity-disposable-password@parity-mysql:3306/php_ref \
             -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" \
             -v "$PWD/parity-evidence:/evidence" -w /source/rust --entrypoint bash \
             mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 -c \
-            'cargo test --locked --lib runtime::mysql:: -- --ignored --skip runtime::mysql::tests::bootstrap_child --nocapture --test-threads=2 > /evidence/mysql-tests.txt && cargo test --locked --test execution -- --nocapture --test-threads=2 > /evidence/mysql-execution-tests.txt && cargo build --locked'
+            'cargo test --locked --offline --lib runtime::mysql:: -- --ignored --skip runtime::mysql::tests::bootstrap_child --nocapture --test-threads=2 > /evidence/mysql-tests.txt && cargo test --locked --offline --test execution -- --nocapture --test-threads=2 > /evidence/mysql-execution-tests.txt && cargo build --locked --offline'
           docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \
             -e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=native_ref \
             -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \
@@ -424,6 +436,11 @@ jobs:
         id: prepare_runtime_comparison
         run: |
           set -euo pipefail
+          # Bounded container log tails make a failure in this step diagnosable from its own log.
+          container_log_tails() {
+            bash scripts/conformance/server-parity/container-log-tails.sh
+          }
+          trap container_log_tails ERR
           image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json)
           docker run --rm --user="$(id -u):$(id -g)" --entrypoint composer \
             -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \
@@ -823,6 +840,17 @@ jobs:
           name: corrected-marker-adapters
           path: corrected-marker-adapters
           digest-mismatch: error
+      - name: Fetch locked Rust dependencies
+        # A registry or network failure belongs to this step, not to the
+        # offline native builds and tests that follow.
+        run: |
+          set -euo pipefail
+          mkdir -p cargo-home
+          docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=2g --memory-swap=2g \
+            -e CARGO_HOME=/cargo -e CARGO_NET_RETRY=10 \
+            -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -w /source/rust \
+            mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \
+            cargo fetch --locked
       - name: Bootstrap independent published PHP and native PostgreSQL databases
         env:
           POSTGRES_IMAGE: ${{ matrix.image }}
@@ -855,14 +883,15 @@ jobs:
             -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \
             -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \
             "$image" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt
-          docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=2g --memory-swap=2g \
+          # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101).
+          docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=4g --memory-swap=4g \
             -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \
             -e DW_TEST_POSTGRES_URL=postgres://parity:parity-disposable-password@parity-pg:5432/php_ref \
             -e DW_EXECUTION_POSTGRES_URL=postgres://parity:parity-disposable-password@parity-pg:5432/php_ref \
             -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" \
             -v "$PWD/parity-evidence:/evidence" -w /source/rust --entrypoint bash \
             mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 -c \
-            'cargo test --locked --lib runtime::postgres::tests::qualification_ -- --ignored --nocapture > /evidence/postgres-tests.txt && cargo test --locked --test execution -- --nocapture > /evidence/postgres-execution-tests.txt && cargo build --locked'
+            'cargo test --locked --offline --lib runtime::postgres::tests::qualification_ -- --ignored --nocapture > /evidence/postgres-tests.txt && cargo test --locked --offline --test execution -- --nocapture > /evidence/postgres-execution-tests.txt && cargo build --locked --offline'
           docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \
             -e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=pgsql -e DB_HOST=parity-pg -e DB_DATABASE=native_ref \
             -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \
@@ -885,6 +914,11 @@ jobs:
           POSTGRES_IMAGE: ${{ matrix.image }}
         run: |
           set -euo pipefail
+          # Bounded container log tails make a failure in this step diagnosable from its own log.
+          container_log_tails() {
+            bash scripts/conformance/server-parity/container-log-tails.sh
+          }
+          trap container_log_tails ERR
           image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json)
           docker run --rm --user="$(id -u):$(id -g)" --entrypoint composer \
             -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \
@@ -1316,17 +1350,29 @@ jobs:
           name: corrected-marker-adapters
           path: corrected-marker-adapters
           digest-mismatch: error
+      - name: Fetch locked Rust dependencies
+        # A registry or network failure belongs to this step, not to the
+        # offline native builds and tests that follow.
+        run: |
+          set -euo pipefail
+          mkdir -p cargo-home
+          docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=2g --memory-swap=2g \
+            -e CARGO_HOME=/cargo -e CARGO_NET_RETRY=10 \
+            -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -w /source/rust \
+            mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \
+            cargo fetch --locked
       - name: Execute official Rust codec and cross-decode with published PHP
         run: |
           set -euo pipefail
           mkdir -p cargo-home rust-target parity-evidence
-          docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=2g --memory-swap=2g \
+          # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101).
+          docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=4g --memory-swap=4g \
             -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \
             -e DW_UPGRADE_TEST_WRITABLE=/evidence -e DW_UPGRADE_TEST_READONLY=/source/parity-evidence \
             -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" \
             -v "$PWD/parity-evidence:/evidence" -w /source/rust --entrypoint bash \
             mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 -c \
-            'cargo test --locked --all-targets > /evidence/rust-tests.txt && cargo build --locked && cargo run --locked --example codec-fixtures -- ../tests/Fixtures/ServerParity/Codec/v1.json > /evidence/rust-codec.json'
+            'cargo test --locked --offline --all-targets > /evidence/rust-tests.txt && cargo build --locked --offline && cargo run --locked --offline --example codec-fixtures -- ../tests/Fixtures/ServerParity/Codec/v1.json > /evidence/rust-codec.json'
           image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json)
           docker run --rm --user="$(id -u):$(id -g)" --network=none --entrypoint php \
             -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \
@@ -1345,6 +1391,11 @@ jobs:
           RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
         run: |
           set -euo pipefail
+          # Bounded container log tails make a failure in this step diagnosable from its own log.
+          container_log_tails() {
+            bash scripts/conformance/server-parity/container-log-tails.sh
+          }
+          trap container_log_tails ERR
           image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.5.14.json)
           mkdir -p parity-state parity-evidence
           chmod 0777 parity-state parity-evidence

scripts/conformance/server-parity/container-log-tails.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Print bounded log tails of this job's containers, so a failed step is
+# diagnosable from its own log. Reads RESOURCE_SCOPE; changes nothing.
+set -uo pipefail
+test -n "${RESOURCE_SCOPE:-}" || exit 0
+echo "::group::Container log tails after failure"
+docker ps -a --filter "name=^${RESOURCE_SCOPE}-" --format '{{.Names}}' | while read -r container; do
+  echo "== $container"
+  docker logs --tail 40 "$container" 2>&1 | cut -c1-400
+done
+echo "::endgroup::"