diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index f2c8a8373..af09c4196 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -323,14 +323,14 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - # Each database runs in five parts on separate runners: the native - # database suites and TLS checks, the shared corpus with its profiles and + # Each database runs in six parts on separate runners: the native + # database suites and TLS checks, the shared corpus, its profiles and # restart cases, the published-SDK direction profiles, the corrected # marker cohort, and the other cohorts. Every part bootstraps the same # database and native build; no part depends on another. matrix: label: [mysql-8.0, existing-php-mysql-matrix, mariadb-10.11] - part: [native-tests, corpus, cohorts, sdk-directions, corrected-markers] + part: [native-tests, corpus, restarts, cohorts, sdk-directions, corrected-markers] include: - label: mysql-8.0 image: mirror.gcr.io/library/mysql@sha256:62fb722c78b24245ddff1796a0fcee4a49cc5b87e0aaaf20c92d1da9e0a2497b @@ -487,8 +487,8 @@ jobs: -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ "$image" artisan server:bootstrap --force > parity-evidence/published-upgrade-bootstrap.txt - - name: Compare three runtimes, recover killed native nodes and refuse published PHP takeover - if: ${{ matrix.part == 'corpus' }} + - name: Start the published PHP and native nodes the corpus parts share + if: ${{ matrix.part == 'corpus' || matrix.part == 'restarts' }} id: prepare_runtime_comparison run: | set -euo pipefail @@ -509,7 +509,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-php:8080/api/ready > parity-evidence/php-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-php:8080/api/ready > parity-evidence/php-readiness.json; then ready=1 break fi @@ -547,7 +547,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-a:8080/api/ready > parity-evidence/native-a-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-a:8080/api/ready > parity-evidence/native-a-readiness.json; then ready=1 break fi @@ -558,13 +558,23 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/native-b-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/native-b-readiness.json; then ready=1 break fi sleep 1 done test "$ready" = 1 + - name: Compare three runtimes over the shared fixture corpus + if: ${{ matrix.part == 'corpus' }} + run: | + set -euo pipefail + # Bounded container log tails make a failure in this step diagnosable from its own log. + container_log_tails() { + bash scripts/conformance/server-parity/container-log-tails.sh + } + trap container_log_tails ERR + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) mkdir -p parity-consumers curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar @@ -617,6 +627,20 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$? + # A native fixture difference fails the step only after the + # comparison has run, so one difference cannot hide the rest. + test "$native_record_status" -eq 0 + test "$compare_status" -eq 0 + - name: Compare published patch, legacy-marker and range-repair profiles, recover killed native nodes and refuse published PHP takeover + if: ${{ matrix.part == 'restarts' }} + run: | + set -euo pipefail + # Bounded container log tails make a failure in this step diagnosable from its own log. + container_log_tails() { + bash scripts/conformance/server-parity/container-log-tails.sh + } + trap container_log_tails ERR + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) bash scripts/conformance/server-parity/published-sdk-patch-profile.sh mysql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/repeated-patch-profile.sh mysql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/legacy-marker-profile.sh mysql http://parity-php:8080 http://parity-native-a:8080 @@ -631,7 +655,7 @@ jobs: docker kill --signal=KILL "$RESOURCE_SCOPE-native-a" test "$(docker inspect --format='{{.State.ExitCode}}' "$RESOURCE_SCOPE-native-a")" = 137 docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/native-survivor-readiness.json + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/native-survivor-readiness.json docker run -d --name "$RESOURCE_SCOPE-recovered" --network "$RESOURCE_SCOPE" --network-alias=parity-recovered \ --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token -e DW_BIND_ADDRESS=0.0.0.0:8080 \ @@ -642,7 +666,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-recovered:8080/api/ready > parity-evidence/native-restart-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-recovered:8080/api/ready > parity-evidence/native-restart-readiness.json; then ready=1 break fi @@ -687,10 +711,6 @@ jobs: catalog php_ref > parity-evidence/catalog-after.json cmp parity-evidence/php-before.json parity-evidence/php-after.json cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json - # A native fixture difference fails the step only after every later - # check has run, so one difference cannot hide the rest. - test "$native_record_status" -eq 0 - test "$compare_status" -eq 0 - name: Compare every published-SDK direction profile in three concurrent lanes if: ${{ matrix.part == 'sdk-directions' }} run: | @@ -716,7 +736,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-a:8080/api/ready > parity-evidence/sdk-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-a:8080/api/ready > parity-evidence/sdk-native-readiness.json; then ready=1 break fi @@ -743,7 +763,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/cohort-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/cohort-native-readiness.json; then ready=1 break fi @@ -796,7 +816,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/corrected-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/corrected-native-readiness.json; then ready=1 break fi @@ -934,14 +954,14 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - # Each database runs in five parts on separate runners: the native - # database suites and TLS checks, the shared corpus with its profiles and + # Each database runs in six parts on separate runners: the native + # database suites and TLS checks, the shared corpus, its profiles and # restart cases, the published-SDK direction profiles, the corrected # marker cohort, and the other cohorts. Every part bootstraps the same # database and native build; no part depends on another. matrix: label: [existing-php-matrix, postgres-17] - part: [native-tests, corpus, cohorts, sdk-directions, corrected-markers] + part: [native-tests, corpus, restarts, cohorts, sdk-directions, corrected-markers] include: - label: existing-php-matrix image: mirror.gcr.io/library/postgres@sha256:f1c3376c26f2609ab9f29f71f824103fe2fcd8ee0346485cb6122a4f93df6f94 @@ -1064,8 +1084,8 @@ jobs: -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ "$image" artisan server:bootstrap --force > parity-evidence/published-upgrade-bootstrap.txt - - name: Compare three runtimes, recover killed native nodes and refuse published PHP takeover - if: ${{ matrix.part == 'corpus' }} + - name: Start the published PHP and native nodes the corpus parts share + if: ${{ matrix.part == 'corpus' || matrix.part == 'restarts' }} id: prepare_runtime_comparison env: POSTGRES_IMAGE: ${{ matrix.image }} @@ -1088,7 +1108,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-php:8080/api/ready > parity-evidence/php-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-php:8080/api/ready > parity-evidence/php-readiness.json; then ready=1 break fi @@ -1125,7 +1145,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-a:8080/api/ready > parity-evidence/native-a-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-a:8080/api/ready > parity-evidence/native-a-readiness.json; then ready=1 break fi @@ -1136,13 +1156,25 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/native-b-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/native-b-readiness.json; then ready=1 break fi sleep 1 done test "$ready" = 1 + - name: Compare three runtimes over the shared fixture corpus + if: ${{ matrix.part == 'corpus' }} + env: + POSTGRES_IMAGE: ${{ matrix.image }} + run: | + set -euo pipefail + # Bounded container log tails make a failure in this step diagnosable from its own log. + container_log_tails() { + bash scripts/conformance/server-parity/container-log-tails.sh + } + trap container_log_tails ERR + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) mkdir -p parity-consumers curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar @@ -1195,6 +1227,22 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$? + # A native fixture difference fails the step only after the + # comparison has run, so one difference cannot hide the rest. + test "$native_record_status" -eq 0 + test "$compare_status" -eq 0 + - name: Compare published patch, legacy-marker and range-repair profiles, recover killed native nodes and refuse published PHP takeover + if: ${{ matrix.part == 'restarts' }} + env: + POSTGRES_IMAGE: ${{ matrix.image }} + run: | + set -euo pipefail + # Bounded container log tails make a failure in this step diagnosable from its own log. + container_log_tails() { + bash scripts/conformance/server-parity/container-log-tails.sh + } + trap container_log_tails ERR + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) bash scripts/conformance/server-parity/published-sdk-patch-profile.sh pgsql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/repeated-patch-profile.sh pgsql http://parity-php:8080 http://parity-native-a:8080 bash scripts/conformance/server-parity/legacy-marker-profile.sh pgsql http://parity-php:8080 http://parity-native-a:8080 @@ -1209,7 +1257,7 @@ jobs: docker kill --signal=KILL "$RESOURCE_SCOPE-native-a" test "$(docker inspect --format='{{.State.ExitCode}}' "$RESOURCE_SCOPE-native-a")" = 137 docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/native-survivor-readiness.json + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/native-survivor-readiness.json docker run -d --name "$RESOURCE_SCOPE-recovered" --network "$RESOURCE_SCOPE" --network-alias=parity-recovered \ --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token -e DW_BIND_ADDRESS=0.0.0.0:8080 \ @@ -1220,7 +1268,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-recovered:8080/api/ready > parity-evidence/native-restart-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-recovered:8080/api/ready > parity-evidence/native-restart-readiness.json; then ready=1 break fi @@ -1265,10 +1313,6 @@ jobs: catalog php_ref > parity-evidence/catalog-after.json cmp parity-evidence/php-before.json parity-evidence/php-after.json cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json - # A native fixture difference fails the step only after every later - # check has run, so one difference cannot hide the rest. - test "$native_record_status" -eq 0 - test "$compare_status" -eq 0 - name: Compare every published-SDK direction profile in three concurrent lanes if: ${{ matrix.part == 'sdk-directions' }} run: | @@ -1294,7 +1338,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-a:8080/api/ready > parity-evidence/sdk-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-a:8080/api/ready > parity-evidence/sdk-native-readiness.json; then ready=1 break fi @@ -1321,7 +1365,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/cohort-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/cohort-native-readiness.json; then ready=1 break fi @@ -1376,7 +1420,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-native-b:8080/api/ready > parity-evidence/corrected-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-native-b:8080/api/ready > parity-evidence/corrected-native-readiness.json; then ready=1 break fi @@ -1556,13 +1600,13 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - # Five parts on separate runners: the native suites and codec, the - # shared corpus with its profiles and restart cases, the published-SDK + # Six parts on separate runners: the native suites and codec, the + # shared corpus, its profiles and restart cases, the published-SDK # direction profiles, the corrected marker cohort, and the other # cohorts. Every part builds the same native server; no part depends on # another. matrix: - part: [native-tests, corpus, cohorts, sdk-directions, corrected-markers] + part: [native-tests, corpus, restarts, cohorts, sdk-directions, corrected-markers] env: DW_PARITY_PART: ${{ matrix.part }} steps: @@ -1648,8 +1692,8 @@ jobs: mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \ cargo run --locked --offline --example codec-fixtures -- \ ../tests/Fixtures/ServerParity/Codec/v1.json /evidence/php-codec.json > parity-evidence/cross-decode.json - - name: Execute and compare Rust, published PHP and embedded fixtures - if: ${{ matrix.part == 'corpus' }} + - name: Start the published PHP and native nodes the corpus parts share + if: ${{ matrix.part == 'corpus' || matrix.part == 'restarts' }} id: prepare_runtime_comparison env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} @@ -1706,7 +1750,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --entrypoint curl --network "$RESOURCE_SCOPE" "$image" \ - -fsS "http://parity-$target:8080/api/ready" > "parity-evidence/$target-readiness.json"; then + -fsS --connect-timeout 2 --max-time 5 "http://parity-$target:8080/api/ready" > "parity-evidence/$target-readiness.json"; then ready=1 break fi @@ -1714,16 +1758,29 @@ jobs: done test "$ready" = 1 done - export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) - mkdir -p parity-consumers - curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar - printf '%s %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt - export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar + # The restarts part needs the maintenance loop as much as the corpus does. docker run -d --name "$RESOURCE_SCOPE-php-maintenance" --network "$RESOURCE_SCOPE" \ --user=1000:1000 --cpus=0.25 --memory=256m --memory-swap=256m --entrypoint sh \ -v "$PWD/parity-state:/state" -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/php.sqlite -e DB_SQLITE_BUSY_TIMEOUT=30000 \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ "$image" -c 'while :; do php artisan workflow:v2:repair-pass --json; php artisan schedule:evaluate --limit=100 --json; sleep 1; done' + - name: Execute and compare Rust, published PHP and embedded fixtures + if: ${{ matrix.part == 'corpus' }} + env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + run: | + set -euo pipefail + # Bounded container log tails make a failure in this step diagnosable from its own log. + container_log_tails() { + bash scripts/conformance/server-parity/container-log-tails.sh + } + trap container_log_tails ERR + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) + export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) + mkdir -p parity-consumers + curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar + printf '%s %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt + export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ @@ -1778,6 +1835,23 @@ jobs: # its published PHP record runs with PHP's maintenance loop stopped. update_validation_status=0 bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 || update_validation_status=$? + # A native fixture difference fails the step only after the + # comparison has run, so one difference cannot hide the rest. + test "$native_record_status" -eq 0 + test "$compare_status" -eq 0 + test "$update_validation_status" -eq 0 + - name: Compare published patch, legacy-marker and range-repair profiles + if: ${{ matrix.part == 'restarts' }} + env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + run: | + set -euo pipefail + # Bounded container log tails make a failure in this step diagnosable from its own log. + container_log_tails() { + bash scripts/conformance/server-parity/container-log-tails.sh + } + trap container_log_tails ERR + image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.1.json) bash scripts/conformance/server-parity/published-sdk-patch-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 bash scripts/conformance/server-parity/repeated-patch-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 bash scripts/conformance/server-parity/legacy-marker-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 @@ -1834,7 +1908,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --entrypoint curl --network "$RESOURCE_SCOPE" "$image" \ - -fsS http://parity-recovered:8080/api/ready > parity-evidence/restart-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-recovered:8080/api/ready > parity-evidence/restart-readiness.json; then ready=1 break fi @@ -1847,11 +1921,6 @@ jobs: php scripts/conformance/server-parity/cooperative-restart.php finish http://parity-recovered:8080 /evidence/cooperative-restart-receipt.json > /evidence/cooperative-restart-finish.json php scripts/conformance/server-parity/restart.php finish http://parity-recovered:8080 /evidence/restart-receipt.json > /evidence/restart-finish.json ' - # A native fixture difference fails the step only after every later - # check has run, so one difference cannot hide the rest. - test "$native_record_status" -eq 0 - test "$compare_status" -eq 0 - test "$update_validation_status" -eq 0 - name: Compare every published-SDK direction profile in three concurrent lanes if: ${{ matrix.part == 'sdk-directions' }} env: @@ -1884,7 +1953,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-rust:8080/api/ready > parity-evidence/sdk-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-rust:8080/api/ready > parity-evidence/sdk-native-readiness.json; then ready=1 break fi @@ -1918,7 +1987,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-recovered:8080/api/ready > parity-evidence/cohort-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-recovered:8080/api/ready > parity-evidence/cohort-native-readiness.json; then ready=1 break fi @@ -1986,7 +2055,7 @@ jobs: ready=0 for attempt in $(seq 1 30); do if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS http://parity-recovered:8080/api/ready > parity-evidence/corrected-native-readiness.json; then + -fsS --connect-timeout 2 --max-time 5 http://parity-recovered:8080/api/ready > parity-evidence/corrected-native-readiness.json; then ready=1 break fi diff --git a/tests/Unit/ServerParityPartsTest.mjs b/tests/Unit/ServerParityPartsTest.mjs index 8d953121d..1cf74c2da 100644 --- a/tests/Unit/ServerParityPartsTest.mjs +++ b/tests/Unit/ServerParityPartsTest.mjs @@ -1,12 +1,15 @@ -// Each Shared Server fixtures database job runs in five parts on separate +// Each Shared Server fixtures database job runs in six parts on separate // runners. Every step either prepares the part (and so runs in every part) or -// belongs to exactly one part, so no check runs twice or not at all. +// belongs to exactly one part, so no check runs twice or not at all. The one +// exception starts the published PHP and native nodes that the corpus and +// restarts parts each need, and runs in exactly those two. import test from 'node:test'; import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; const workflow = readFileSync('.github/workflows/server-parity.yml', 'utf8'); -const PARTS = ['native-tests', 'corpus', 'cohorts', 'sdk-directions', 'corrected-markers']; +const PARTS = ['native-tests', 'corpus', 'restarts', 'cohorts', 'sdk-directions', 'corrected-markers']; +const SHARED_NODES = 'Start the published PHP and native nodes the corpus parts share'; const EVERY_PART = [/^actions\/checkout@/, /^Fetch /, /^Create TLS fixture certificates$/, /^Bootstrap independent /, /^Collect /, /^Summarize /, /^Retain /]; @@ -30,6 +33,8 @@ function steps(block) { // The parts a step runs in, read from its condition. function partsOf(step) { if (EVERY_PART.some(pattern => pattern.test(step.name))) return PARTS; + const either = /^\$\{\{ matrix\.part == '([a-z-]+)' \|\| matrix\.part == '([a-z-]+)' \}\}$/.exec(step.condition ?? ''); + if (either) return [either[1], either[2]]; const only = /matrix\.part == '([a-z-]+)'/.exec(step.condition ?? ''); if (only) return [only[1]]; const except = /matrix\.part != '([a-z-]+)'/.exec(step.condition ?? ''); @@ -39,9 +44,9 @@ function partsOf(step) { for (const [name, key] of [['mysql-execution', 'Compare three runtimes'], ['postgres-storage', 'Compare three runtimes'], ['execution-and-codec', 'Execute and compare Rust']]) { - test(`${name} runs every check in exactly one of five parts`, () => { + test(`${name} runs every check in exactly one of six parts`, () => { const block = job(name); - assert.match(block, /^ {8}part: \[native-tests, corpus, cohorts, sdk-directions, corrected-markers\]$/m); + assert.match(block, /^ {8}part: \[native-tests, corpus, restarts, cohorts, sdk-directions, corrected-markers\]$/m); assert.match(block, /^ {6}DW_PARITY_PART: \$\{\{ matrix\.part \}\}$/m); assert.match(block, /^ {4}timeout-minutes: 60$/m); const all = steps(block); @@ -49,12 +54,31 @@ for (const [name, key] of [['mysql-execution', 'Compare three runtimes'], ['post const parts = partsOf(step); if (parts.length === PARTS.length) { assert.ok(EVERY_PART.some(pattern => pattern.test(step.name)), `${name}: "${step.name}" must belong to one part`); + } else if (step.name === SHARED_NODES) { + assert.deepEqual(parts, ['corpus', 'restarts'], `${name}: the shared nodes start for exactly the corpus parts`); } else if (step.name !== 'Build the native server') { assert.equal(parts.length, 1, `${name}: "${step.name}"`); } } const inPart = part => all.filter(step => partsOf(step).length === 1 && partsOf(step)[0] === part).map(step => step.name); assert.ok(inPart('corpus').some(step => step.startsWith(key)), `${name}: the corpus comparison is in the corpus part`); + assert.equal(all.filter(step => step.name === SHARED_NODES).length, 1, `${name}: one shared node start`); + assert.equal(all.find(step => step.name === SHARED_NODES).id, 'prepare_runtime_comparison', name); + // Both parts need every node the original step started, including the + // published PHP maintenance loop the patch profiles pause and resume. + assert.match(all.find(step => step.name === SHARED_NODES).text, /--name "\$RESOURCE_SCOPE-php-maintenance"/, name); + // The profiles and restart cases run in their own part, against nodes the + // part starts itself, so the corpus part no longer waits for them. + const restarts = inPart('restarts'); + assert.equal(restarts.length, 1, `${name}: ${restarts}`); + assert.match(restarts[0], /^Compare published patch, legacy-marker and range-repair profiles/, name); + const profiles = all.find(step => step.name === restarts[0]).text; + for (const profile of ['published-sdk-patch-profile.sh', 'legacy-marker-profile.sh', 'range-repair-profile.sh']) { + assert.ok(profiles.includes(profile), `${name}: ${profile} runs in the restarts part`); + } + const corpus = all.find(step => step.name.startsWith(key)).text; + assert.ok(!/published-sdk-patch-profile|legacy-marker-profile|range-repair-profile|docker kill/.test(corpus), + `${name}: the corpus comparison no longer runs the profiles or restart cases`); // The published-SDK direction lanes run once, in their own part, so the // corpus part no longer waits for them. assert.deepEqual(inPart('sdk-directions'), ['Compare every published-SDK direction profile in three concurrent lanes'], name); @@ -99,3 +123,9 @@ test('the database bootstraps run the native suites only in the native-tests par assert.match(bootstrap.text, /'if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']* --test execution [^']*; fi && cargo build --locked --offline'/, name); } }); + +test('every readiness probe bounds each request, so one silent connection cannot stall a part', () => { + const probes = workflow.match(/curl[^\n]*\n?[^\n]*\/api\/ready/g) ?? []; + assert.ok(probes.length >= 20, `${probes.length} readiness probes`); + for (const probe of probes) assert.match(probe, /--connect-timeout \d+ --max-time \d+ "?http:/, probe); +});