From 44c67d9b7838070985735e67de9a6b612b62c69c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:00:38 +0000 Subject: [PATCH 1/8] Register DW_RUST_MAINTENANCE_LOOP as a native-only setting The opt-in for native's maintenance loop (#797) was missing from the config audit's native-only names, so setting it logged "Unknown DW_ variables set in environment". The native image smoke test showed it. The shipped-environment test now sets it. Co-Authored-By: Claude Opus 5.5 --- rust/src/config_audit.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rust/src/config_audit.rs b/rust/src/config_audit.rs index a23e2ada0..1e583efed 100644 --- a/rust/src/config_audit.rs +++ b/rust/src/config_audit.rs @@ -502,7 +502,10 @@ pub fn audit(environment: &BTreeMap) -> Report { /// Names native reads that PHP's contract does not list. fn native_only(name: &str) -> bool { - matches!(name, "DW_BIND_ADDRESS" | "DW_ENV" | "DW_RUST_EXPERIMENTAL" | "DW_UPGRADE_BACKUP_FINGERPRINT") + matches!( + name, + "DW_BIND_ADDRESS" | "DW_ENV" | "DW_RUST_EXPERIMENTAL" | "DW_RUST_MAINTENANCE_LOOP" | "DW_UPGRADE_BACKUP_FINGERPRINT" + ) || ["DW_RUST_UPGRADE_QUEUE_", "DW_SCHEMA_TEST_", "DW_TAKEOVER_TEST_", "DW_TEST_"] .iter() .any(|prefix| name.starts_with(prefix)) @@ -820,6 +823,7 @@ mod tests { ("DW_WORKFLOW_MEMO_MIGRATION_RECOVERY", ""), ("DW_ENV_AUDIT_STRICT", "1"), ("DW_RUST_EXPERIMENTAL", "1"), + ("DW_RUST_MAINTENANCE_LOOP", "1"), ("DW_BIND_ADDRESS", "0.0.0.0:8080"), ]; let report = audit(&environment(&shipped)); From ed7a175a6764aa5fc85f3649411ccd3c0fdc6375 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:02:36 +0000 Subject: [PATCH 2/8] Package the native server as an opt-in release-candidate image Server #325's native server has no image yet. This adds one, as a separate repository (durableworkflow/server-native and ghcr.io/durable-workflow/server-native); the PHP image and its Release workflow are unchanged. Dockerfile.native builds the binary with the Shared Server fixtures' pinned compiler (--locked) and copies it onto a digest-pinned Debian 12 slim runtime. It runs as 1000:1000, stops on SIGTERM and listens on 8080. It uses the PHP image's paths, /app/database for SQLite and /app/storage for local external payload objects, whose absolute file:// paths PHP records in the database. It sets DW_RUST_MAINTENANCE_LOOP=1, because there is no separate scheduler container, and never DW_RUST_EXPERIMENTAL: the server still refuses to start without the explicit opt-in. The same image runs the takeover commands. scripts/ci/native-image-smoke.sh checks: - the image config; - the refusal without the opt-in; - a cold start on SQLite, PostgreSQL 17 and MySQL 8.0, with /api/ready, an authenticated /api/cluster/info and a clean SIGTERM exit; - upgrade-inspect refusing a missing SQLite source without creating it. Docker resources are named from RESOURCE_SCOPE, and the workflows remove them in always-steps. Native release candidate (workflow_dispatch from main, for an existing release tag and commit): - each architecture builds on its own runner, pushes by digest only and smoke-tests that pushed digest; - the publish job refuses a tag either registry already has (failing closed on any lookup error); - it tags both smoked digests as one index in both registries and checks that both name exactly those digests; - it attests the index's build provenance in both registries. Native image runs the same build and smoke test on pull requests that change these files, without pushing. The takeover runbook gains concrete commands for the image: stop PHP, inspect, back up and prove the restore by equal fingerprints, take over with that fingerprint, start native, and never restart PHP. It also covers volume ownership and SQLite. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/native-image.yml | 69 +++++++ .github/workflows/native-release.yml | 265 +++++++++++++++++++++++++++ Dockerfile.native | 47 +++++ Dockerfile.native.dockerignore | 6 + docs/rust-server-upgrade-takeover.md | 84 ++++++++- scripts/ci/native-image-smoke.sh | 130 +++++++++++++ scripts/ci/native-release-publish.sh | 52 ++++++ 7 files changed, 650 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/native-image.yml create mode 100644 .github/workflows/native-release.yml create mode 100644 Dockerfile.native create mode 100644 Dockerfile.native.dockerignore create mode 100755 scripts/ci/native-image-smoke.sh create mode 100755 scripts/ci/native-release-publish.sh diff --git a/.github/workflows/native-image.yml b/.github/workflows/native-image.yml new file mode 100644 index 000000000..ca8e2bde8 --- /dev/null +++ b/.github/workflows/native-image.yml @@ -0,0 +1,69 @@ +name: Native image + +# Builds the native (Rust) server image from Dockerfile.native and runs the +# same smoke test the native release runs (Server #325). Only changes to the +# image's own files run it: the release workflow is dispatch-only, so this is +# where a pull request finds out that the Dockerfile, its ignore file or the +# smoke and publish scripts no longer work. Nothing is pushed. + +on: + pull_request: + paths: + - Dockerfile.native + - Dockerfile.native.dockerignore + - scripts/ci/native-image-smoke.sh + - scripts/ci/native-release-publish.sh + - .github/workflows/native-image.yml + - .github/workflows/native-release.yml + +permissions: + contents: read + +concurrency: + group: native-image-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + smoke: + name: Build and smoke the native image (linux/amd64) + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + RESOURCE_SCOPE: dw-native-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + steps: + - name: Check out + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + + - name: Build the image into the local daemon + timeout-minutes: 45 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + file: Dockerfile.native + platforms: linux/amd64 + pull: true + provenance: false + load: true + push: false + tags: ${{ env.RESOURCE_SCOPE }}:smoke + + - name: Smoke-test the image + run: bash scripts/ci/native-image-smoke.sh "$RESOURCE_SCOPE:smoke" linux/amd64 native-image/smoke-amd64.json + + - name: Remove this job's smoke containers and network + if: always() + run: | + docker rm -f "$RESOURCE_SCOPE-native-sqlite" "$RESOURCE_SCOPE-native-pgsql" "$RESOURCE_SCOPE-native-mysql" "$RESOURCE_SCOPE-postgres" "$RESOURCE_SCOPE-mysql" || true + docker network rm "$RESOURCE_SCOPE" || true + + - name: Keep the smoke receipt + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: native-image-smoke-amd64 + path: native-image + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml new file mode 100644 index 000000000..767b8b16c --- /dev/null +++ b/.github/workflows/native-release.yml @@ -0,0 +1,265 @@ +name: Native release candidate + +# Publishes the native (Rust) server image for an existing immutable release +# tag (Server #325). Each architecture builds on its own runner and pushes by +# digest only; that exact digest is pulled back and smoke-tested there, so a +# failed smoke leaves no tag. The publish job then refuses tags that already +# exist, gives both smoked digests +# the release tag in both registries as one index, verifies it, and attests +# its build provenance. The PHP image and its Release workflow are unchanged. + +run-name: Native ${{ inputs.tag }} at ${{ inputs.release_commit }} + +on: + workflow_dispatch: + inputs: + tag: + description: 'Existing immutable release tag to publish natively (for example 2.12.0-rc.1)' + required: true + release_commit: + description: 'Full source commit of that tag' + required: true + +permissions: + contents: read + +concurrency: + group: native-release-${{ inputs.tag }} + cancel-in-progress: false + +env: + DOCKERHUB_IMAGE: durableworkflow/server-native + GHCR_IMAGE: ghcr.io/durable-workflow/server-native + +jobs: + build-amd64: + name: Build and smoke linux/amd64 + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 75 + environment: dockerhub + permissions: + contents: read + packages: write + env: + PLATFORM: linux/amd64 + RESOURCE_SCOPE: dw-native-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + outputs: + digest: ${{ steps.build.outputs.digest }} + steps: + - name: Check out trusted release tooling + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + + - name: Check out immutable release source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + path: release-source + ref: refs/tags/${{ inputs.tag }} + + - name: Verify the release tag names the planned commit + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + RELEASE_COMMIT: ${{ inputs.release_commit }} + run: | + set -euo pipefail + scripts/ci/verify-release-tag-source.sh + test "$(git -C release-source rev-parse HEAD)" = "$RELEASE_COMMIT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + + - name: Log in to Docker Hub + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push linux/amd64 by digest only + id: build + timeout-minutes: 60 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: release-source + file: release-source/Dockerfile.native + platforms: linux/amd64 + pull: true + provenance: false + labels: | + org.opencontainers.image.revision=${{ inputs.release_commit }} + dev.durable-workflow.release.tag=${{ inputs.tag }} + dev.durable-workflow.release.run-id=${{ github.run_id }} + outputs: type=image,"name=${{ env.GHCR_IMAGE }},${{ env.DOCKERHUB_IMAGE }}",push-by-digest=true,name-canonical=true,push=true + + - name: Smoke-test the pushed linux/amd64 digest on its own architecture + env: + IMAGE: ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} + run: bash scripts/ci/native-image-smoke.sh "$IMAGE" "$PLATFORM" native-release/smoke-amd64.json + + - name: Remove this job's smoke containers and network + if: always() + run: | + docker rm -f "$RESOURCE_SCOPE-native-sqlite" "$RESOURCE_SCOPE-native-pgsql" "$RESOURCE_SCOPE-native-mysql" "$RESOURCE_SCOPE-postgres" "$RESOURCE_SCOPE-mysql" || true + docker network rm "$RESOURCE_SCOPE" || true + + - name: Keep the linux/amd64 smoke receipt + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: native-smoke-amd64 + path: native-release + retention-days: 30 + if-no-files-found: ignore + + build-arm64: + name: Build and smoke linux/arm64 + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04-arm + timeout-minutes: 75 + environment: dockerhub + permissions: + contents: read + packages: write + env: + PLATFORM: linux/arm64 + RESOURCE_SCOPE: dw-native-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + outputs: + digest: ${{ steps.build.outputs.digest }} + steps: + - name: Check out trusted release tooling + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + + - name: Check out immutable release source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + path: release-source + ref: refs/tags/${{ inputs.tag }} + + - name: Verify the release tag names the planned commit + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + RELEASE_COMMIT: ${{ inputs.release_commit }} + run: | + set -euo pipefail + scripts/ci/verify-release-tag-source.sh + test "$(git -C release-source rev-parse HEAD)" = "$RELEASE_COMMIT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + + - name: Log in to Docker Hub + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push linux/arm64 by digest only + id: build + timeout-minutes: 60 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: release-source + file: release-source/Dockerfile.native + platforms: linux/arm64 + pull: true + provenance: false + labels: | + org.opencontainers.image.revision=${{ inputs.release_commit }} + dev.durable-workflow.release.tag=${{ inputs.tag }} + dev.durable-workflow.release.run-id=${{ github.run_id }} + outputs: type=image,"name=${{ env.GHCR_IMAGE }},${{ env.DOCKERHUB_IMAGE }}",push-by-digest=true,name-canonical=true,push=true + + - name: Smoke-test the pushed linux/arm64 digest on its own architecture + env: + IMAGE: ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} + run: bash scripts/ci/native-image-smoke.sh "$IMAGE" "$PLATFORM" native-release/smoke-arm64.json + + - name: Remove this job's smoke containers and network + if: always() + run: | + docker rm -f "$RESOURCE_SCOPE-native-sqlite" "$RESOURCE_SCOPE-native-pgsql" "$RESOURCE_SCOPE-native-mysql" "$RESOURCE_SCOPE-postgres" "$RESOURCE_SCOPE-mysql" || true + docker network rm "$RESOURCE_SCOPE" || true + + - name: Keep the linux/arm64 smoke receipt + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: native-smoke-arm64 + path: native-release + retention-days: 30 + if-no-files-found: ignore + + publish: + name: Tag, verify and attest the native index + needs: [build-amd64, build-arm64] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: dockerhub + permissions: + contents: read + packages: write + id-token: write + attestations: write + steps: + - name: Check out trusted release tooling + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + + - name: Log in to Docker Hub + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tag and verify the index in both registries + id: index + env: + RELEASE_TAG: ${{ inputs.tag }} + DIGEST_AMD64: ${{ needs.build-amd64.outputs.digest }} + DIGEST_ARM64: ${{ needs.build-arm64.outputs.digest }} + run: bash scripts/ci/native-release-publish.sh + + - name: Attest the index's build provenance in GHCR + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4 + with: + subject-name: ${{ env.GHCR_IMAGE }} + subject-digest: ${{ steps.index.outputs.digest }} + push-to-registry: true + + - name: Attest the index's build provenance in Docker Hub + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4 + with: + subject-name: index.docker.io/${{ env.DOCKERHUB_IMAGE }} + subject-digest: ${{ steps.index.outputs.digest }} + push-to-registry: true diff --git a/Dockerfile.native b/Dockerfile.native new file mode 100644 index 000000000..f42e20d53 --- /dev/null +++ b/Dockerfile.native @@ -0,0 +1,47 @@ +# Durable Workflow native (Rust) server, published as a release candidate +# (Server #325). It is opt-in: the server refuses to start without +# DW_RUST_EXPERIMENTAL=1. The published PHP image remains the default. +# +# The same image runs the takeover commands an operator uses to move a +# stopped, backed-up PHP database to native: upgrade-inspect, +# upgrade-queue-inspect, upgrade-payload-inspect, upgrade-fingerprint, +# upgrade-prepare-sqlite and upgrade-takeover +# (docs/rust-server-upgrade-takeover.md). + +# The pinned compiler the Shared Server fixtures build with (Debian 12, +# glibc 2.36, matching the runtime below). +FROM mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 AS build +ARG CARGO_BUILD_JOBS=2 +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} \ + CARGO_INCREMENTAL=0 \ + CARGO_PROFILE_RELEASE_DEBUG=0 +WORKDIR /source/rust +COPY rust/ /source/rust/ +# --locked: every crate is the one Cargo.lock names, checksum included. +RUN cargo build --release --locked \ + && install -m 0755 target/release/durable-workflow-server /durable-workflow-server + +# A digest-pinned slim runtime: glibc for the binary, nothing else. TLS uses +# rustls with compiled-in web PKI roots, and SQLite is bundled. +FROM mirror.gcr.io/library/debian@sha256:7ff6ba31527b9e336162fa061746c366c4b2032f4d2cff2078bea43f78efe3f8 +COPY --from=build /durable-workflow-server /usr/local/bin/durable-workflow-server +# The PHP image's paths, so a taken-over installation keeps its volumes: +# SQLite in /app/database, and external payload objects under /app/storage, +# whose absolute file:// paths PHP records in the database. Native's default +# local payload root is /storage/app/external-payloads. +RUN install -d -o 1000 -g 1000 -m 0750 /app /app/database /app/storage +# Native runs the repair pass itself: this image has no separate scheduler +# container, as published PHP's Compose deployment does. +ENV DW_BIND_ADDRESS=0.0.0.0:8080 \ + DW_RUST_MAINTENANCE_LOOP=1 \ + DB_DATABASE=/app/database/database.sqlite +USER 1000:1000 +WORKDIR /app +VOLUME ["/app/database", "/app/storage"] +EXPOSE 8080 +# Native stops gracefully on SIGTERM; PHP's image declares SIGWINCH. +STOPSIGNAL SIGTERM +ENTRYPOINT ["/usr/local/bin/durable-workflow-server"] +LABEL org.opencontainers.image.title="Durable Workflow Server (native release candidate)" \ + org.opencontainers.image.source="https://github.com/durable-workflow/server" \ + org.opencontainers.image.licenses="MIT" diff --git a/Dockerfile.native.dockerignore b/Dockerfile.native.dockerignore new file mode 100644 index 000000000..0465ffb33 --- /dev/null +++ b/Dockerfile.native.dockerignore @@ -0,0 +1,6 @@ +# The native build reads only the Rust crate: its sources, migrations, +# recorded upgrade layouts, schema, test fixtures and vendored patches. +* +!rust +rust/target +rust/*.md diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 7439f085a..7141ae1d1 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -38,6 +38,81 @@ There is no PHP rollback after step 4. Recovery restores the verified backup into a new database for a PHP installation, with the data boundary of that backup. +## Running it with the native image + +The native release-candidate image (`durableworkflow/server-native`, also +`ghcr.io/durable-workflow/server-native`) carries every command below. It +reads the same `DB_*` settings as the PHP image. Every command needs +`DW_RUST_EXPERIMENTAL=1`: native is opt-in until it is declared qualified. + +The inspection, the fingerprints and the takeover must all name the same +stopped source. Native refuses a fingerprint that does not match the source +in the takeover's own transaction, but only the operator can make sure the +backup restores. The backup check in step 3 is therefore not optional. +Shown here for PostgreSQL; MySQL is the same with `DB_CONNECTION=mysql`. +`-e DB_PASSWORD` passes the password from the shell's environment, so export +it first rather than writing it on the command line. Add `--network` when the +database is only reachable on a Docker network. + +1. Stop every PHP role that can write: HTTP, queue workers, the scheduler and + any bootstrap. Keep them stopped until step 6. +2. Inspect the stopped source. Every report must say `inspected` and name a + qualified `source_layout`. + + ```sh + source="-e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=pgsql -e DB_HOST=db -e DB_PORT=5432 \ + -e DB_DATABASE=workflow -e DB_USERNAME=workflow -e DB_PASSWORD" + queues="-e DW_RUST_UPGRADE_QUEUE_CONNECTION=database \ + -e DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION=database -e DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE=default" + docker run --rm $source $queues durableworkflow/server-native: upgrade-inspect + docker run --rm $source $queues durableworkflow/server-native: upgrade-queue-inspect + docker run --rm $source $queues durableworkflow/server-native: upgrade-payload-inspect + ``` +3. Back up with the [existing procedure](self-hosted-backup-and-restore.md). + Restore the backup into a separate database (here `workflow_restored`). + Fingerprint both. The two digests must be equal: that proves the backup + restores to exactly the stopped source. + + ```sh + docker run --rm $source durableworkflow/server-native: upgrade-fingerprint + docker run --rm $source -e DB_DATABASE=workflow_restored durableworkflow/server-native: upgrade-fingerprint + ``` +4. Take over the source with that digest. Native refuses unless the source + still has it, so a write after the backup stops the takeover before any + change. + + ```sh + docker run --rm $source $queues -e DW_UPGRADE_BACKUP_FINGERPRINT=sha256: \ + durableworkflow/server-native: upgrade-takeover + ``` +5. Start native on the taken-over database. + + ```sh + docker run -d -p 8080:8080 $source -e DW_AUTH_TOKEN= durableworkflow/server-native: + ``` + + The image runs the repair pass itself (`DW_RUST_MAINTENANCE_LOOP=1`), + because it has no separate scheduler container. +6. Do not restart PHP against the database: its ownership check refuses it. + +The native image uses the PHP image's paths: SQLite at +`/app/database/database.sqlite` and local external payload objects under +`/app/storage/app/external-payloads/`. PHP records each object's +absolute `file://` path in the database, so mount the PHP installation's +volumes at the same paths. The PHP image writes them as `www-data`; native +runs as UID and GID 1000. Once PHP is stopped (step 1), give both volumes to +1000 (for example `chown -R 1000:1000`): ownership is not part of the +fingerprint, and `upgrade-prepare-sqlite`, the takeover and the server all +write. Takeover still refuses a source that holds external payload references. + +For SQLite, stop PHP first. Then switch the file from WAL to the rollback +journal with `upgrade-prepare-sqlite`, and mount it read-only for the +inspections (`-v :/app/database:ro`). The backup check in step 3 is a +byte copy of the prepared file, fingerprinted like the source. + +Rollback is restoring the verified backup into a new database for a PHP +installation. There is no in-place rollback after step 4. + ## Fingerprint The digest covers every relation in the database, not only the ones PHP @@ -162,9 +237,12 @@ with a recorded recipe: use PHP's queue, and the queue inspection has already bound every queued job to its durable task row, so the column is removed and the takeover report counts the references it held. -- Service calls are not part of native yet, so a source with service calls - is refused before any change, and `call_deadline_at` is removed only from - an empty table. +- Each service call's `call_deadline_at` is copied into native's + `dw_service_call_deadlines` before the column is removed. +- Later releases add their own recipes: 2.11.1's trace carriers, and + 2.11.3's session holders, full failure messages, idempotency digest, + dispatch outbox and stream reclaim deadlines. The layout READMEs under + `rust/upgrade/` and the port checkpoints describe each one. A release whose layout has no recipe is refused by step 1. diff --git a/scripts/ci/native-image-smoke.sh b/scripts/ci/native-image-smoke.sh new file mode 100755 index 000000000..7c8cecbd4 --- /dev/null +++ b/scripts/ci/native-image-smoke.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Smoke-test one platform of the native server image (Server #325) and write +# a receipt. Usage: native-image-smoke.sh +# +# Checks: +# - the image config: non-root 1000:1000, SIGTERM, port 8080, entrypoint and +# the PHP image's /app/database and /app/storage paths; +# - that the server refuses to start without the explicit opt-in; +# - a cold start on SQLite, PostgreSQL 17 and MySQL 8.0, with /api/ready and +# an authenticated /api/cluster/info; +# - a graceful SIGTERM stop with exit code 0; +# - that the takeover commands answer, refusing a missing source rather +# than creating one. +set -euo pipefail +# A failure inside $(serve ...) must stop the run too. +shopt -s inherit_errexit +test "$#" = 3 || { echo "usage: $0 " >&2; exit 2; } +image=$1 +platform=$2 +receipt=$3 +mkdir -p "$(dirname "$receipt")" +probe=mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c +postgres=mirror.gcr.io/library/postgres@sha256:f1c3376c26f2609ab9f29f71f824103fe2fcd8ee0346485cb6122a4f93df6f94 +mysql=mirror.gcr.io/library/mysql@sha256:62fb722c78b24245ddff1796a0fcee4a49cc5b87e0aaaf20c92d1da9e0a2497b +# Every named container and the network derive from RESOURCE_SCOPE: the +# workflow sets it per run, attempt and job and removes them again in an +# always step. A local run gets its own scope. +scope_identity="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-native-smoke}" +if test -z "${RESOURCE_SCOPE:-}"; then + RESOURCE_SCOPE="dw-native-$scope_identity-$(date -u +%Y%m%d%H%M%S)-$$" +fi +export RESOURCE_SCOPE +token=native-smoke-token +password=native-smoke-disposable-password +state=$(mktemp -d) +chmod 0777 "$state" +# The workflow's always step removes the containers and the network. +trap 'rm -rf -- "$state"' EXIT +# CI smoke-tests the pushed digest; a local build is already present. +docker image inspect "$image" >/dev/null 2>&1 || docker pull --platform "$platform" "$image" >/dev/null +docker network create "$RESOURCE_SCOPE" >/dev/null + +# The image config. +config=$(docker image inspect --format '{{json .Config}}' "$image") +jq -e '.User=="1000:1000" and .StopSignal=="SIGTERM" and (.ExposedPorts|has("8080/tcp")) + and .Entrypoint==["/usr/local/bin/durable-workflow-server"] + and .WorkingDir=="/app" and (.Volumes|keys)==["/app/database","/app/storage"] + and (.Env|index("DB_DATABASE=/app/database/database.sqlite")!=null) + and (.Env|index("DW_BIND_ADDRESS=0.0.0.0:8080")!=null) + and (.Env|index("DW_RUST_MAINTENANCE_LOOP=1")!=null) + and ([.Env[]|select(startswith("DW_RUST_EXPERIMENTAL="))]|length)==0' <<<"$config" >/dev/null + +# Without the opt-in the server does not start. +status=0 +refusal=$(docker run --rm --platform "$platform" -e DW_AUTH_TOKEN=$token "$image" 2>&1) || status=$? +test "$status" != 0 +grep -q 'requires DW_RUST_EXPERIMENTAL=1' <<<"$refusal" + +# GET http://:8080 from inside the network, with the bearer token +# when a third argument is given; prints the body and fails on any non-2xx. +get() { + docker run --rm --network "$RESOURCE_SCOPE" --entrypoint python "$probe" -I -c ' +import sys, urllib.request +headers = {"Authorization": "Bearer " + sys.argv[2]} if len(sys.argv) > 2 else {} +request = urllib.request.Request(sys.argv[1], headers=headers) +print(urllib.request.urlopen(request, timeout=5).read().decode()) +' "http://$1:8080$2" "${@:3}" +} + +wait_ready() { + local host=$1 + for _ in $(seq 1 60); do + if get "$host" /api/ready >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + docker logs --tail 40 "$host" >&2 || true + return 1 +} + +# Start the server on one backend, read readiness and cluster info, stop it +# with SIGTERM and require a clean exit. Prints the backend's receipt line. +serve() { + local backend=$1; shift + docker run -d --platform "$platform" --name "$RESOURCE_SCOPE-native-$backend" --network "$RESOURCE_SCOPE" \ + -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=$token "$@" "$image" >/dev/null + wait_ready "$RESOURCE_SCOPE-native-$backend" + local info + info=$(get "$RESOURCE_SCOPE-native-$backend" /api/cluster/info "$token") + started=$(date +%s) + docker stop --time 20 "$RESOURCE_SCOPE-native-$backend" >/dev/null + local exit_code + exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$RESOURCE_SCOPE-native-$backend") + test "$exit_code" = 0 + jq -cn --arg backend "$backend" --argjson info "$info" --argjson seconds "$(( $(date +%s) - started ))" \ + '{backend:$backend, ready:true, cluster_info_keys:($info|keys|length), sigterm_exit:0, stop_seconds:$seconds}' +} + +results=() +results+=("$(serve sqlite -v "$state:/app/database")") + +docker run -d --name "$RESOURCE_SCOPE-postgres" --network "$RESOURCE_SCOPE" \ + -e POSTGRES_USER=native -e POSTGRES_PASSWORD=$password -e POSTGRES_DB=native "$postgres" >/dev/null +until docker exec "$RESOURCE_SCOPE-postgres" pg_isready -h 127.0.0.1 -U native -d native >/dev/null 2>&1; do sleep 1; done +results+=("$(serve pgsql -e DB_CONNECTION=pgsql -e DB_HOST="$RESOURCE_SCOPE-postgres" -e DB_PORT=5432 \ + -e DB_DATABASE=native -e DB_USERNAME=native -e DB_PASSWORD=$password)") + +docker run -d --name "$RESOURCE_SCOPE-mysql" --network "$RESOURCE_SCOPE" \ + -e MYSQL_ROOT_PASSWORD=$password -e MYSQL_ROOT_HOST=% -e MYSQL_DATABASE=native "$mysql" >/dev/null +until docker exec "$RESOURCE_SCOPE-mysql" mysql --protocol=TCP -h127.0.0.1 -uroot -p$password -e 'select 1' native >/dev/null 2>&1; do sleep 2; done +results+=("$(serve mysql -e DB_CONNECTION=mysql -e DB_HOST="$RESOURCE_SCOPE-mysql" -e DB_PORT=3306 \ + -e DB_DATABASE=native -e DB_USERNAME=root -e DB_PASSWORD=$password)") + +# The takeover commands are present and refuse a source that does not exist, +# without creating it in the mounted state directory. +status=0 +missing=$(docker run --rm --platform "$platform" -v "$state:/app/database" -e DW_RUST_EXPERIMENTAL=1 \ + -e DB_CONNECTION=sqlite -e DB_DATABASE=/app/database/missing.sqlite "$image" upgrade-inspect 2>&1) || status=$? +test "$status" != 0 +grep -qx 'upgrade_inspection_unavailable' <<<"$missing" +test ! -e "$state/missing.sqlite" +test -e "$state/database.sqlite" + +digest=$(docker image inspect --format '{{.Id}}' "$image") +printf '%s\n' "${results[@]}" | jq -s --arg image "$image" --arg platform "$platform" --arg id "$digest" \ + '{schema:"durable-workflow.native-image-smoke.v1", image:$image, platform:$platform, image_id:$id, + config:{user:"1000:1000", stop_signal:"SIGTERM", port:8080, opt_in_required:true}, + backends:., takeover_commands:"present"}' > "$receipt" +cat "$receipt" diff --git a/scripts/ci/native-release-publish.sh b/scripts/ci/native-release-publish.sh new file mode 100755 index 000000000..e5b6df6bd --- /dev/null +++ b/scripts/ci/native-release-publish.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Give the smoked native platform digests (Server #325) one release tag in +# both registries, as one index, and verify it. +# Env: RELEASE_TAG, GHCR_IMAGE, DOCKERHUB_IMAGE, and DIGEST_AMD64 and +# DIGEST_ARM64 from the build jobs, which push by digest and fail unless that +# exact digest passed its smoke test. Writes digest= to $GITHUB_OUTPUT. +set -euo pipefail +fail() { printf 'Native release publication failed: %s\n' "$1" >&2; exit 1; } +[[ "${RELEASE_TAG:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z][0-9A-Za-z.-]*)?$ ]] || fail "RELEASE_TAG must be an exact release version." +: "${GHCR_IMAGE:?}" "${DOCKERHUB_IMAGE:?}" + +digests=("${DIGEST_AMD64:-}" "${DIGEST_ARM64:-}") +for digest in "${digests[@]}"; do + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "each platform needs an exact smoked digest." +done + +# A release tag is immutable: refuse one that either registry already has. +# Only a registry's own "not found" counts as absent; any other failure to +# look (authentication, network) stops here rather than risk an overwrite. +for image in "$GHCR_IMAGE" "$DOCKERHUB_IMAGE"; do + if lookup=$(docker buildx imagetools inspect "$image:$RELEASE_TAG" 2>&1); then + fail "$image:$RELEASE_TAG already exists." + fi + grep -qiE 'not found|manifest unknown|name unknown' <<<"$lookup" \ + || fail "could not confirm that $image:$RELEASE_TAG is absent: $(head -c 300 <<<"$lookup")" +done + +for image in "$GHCR_IMAGE" "$DOCKERHUB_IMAGE"; do + docker buildx imagetools create --tag "$image:$RELEASE_TAG" \ + "$image@${digests[0]}" "$image@${digests[1]}" +done + +# Both registries hold the same index, naming exactly the smoked digests. +index="" +for image in "$GHCR_IMAGE" "$DOCKERHUB_IMAGE"; do + raw=$(docker buildx imagetools inspect --raw "$image:$RELEASE_TAG") + digest=$(docker buildx imagetools inspect --format '{{.Manifest.Digest}}' "$image:$RELEASE_TAG") + if test -z "$index"; then index=$digest; fi + test "$digest" = "$index" || fail "$image:$RELEASE_TAG is $digest, not $index." + jq -e --arg amd64 "${digests[0]}" --arg arm64 "${digests[1]}" ' + [.manifests[] | {digest, platform: "\(.platform.os)/\(.platform.architecture)"}] | sort_by(.platform) + == [{digest: $amd64, platform: "linux/amd64"}, {digest: $arm64, platform: "linux/arm64"}]' <<<"$raw" >/dev/null \ + || fail "$image:$RELEASE_TAG does not name exactly the smoked platform digests." +done + +printf 'digest=%s\n' "$index" >> "${GITHUB_OUTPUT:-/dev/null}" +{ + printf '### Native %s\n\n' "$RELEASE_TAG" + printf -- '- index `%s`, in `%s` and `%s`\n' "$index" "$GHCR_IMAGE:$RELEASE_TAG" "$DOCKERHUB_IMAGE:$RELEASE_TAG" + printf -- '- linux/amd64 `%s`, linux/arm64 `%s`, each smoke-tested on its own architecture\n' "${digests[0]}" "${digests[1]}" +} >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +printf '%s\n' "$index" From b2d62ea700f8bd1e394244587584f86961189337 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:08:15 +0000 Subject: [PATCH 3/8] Let the matched comparison run the published native image The release-candidate performance gate measures the artifact operators will run, not a binary mounted into the PHP image. A native-image implementation runs the image named by digest (DW_PORT_NATIVE_IMAGE), with its own entrypoint, user and maintenance loop and native-server's CPU and memory limits. The driver refuses a reference without a digest or one other than the profile's artifacts.native_image, and records it in identity.json; the summarizer counts native-image-server as the runtime. The php and native implementations are unchanged; the binary mount no longer requires DW_PORT_NATIVE_BINARY when native does not run. Checked: Compose renders the php, native and native-image profiles. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/v3/README.md | 18 +++++++ scripts/perf/run-server-port-matched.sh | 49 +++++++++++++------ scripts/perf/server-port-matched.compose.yml | 24 ++++++++- scripts/perf/summarize-server-port-matched.py | 6 ++- 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/benchmarks/server-port/v3/README.md b/benchmarks/server-port/v3/README.md index f74ef55e1..ba4eda85f 100644 --- a/benchmarks/server-port/v3/README.md +++ b/benchmarks/server-port/v3/README.md @@ -28,3 +28,21 @@ docker run --rm -v "$PWD/results:/data" -v "$PWD/scripts/perf:/scripts:ro" pytho the driver refuses a binary or probe script that does not match. Run the two implementations sequentially on a host with no other workload, then delete the host. The driver removes each Compose project and its volumes on exit. + +## Against the published native image + +The release-candidate measurement runs the published native image instead of a +binary mounted into the PHP image. Select the `native-image` implementation +and name the image by digest: + +```sh +export DW_PORT_NATIVE_IMAGE=ghcr.io/durable-workflow/server-native@sha256: +bash "$driver" PROFILE results/smoke-native-image native-image --smoke +bash "$driver" PROFILE results/native-image native-image +``` + +It runs as an operator runs it: the image's own entrypoint, user and +maintenance loop (`DW_RUST_MAINTENANCE_LOOP=1`), with the same CPU and memory +limits as `native-server`. PHP's comparison run already includes its scheduler +container. Set `artifacts.native_image` in the profile copy to that reference; +the driver refuses any other image, and `identity.json` records it. diff --git a/scripts/perf/run-server-port-matched.sh b/scripts/perf/run-server-port-matched.sh index 5ccb861e6..10e606d2f 100755 --- a/scripts/perf/run-server-port-matched.sh +++ b/scripts/perf/run-server-port-matched.sh @@ -2,11 +2,13 @@ # Run one implementation of a server-port matched development comparison on a # dedicated host, following the profile's workload, windows and idle checks. # -# run-server-port-matched.sh PROFILE OUTPUT_DIR php|native [--smoke|--idle-only] +# run-server-port-matched.sh PROFILE OUTPUT_DIR php|native|native-image [--smoke|--idle-only] # -# Set DW_PORT_SOURCE_ROOT, DW_PORT_SDK_ROOT and DW_PORT_NATIVE_BINARY as the -# v2 README describes. Run php and native sequentially, each into its own -# output directory, never together. --smoke shortens every window to five +# Set DW_PORT_SOURCE_ROOT and DW_PORT_SDK_ROOT as the v2 README describes, plus +# DW_PORT_NATIVE_BINARY for native (an optimized binary run in the PHP image) +# or DW_PORT_NATIVE_IMAGE for native-image (the published native image, by +# digest). Run the implementations sequentially, each into its own output +# directory, never together. --smoke shortens every window to five # seconds and one repetition to check the setup; it is never a measurement. # --idle-only runs only the idle long-poll windows, without the SDK worker or # offered work, to compare idle cost on its own. @@ -19,10 +21,16 @@ out=$2 impl=$3 smoke=${4:-} case "$smoke" in ""|--smoke|--idle-only) ;; *) echo "Unknown option $smoke." >&2; exit 2 ;; esac -[[ $impl == php || $impl == native ]] || { echo "Choose php or native." >&2; exit 2; } +[[ $impl == php || $impl == native || $impl == native-image ]] || { echo "Choose php, native or native-image." >&2; exit 2; } : "${DW_PORT_SOURCE_ROOT:?Set the absolute source directory}" : "${DW_PORT_SDK_ROOT:?Set the normally installed SDK directory}" -: "${DW_PORT_NATIVE_BINARY:?Set the absolute optimized binary path}" +if [[ $impl == native ]]; then + : "${DW_PORT_NATIVE_BINARY:?Set the absolute optimized binary path}" +fi +if [[ $impl == native-image ]]; then + [[ ${DW_PORT_NATIVE_IMAGE:-} =~ ^[a-z0-9./_-]+(:[A-Za-z0-9._-]+)?@sha256:[0-9a-f]{64}$ ]] \ + || { echo "Set DW_PORT_NATIVE_IMAGE to the native image by digest (name@sha256:...)." >&2; exit 2; } +fi compose="$DW_PORT_SOURCE_ROOT/scripts/perf/server-port-matched.compose.yml" [[ ! -e $out ]] || { echo "Refusing to overwrite $out." >&2; exit 2; } mkdir -p "$out" @@ -37,11 +45,22 @@ for pair in .workload.adapter:.workload.adapter_sha256 .workload.command:.worklo got=$(sha256sum "$DW_PORT_SOURCE_ROOT/$file" | cut -d ' ' -f1) [[ $got == "$want" ]] || { echo "$file does not match the profile." >&2; exit 1; } done -binary_sha=$(sha256sum "$DW_PORT_NATIVE_BINARY" | cut -d ' ' -f1) -want_binary=$(j '.artifacts.native_binary_sha256 // empty') -if [[ -n $want_binary && $binary_sha != "$want_binary" ]]; then - echo "The native binary does not match the profile." >&2 - exit 1 +binary_sha="" native_image="" +if [[ $impl == native ]]; then + binary_sha=$(sha256sum "$DW_PORT_NATIVE_BINARY" | cut -d ' ' -f1) + want_binary=$(j '.artifacts.native_binary_sha256 // empty') + if [[ -n $want_binary && $binary_sha != "$want_binary" ]]; then + echo "The native binary does not match the profile." >&2 + exit 1 + fi +fi +if [[ $impl == native-image ]]; then + native_image=$DW_PORT_NATIVE_IMAGE + want_image=$(j '.artifacts.native_image // empty') + if [[ -n $want_image && $native_image != "$want_image" ]]; then + echo "The native image does not match the profile." >&2 + exit 1 + fi fi if [[ $smoke == --smoke ]]; then @@ -61,7 +80,9 @@ case "$impl" in export DW_PORT_RUNTIME_URL=http://php-server:8080 ;; native) profiles=(--profile native) services=(mysql native-server) export DW_PORT_RUNTIME_URL=http://native-server:8080 ;; - *) echo "Choose php or native." >&2; exit 2 ;; + native-image) profiles=(--profile native-image) services=(mysql native-image-server) + export DW_PORT_RUNTIME_URL=http://native-image-server:8080 ;; + *) echo "Choose php, native or native-image." >&2; exit 2 ;; esac dc() { docker compose -p "$project" -f "$compose" "$@"; } probe() { dc run --rm --no-deps probe "$@"; } @@ -69,7 +90,7 @@ probe() { dc run --rm --no-deps probe "$@"; } cgroup_driver=$(docker info --format '{{.CgroupDriver}}') jq -n --arg impl "$impl" --arg project "$project" --arg profile "$profile" \ --arg profile_sha "$(sha256sum "$profile" | cut -d ' ' -f1)" \ - --arg source "$(git -C "$DW_PORT_SOURCE_ROOT" rev-parse HEAD)" --arg binary "$binary_sha" \ + --arg source "$(git -C "$DW_PORT_SOURCE_ROOT" rev-parse HEAD)" --arg binary "$binary_sha" --arg native_image "$native_image" \ --arg smoke "$smoke" --arg kernel "$(uname -r)" --arg cpus "$(nproc)" \ --arg memory "$(awk '/MemTotal/ {print $2 * 1024}' /proc/meminfo)" \ --arg docker "$(docker version --format '{{.Server.Version}}')" --arg cgroup "$cgroup_driver" \ @@ -77,7 +98,7 @@ jq -n --arg impl "$impl" --arg project "$project" --arg profile "$profile" \ --arg filesystem "$(df -T "$(docker info --format '{{.DockerRootDir}}')" | awk 'NR==2 {print $2}')" \ --arg cpu_model "$(awk -F': ' '/model name/ {print $2; exit}' /proc/cpuinfo)" \ '{implementation:$impl,compose_project:$project,profile:$profile,profile_sha256:$profile_sha,source_revision:$source, - native_binary_sha256:$binary,smoke:($smoke=="--smoke"),idle_only:($smoke=="--idle-only"),machine:{kernel:$kernel,logical_cpus:($cpus|tonumber), + native_binary_sha256:$binary,native_image:$native_image,smoke:($smoke=="--smoke"),idle_only:($smoke=="--idle-only"),machine:{kernel:$kernel,logical_cpus:($cpus|tonumber), physical_memory_bytes:($memory|tonumber),cpu_model:$cpu_model,docker:$docker,cgroup_driver:$cgroup, docker_storage_driver:$storage,docker_root_filesystem:$filesystem}}' > "$out/identity.json" diff --git a/scripts/perf/server-port-matched.compose.yml b/scripts/perf/server-port-matched.compose.yml index 3f619d503..792f31c57 100644 --- a/scripts/perf/server-port-matched.compose.yml +++ b/scripts/perf/server-port-matched.compose.yml @@ -140,7 +140,29 @@ services: DB_USERNAME: workflow DB_PASSWORD: port-mysql volumes: - - "${DW_PORT_NATIVE_BINARY:?Set the absolute optimized binary path}:/native/server:ro" + - "${DW_PORT_NATIVE_BINARY:-/dev/null}:/native/server:ro" + cpus: 2 + mem_limit: 2048m + memswap_limit: 2048m + restart: "no" + + # The published native image, by digest, as an operator runs it: its own + # entrypoint, user and maintenance loop. Select it with the native-image + # implementation; it never runs beside native-server. + native-image-server: + profiles: [native-image] + stop_signal: SIGTERM + image: "${DW_PORT_NATIVE_IMAGE:-unset-native-image}" + platform: linux/amd64 + environment: + DW_RUST_EXPERIMENTAL: "1" + DW_AUTH_DRIVER: token + DW_AUTH_TOKEN: port-test-token + DB_CONNECTION: mysql + DB_HOST: mysql + DB_DATABASE: durable_workflow + DB_USERNAME: workflow + DB_PASSWORD: port-mysql cpus: 2 mem_limit: 2048m memswap_limit: 2048m diff --git a/scripts/perf/summarize-server-port-matched.py b/scripts/perf/summarize-server-port-matched.py index 07e566100..8414df556 100644 --- a/scripts/perf/summarize-server-port-matched.py +++ b/scripts/perf/summarize-server-port-matched.py @@ -14,7 +14,11 @@ from collections import defaultdict from pathlib import Path -RUNTIME = {"php": ["php-server", "php-worker", "php-scheduler", "redis"], "native": ["native-server"]} +RUNTIME = { + "php": ["php-server", "php-worker", "php-scheduler", "redis"], + "native": ["native-server"], + "native-image": ["native-image-server"], +} def load_samples(path): From 7e28e5952fec46a2ced6b48b49992fd1c9cdde8a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:31:10 +0000 Subject: [PATCH 4/8] Pass the runbook's auth token by name Security's review of the native image draft (server#325 comment 6431969727): the start command wrote DW_AUTH_TOKEN= on the command line. It now passes the exported value by name, as DB_PASSWORD already was, so no credential enters arguments or shell history. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 7141ae1d1..d79788176 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -50,8 +50,9 @@ stopped source. Native refuses a fingerprint that does not match the source in the takeover's own transaction, but only the operator can make sure the backup restores. The backup check in step 3 is therefore not optional. Shown here for PostgreSQL; MySQL is the same with `DB_CONNECTION=mysql`. -`-e DB_PASSWORD` passes the password from the shell's environment, so export -it first rather than writing it on the command line. Add `--network` when the +`-e DB_PASSWORD` and `-e DW_AUTH_TOKEN` pass those values from the shell's +environment, so export them first rather than writing a credential on the +command line or into shell history. Add `--network` when the database is only reachable on a Docker network. 1. Stop every PHP role that can write: HTTP, queue workers, the scheduler and @@ -88,7 +89,7 @@ database is only reachable on a Docker network. 5. Start native on the taken-over database. ```sh - docker run -d -p 8080:8080 $source -e DW_AUTH_TOKEN= durableworkflow/server-native: + docker run -d -p 8080:8080 $source -e DW_AUTH_TOKEN durableworkflow/server-native: ``` The image runs the repair pass itself (`DW_RUST_MAINTENANCE_LOOP=1`), From d638cc156fcaf9b405b704f9a20a0eb980fdc6f6 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:44:56 +0000 Subject: [PATCH 5/8] Require the profile to name the native image Security's advisory on b2d62ea70: native-image mode compared the image only when the profile named one, so a profile without artifacts.native_image accepted any digest-pinned image. The driver now refuses such a profile, so every native-image result binds to the profile's exact image. Checked: a profile without the field and one naming another image are refused before anything starts; the matching profile proceeds. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/v3/README.md | 5 +++-- scripts/perf/run-server-port-matched.sh | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/benchmarks/server-port/v3/README.md b/benchmarks/server-port/v3/README.md index ba4eda85f..8ec806c27 100644 --- a/benchmarks/server-port/v3/README.md +++ b/benchmarks/server-port/v3/README.md @@ -44,5 +44,6 @@ bash "$driver" PROFILE results/native-image native-image It runs as an operator runs it: the image's own entrypoint, user and maintenance loop (`DW_RUST_MAINTENANCE_LOOP=1`), with the same CPU and memory limits as `native-server`. PHP's comparison run already includes its scheduler -container. Set `artifacts.native_image` in the profile copy to that reference; -the driver refuses any other image, and `identity.json` records it. +container. Set `artifacts.native_image` in the profile copy to that reference. +The driver refuses a profile without it and any other image, and +`identity.json` records it. diff --git a/scripts/perf/run-server-port-matched.sh b/scripts/perf/run-server-port-matched.sh index 10e606d2f..b2034f41c 100755 --- a/scripts/perf/run-server-port-matched.sh +++ b/scripts/perf/run-server-port-matched.sh @@ -55,9 +55,14 @@ if [[ $impl == native ]]; then fi fi if [[ $impl == native-image ]]; then + # The profile must name the image, so results always bind to it. native_image=$DW_PORT_NATIVE_IMAGE want_image=$(j '.artifacts.native_image // empty') - if [[ -n $want_image && $native_image != "$want_image" ]]; then + if [[ -z $want_image ]]; then + echo "The profile must name artifacts.native_image for native-image runs." >&2 + exit 1 + fi + if [[ $native_image != "$want_image" ]]; then echo "The native image does not match the profile." >&2 exit 1 fi From 050beac16625f1eecb21ebb9f3084f424cfd8923 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:31:21 +0000 Subject: [PATCH 6/8] Restore native-server's readiness and share its settings with the image SonarCloud flagged two new security findings on this PR, and reading the first exposed a real defect: - b2d62ea70 inserted native-image-server in the middle of native-server's block, so native-server lost its depends_on and curl healthcheck to the new service, which has no curl. native-server is back to main's block. native-image-server reuses its environment through a YAML anchor (no second copy of the disposable benchmark password, yaml:S2068), waits for MySQL, and checks readiness with a plain GET over bash's /dev/tcp. - The smoke probe built an http:// URL string (shell:S5332). It now uses http.client with the host and port: the server answers plain HTTP on the job's private network. The probe's positional parameters are named locals. Checked: Compose renders both services with their own depends_on and healthcheck; the /dev/tcp check answers HTTP/1.0 200 OK against the image; the local smoke test passes on SQLite, PostgreSQL and MySQL. Co-Authored-By: Claude Opus 5.5 --- scripts/ci/native-image-smoke.sh | 22 +++++++++++----- scripts/perf/server-port-matched.compose.yml | 27 ++++++++++---------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/scripts/ci/native-image-smoke.sh b/scripts/ci/native-image-smoke.sh index 7c8cecbd4..ee4e03e09 100755 --- a/scripts/ci/native-image-smoke.sh +++ b/scripts/ci/native-image-smoke.sh @@ -56,15 +56,23 @@ refusal=$(docker run --rm --platform "$platform" -e DW_AUTH_TOKEN=$token "$image test "$status" != 0 grep -q 'requires DW_RUST_EXPERIMENTAL=1' <<<"$refusal" -# GET http://:8080 from inside the network, with the bearer token -# when a third argument is given; prints the body and fails on any non-2xx. +# GET from port 8080 inside the job's network, with the bearer +# token when a third argument is given; prints the body and fails on any +# non-2xx. The server answers plain HTTP on that private network. get() { + local host=$1 path=$2 + shift 2 docker run --rm --network "$RESOURCE_SCOPE" --entrypoint python "$probe" -I -c ' -import sys, urllib.request -headers = {"Authorization": "Bearer " + sys.argv[2]} if len(sys.argv) > 2 else {} -request = urllib.request.Request(sys.argv[1], headers=headers) -print(urllib.request.urlopen(request, timeout=5).read().decode()) -' "http://$1:8080$2" "${@:3}" +import sys, http.client +connection = http.client.HTTPConnection(sys.argv[1], 8080, timeout=5) +headers = {"Authorization": "Bearer " + sys.argv[3]} if len(sys.argv) > 3 else {} +connection.request("GET", sys.argv[2], headers=headers) +response = connection.getresponse() +body = response.read().decode() +if not 200 <= response.status < 300: + sys.exit("HTTP %d: %s" % (response.status, body[:200])) +print(body) +' "$host" "$path" "$@" } wait_ready() { diff --git a/scripts/perf/server-port-matched.compose.yml b/scripts/perf/server-port-matched.compose.yml index 792f31c57..8c4cd2626 100644 --- a/scripts/perf/server-port-matched.compose.yml +++ b/scripts/perf/server-port-matched.compose.yml @@ -129,7 +129,7 @@ services: platform: linux/amd64 user: "1000:1000" entrypoint: [/native/server] - environment: + environment: &native_environment DW_RUST_EXPERIMENTAL: "1" DW_BIND_ADDRESS: 0.0.0.0:8080 DW_AUTH_DRIVER: token @@ -145,24 +145,25 @@ services: mem_limit: 2048m memswap_limit: 2048m restart: "no" + depends_on: + mysql: {condition: service_healthy} + healthcheck: + test: [CMD, curl, --fail, --silent, http://127.0.0.1:8080/api/ready] + interval: 2s + timeout: 3s + retries: 20 # The published native image, by digest, as an operator runs it: its own - # entrypoint, user and maintenance loop. Select it with the native-image - # implementation; it never runs beside native-server. + # entrypoint, user and maintenance loop, with native-server's settings and + # limits. Select it with the native-image implementation; it never runs + # beside native-server. The image has bash but no curl, so readiness is a + # plain GET over bash's /dev/tcp. native-image-server: profiles: [native-image] stop_signal: SIGTERM image: "${DW_PORT_NATIVE_IMAGE:-unset-native-image}" platform: linux/amd64 - environment: - DW_RUST_EXPERIMENTAL: "1" - DW_AUTH_DRIVER: token - DW_AUTH_TOKEN: port-test-token - DB_CONNECTION: mysql - DB_HOST: mysql - DB_DATABASE: durable_workflow - DB_USERNAME: workflow - DB_PASSWORD: port-mysql + environment: *native_environment cpus: 2 mem_limit: 2048m memswap_limit: 2048m @@ -170,7 +171,7 @@ services: depends_on: mysql: {condition: service_healthy} healthcheck: - test: [CMD, curl, --fail, --silent, http://127.0.0.1:8080/api/ready] + test: [CMD, bash, -c, "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET /api/ready HTTP/1.0\r\n\r\n' >&3 && head -n1 <&3 | grep -q ' 200 '"] interval: 2s timeout: 3s retries: 20 From be4fcbc11e79eb9a1f7d4ec4c20daefd3d54454d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:11:07 +0000 Subject: [PATCH 7/8] Build and smoke the native image natively on arm64 too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security found that the pinned Rust builder (ba81bc3e…) is a single linux/amd64 manifest, so the release's ubuntu-24.04-arm job could not build natively (server#803 comment 6433955707). The smoke test's MySQL pin (62fb722c…) is amd64-only as well. - Dockerfile.native builds from Security's verified Rust 1.99.0/bookworm multi-architecture index 114c7a44… (amd64 b36c2467…, arm64 e393e07b…, comment 6434047541). - The smoke test runs MySQL 8.0 from its index 7dcddc01…, whose amd64 entry is the 62fb722c… image already pinned and whose arm64 entry is 213bbfaf…. The Python probe, PostgreSQL and the Debian runtime were already multi-architecture indexes. - Native image runs on pull requests as a matrix: linux/amd64 on ubuntu-latest and linux/arm64 on ubuntu-24.04-arm, each building and smoke-testing natively, with a receipt per architecture. Checked: a local linux/amd64 build from the new builder passes the smoke test on SQLite, PostgreSQL and MySQL; both organization policies pass. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/native-image.yml | 23 ++++++++++++++++++----- Dockerfile.native | 8 +++++--- scripts/ci/native-image-smoke.sh | 4 +++- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.github/workflows/native-image.yml b/.github/workflows/native-image.yml index ca8e2bde8..ecb295a0b 100644 --- a/.github/workflows/native-image.yml +++ b/.github/workflows/native-image.yml @@ -25,9 +25,19 @@ concurrency: jobs: smoke: - name: Build and smoke the native image (linux/amd64) - runs-on: ubuntu-latest + name: Build and smoke the native image (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-latest + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-24.04-arm env: RESOURCE_SCOPE: dw-native-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} steps: @@ -43,7 +53,7 @@ jobs: with: context: . file: Dockerfile.native - platforms: linux/amd64 + platforms: ${{ matrix.platform }} pull: true provenance: false load: true @@ -51,7 +61,10 @@ jobs: tags: ${{ env.RESOURCE_SCOPE }}:smoke - name: Smoke-test the image - run: bash scripts/ci/native-image-smoke.sh "$RESOURCE_SCOPE:smoke" linux/amd64 native-image/smoke-amd64.json + env: + PLATFORM: ${{ matrix.platform }} + ARCH: ${{ matrix.arch }} + run: bash scripts/ci/native-image-smoke.sh "$RESOURCE_SCOPE:smoke" "$PLATFORM" "native-image/smoke-$ARCH.json" - name: Remove this job's smoke containers and network if: always() @@ -63,7 +76,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: native-image-smoke-amd64 + name: native-image-smoke-${{ matrix.arch }} path: native-image retention-days: 7 if-no-files-found: ignore diff --git a/Dockerfile.native b/Dockerfile.native index f42e20d53..9203f6528 100644 --- a/Dockerfile.native +++ b/Dockerfile.native @@ -8,9 +8,11 @@ # upgrade-prepare-sqlite and upgrade-takeover # (docs/rust-server-upgrade-takeover.md). -# The pinned compiler the Shared Server fixtures build with (Debian 12, -# glibc 2.36, matching the runtime below). -FROM mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 AS build +# Rust 1.99.0 on Debian 12 (glibc 2.36, matching the runtime below), pinned +# as its multi-architecture index so each architecture builds natively: +# linux/amd64 b36c246742b4…, linux/arm64 e393e07b3c3c… (Security verified +# the index, server#803 comment 6434047541). +FROM mirror.gcr.io/library/rust@sha256:114c7a4425406451c2866b6aafe69fe29b1b298832db1277d411ac73c82d04d6 AS build ARG CARGO_BUILD_JOBS=2 ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} \ CARGO_INCREMENTAL=0 \ diff --git a/scripts/ci/native-image-smoke.sh b/scripts/ci/native-image-smoke.sh index ee4e03e09..b9c29b64d 100755 --- a/scripts/ci/native-image-smoke.sh +++ b/scripts/ci/native-image-smoke.sh @@ -21,7 +21,9 @@ receipt=$3 mkdir -p "$(dirname "$receipt")" probe=mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c postgres=mirror.gcr.io/library/postgres@sha256:f1c3376c26f2609ab9f29f71f824103fe2fcd8ee0346485cb6122a4f93df6f94 -mysql=mirror.gcr.io/library/mysql@sha256:62fb722c78b24245ddff1796a0fcee4a49cc5b87e0aaaf20c92d1da9e0a2497b +# MySQL 8.0 as its multi-architecture index: linux/amd64 is the 62fb722c… +# image the Shared Server fixtures pin, linux/arm64 is 213bbfaf…. +mysql=mirror.gcr.io/library/mysql@sha256:7dcddc01f13bab2f15cde676d44d01f61fc9f99fe7785e86196dfc07d358ae2b # Every named container and the network derive from RESOURCE_SCOPE: the # workflow sets it per run, attempt and job and removes them again in an # always step. A local run gets its own scope. From 56eb1d34bddff96688583598b58e9e5f48b1e363 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:17:09 +0000 Subject: [PATCH 8/8] Bind the takeover runbook to one exact image digest Security's re-check of 050beac16 (server#803 comment 6433835833): every inspection, fingerprint, the irreversible takeover and the final start named durableworkflow/server-native:. A tag can move between steps, so the operator could inspect with one executable and take over or start with another. The runbook now exports one reference, the multi-architecture index digest the release receipt records (IMAGE=durableworkflow/server-native@sha256:...), pulls it, keeps it with the backup and uses "$IMAGE" in every command through startup. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 818eb64dc..5d89ab2d9 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -55,6 +55,16 @@ environment, so export them first rather than writing a credential on the command line or into shell history. Add `--network` when the database is only reachable on a Docker network. +Every command uses one exact image, the multi-architecture index digest that +the release receipt records, never a tag. A tag can move between steps; a +digest cannot, so the image that inspected the source is the one that takes +it over and serves it. Keep the reference with the backup. + +```sh +export IMAGE=durableworkflow/server-native@sha256: +docker pull "$IMAGE" +``` + 1. Stop every PHP role that can write: HTTP, queue workers, the scheduler and any bootstrap. Keep them stopped until step 6. 2. Inspect the stopped source. Every report must say `inspected` and name a @@ -65,9 +75,9 @@ database is only reachable on a Docker network. -e DB_DATABASE=workflow -e DB_USERNAME=workflow -e DB_PASSWORD" queues="-e DW_RUST_UPGRADE_QUEUE_CONNECTION=database \ -e DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION=database -e DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE=default" - docker run --rm $source $queues durableworkflow/server-native: upgrade-inspect - docker run --rm $source $queues durableworkflow/server-native: upgrade-queue-inspect - docker run --rm $source $queues durableworkflow/server-native: upgrade-payload-inspect + docker run --rm $source $queues "$IMAGE" upgrade-inspect + docker run --rm $source $queues "$IMAGE" upgrade-queue-inspect + docker run --rm $source $queues "$IMAGE" upgrade-payload-inspect ``` 3. Back up with the [existing procedure](self-hosted-backup-and-restore.md). Restore the backup into a separate database (here `workflow_restored`). @@ -75,8 +85,8 @@ database is only reachable on a Docker network. restores to exactly the stopped source. ```sh - docker run --rm $source durableworkflow/server-native: upgrade-fingerprint - docker run --rm $source -e DB_DATABASE=workflow_restored durableworkflow/server-native: upgrade-fingerprint + docker run --rm $source "$IMAGE" upgrade-fingerprint + docker run --rm $source -e DB_DATABASE=workflow_restored "$IMAGE" upgrade-fingerprint ``` 4. Take over the source with that digest. Native refuses unless the source still has it, so a write after the backup stops the takeover before any @@ -84,12 +94,12 @@ database is only reachable on a Docker network. ```sh docker run --rm $source $queues -e DW_UPGRADE_BACKUP_FINGERPRINT=sha256: \ - durableworkflow/server-native: upgrade-takeover + "$IMAGE" upgrade-takeover ``` 5. Start native on the taken-over database. ```sh - docker run -d -p 8080:8080 $source -e DW_AUTH_TOKEN durableworkflow/server-native: + docker run -d -p 8080:8080 $source -e DW_AUTH_TOKEN "$IMAGE" ``` The image runs the repair pass itself (`DW_RUST_MAINTENANCE_LOOP=1`),