Rust #325: maintenance loop, takeover of credentials, update validation and 2.11.3, CI adapter cache

.github/workflows/server-parity.yml

@@ -867,6 +867,22 @@ jobs:
           DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.1
           DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102
         run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql
+      - name: Back up, take over and resume a published PHP 2.11.3 upgrade source
+        if: ${{ matrix.part == 'cohorts' }}
+        timeout-minutes: 8
+        env:
+          DW_PARITY_UPGRADE_LABEL: php2113
+          DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.3
+          DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:923a8924ecad13485562d7af30dc87d7fd6d9d7d4758ec8113bac0766fa8603e
+        run: bash scripts/conformance/server-parity/upgrade-source-profile.sh mysql
+      - name: Retire PHP 2.11.3 upgrade source cohort
+        if: ${{ always() && matrix.part == 'cohorts' }}
+        timeout-minutes: 3
+        env:
+          DW_PARITY_UPGRADE_LABEL: php2113
+          DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.3
+          DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:923a8924ecad13485562d7af30dc87d7fd6d9d7d4758ec8113bac0766fa8603e
+        run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql
       - name: Start the native node the corrected marker cohort uses
         id: prepare_corrected
         if: ${{ matrix.part == 'corrected-markers' }}
@@ -1517,6 +1533,22 @@ jobs:
           DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.1
           DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102
         run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql
+      - name: Back up, take over and resume a published PHP 2.11.3 upgrade source
+        if: ${{ matrix.part == 'cohorts' }}
+        timeout-minutes: 8
+        env:
+          DW_PARITY_UPGRADE_LABEL: php2113
+          DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.3
+          DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:923a8924ecad13485562d7af30dc87d7fd6d9d7d4758ec8113bac0766fa8603e
+        run: bash scripts/conformance/server-parity/upgrade-source-profile.sh pgsql
+      - name: Retire PHP 2.11.3 upgrade source cohort
+        if: ${{ always() && matrix.part == 'cohorts' }}
+        timeout-minutes: 3
+        env:
+          DW_PARITY_UPGRADE_LABEL: php2113
+          DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.3
+          DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:923a8924ecad13485562d7af30dc87d7fd6d9d7d4758ec8113bac0766fa8603e
+        run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql
       - name: Start the native node the corrected marker cohort uses
         id: prepare_corrected
         if: ${{ matrix.part == 'corrected-markers' }}
@@ -2182,6 +2214,24 @@ jobs:
           DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.1
           DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102
         run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite
+      - name: Back up, take over and resume a published PHP 2.11.3 upgrade source
+        if: ${{ matrix.part == 'cohorts' }}
+        timeout-minutes: 8
+        env:
+          RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
+          DW_PARITY_UPGRADE_LABEL: php2113
+          DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.3
+          DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:923a8924ecad13485562d7af30dc87d7fd6d9d7d4758ec8113bac0766fa8603e
+        run: bash scripts/conformance/server-parity/upgrade-source-profile.sh sqlite
+      - name: Retire PHP 2.11.3 upgrade source cohort
+        if: ${{ always() && matrix.part == 'cohorts' }}
+        timeout-minutes: 3
+        env:
+          RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
+          DW_PARITY_UPGRADE_LABEL: php2113
+          DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.11.3
+          DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:923a8924ecad13485562d7af30dc87d7fd6d9d7d4758ec8113bac0766fa8603e
+        run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite
       - name: Start the native node the corrected marker cohort uses
         id: prepare_corrected
         if: ${{ matrix.part == 'corrected-markers' }}

docs/rust-server-port-checkpoints/2026-10-26-maintenance-repair.md

@@ -0,0 +1,43 @@
+# October 26, the repair pass runs on its own
+
+- **Found by.** A review of what native does without a request. The published
+  image's `scheduler` service (`docker-compose.yml`) loops every 10 seconds
+  through:
+  - `schedule:evaluate`;
+  - `workflow:v2:repair-pass`, unless `DW_V2_MATCHING_ROLE_QUEUE_WAKE` is false
+    or 0;
+  - activity timeout enforcement;
+  - external payload cleanup;
+  - history pruning.
+
+  The parity reference runs the same loop every second. Native's ticker
+  already fired due timers and schedules every 250 ms, and enforced activity
+  and cooperative deadlines. The repair pass ran only when someone called
+  `POST /api/system/repair/pass`. So an idle run past its execution or run
+  deadline was never given the task that closes it, and overdue ready tasks
+  and due service calls waited for a request.
+- **Change.** The ticker can now also run the repair pass with PHP's loop
+  throttle: at most once per 5 seconds, shared with requested passes that
+  respect it.
+  - **Opt-in.** It runs only with `DW_RUST_MAINTENANCE_LOOP=1`, which the
+    native image will set. A PHP server container never repairs on its own:
+    PHP's separate `scheduler` container does. So the bare binary keeps the
+    server role, and the Shared Server fixtures, which pair each native node
+    with a PHP server container, compare like with like.
+  - On PR #797's first run, a default-on loop repaired a task the corrected
+    marker cohort expects untouched, and raced `repair_status`'s staged
+    candidates.
+  - `DW_V2_MATCHING_ROLE_QUEUE_WAKE=false` (or `0`) still turns it off, as it
+    does PHP's.
+  - `Runtime::set_maintenance_repair` sets it for one runtime.
+- **Tests.**
+  - `the_maintenance_loop_gives_an_overdue_idle_run_its_deadline_task` waits
+    for the loop alone to give an idle run past its deadline its workflow task.
+  - The execution suite passes with the loop off by default, and passed with
+    it on (558).
+  - Native passes, with the loop on, the shared fixtures most exposed to
+    background repair: repair-pass, the lease, cancellation and deadline
+    fixtures, worker sessions and service calls.
+- **Still to do.** History pruning and external payload cleanup also run in
+  PHP's loop; native runs them only on request (`POST
+  /api/system/retention/pass` and its payload cleanup).

docs/rust-server-port-checkpoints/2026-10-26-php-2111-takeover.md

@@ -82,5 +82,6 @@
   Before, it reported the three service-call counters as zeros.
   `the_repair_pass_times_out_calls_past_their_deadline` covers it.
 - **Still refused.** Takeover still refuses a source holding rows native
-  cannot carry over yet: runtime credentials, message streams, update
-  validation tasks and external payloads.
+  cannot carry over yet: message streams, update validation tasks and
+  external payloads. Runtime credentials carried over later the same day
+  ([2026-10-26-takeover-runtime-credentials.md](2026-10-26-takeover-runtime-credentials.md)).

docs/rust-server-port-checkpoints/2026-10-26-php-2113-takeover.md

@@ -0,0 +1,60 @@
+# October 26, takeover of published Server 2.11.3
+
+- **Before.** Native refused every published 2.11.3 database as
+  `upgrade_source_catalog_unqualified`. Server 2.11.3 (Core 2.14.2) was
+  published today, image index
+  `sha256:923a8924ecad13485562d7af30dc87d7fd6d9d7d4758ec8113bac0766fa8603e`.
+- **Layouts name their parent.** `SOURCE_LAYOUTS` was a chain: each delta
+  applied to the layout before it. 2.11.3 has to apply to both 2.11.1
+  variants, so each layout now names the layout its delta applies to.
+  `layout_lineage` gives the path from the frozen layout, and
+  `every_recorded_layout_descends_from_an_earlier_one` checks the order.
+  - `php-2.11.3` descends from fresh `php-2.11.1`.
+  - `php-2.11.3-from-2.9.7` descends from `php-2.11.1-from-2.9.7`.
+- **Captured from real databases.** The published images bootstrapped and
+  migrated SQLite, PostgreSQL 17, MySQL 8.0 and MariaDB 10.11 databases along
+  five paths:
+  - fresh 2.11.3;
+  - 2.11.1 then 2.11.3;
+  - 2.8.0 then 2.11.3;
+  - 2.9.7 then 2.11.3;
+  - 2.9.7, 2.11.1, then 2.11.3.
+
+  Each path gave the same delta from its parent on every backend. The first
+  three end at `php-2.11.3` and the last two at `php-2.11.3-from-2.9.7`. The
+  native binary then inspected, fingerprinted and took over all five real
+  SQLite databases.
+- **Recipe** (details in [the layout README](../../rust/upgrade/php-2.11.3/README.md)).
+  - Each active session's recorded holder incarnation is copied to
+    `dw_worker_session_holders`. Earlier releases record none, so takeover
+    derives the holder from the current registration. A 2.11.3 session that
+    PHP already orphaned stays orphaned.
+  - `message_full` is counted (`full_failure_messages_removed`) and dropped.
+  - The idempotency digest and its index are dropped.
+  - Completed dispatches are counted (`service_call_dispatches_removed`) and
+    the outbox is dropped. A scheduled or claimed dispatch refuses takeover
+    before any change (`upgrade_source_service_call_dispatches_pending`), and
+    inspection reports `service_call_dispatch_inspection`.
+  - `reclaim_at` and its index are dropped.
+  - MySQL and MariaDB run this as journal step 10. Each part checks what is
+    left to do, so a resumed step repeats safely.
+- **Tests.**
+  - SQLite: `takeover_converts_php_2113_databases` covers both layouts, with
+    a held session, a replaced holder, a full failure message, a completed
+    dispatch and an idempotency digest.
+    `takeover_refuses_a_php_2113_dispatch_still_pending` checks that a
+    pending dispatch refuses and leaves the file byte-identical.
+  - PostgreSQL, MySQL and MariaDB:
+    `qualification_takeover_converts_php_2113_and_native_opens_it`.
+  - MySQL and MariaDB: the step-kill test also kills a 2.11.3 takeover
+    after step 9 and resumes it.
+  - CI: a `php2113` upgrade cohort backs up, takes over and resumes a
+    published 2.11.3 source on SQLite, PostgreSQL and MySQL.
+- **Not yet native.** The parity reference is still 2.11.1. Core 2.14 also
+  changed behavior that native does not follow yet:
+  - full failure-message storage on write;
+  - service-call scheduling through the outbox;
+  - the idempotency scope of target namespace, names and key, with
+    `service_call_idempotency_scope_conflict`.
+
+  These belong to the parity move to 2.11.3.

docs/rust-server-port-checkpoints/2026-10-26-takeover-runtime-credentials.md

@@ -0,0 +1,28 @@
+# October 26, takeover keeps runtime credentials
+
+- **Before.** Takeover refused any PHP database holding a
+  `runtime_credentials` row (`upgrade_source_runtime_credentials_unsupported`).
+  An operator who had issued worker tokens through
+  `/api/runtime-credentials` could not cut over without
+  revoking them all and reissuing tokens to every worker.
+- **Why it is safe to keep them.** Native serves runtime credentials from the
+  same `runtime_credentials` table, with the same columns. PHP stores
+  `hash('sha256', $token)` as lowercase hex; native looks a token up by the same
+  digest. Expiry, revocation, roles, tenant and claims are read from the same
+  columns. So the rows need no conversion, and a token PHP issued keeps
+  working after takeover.
+- **Change.** The refusal is removed. Takeover leaves the table as it is.
+- **Tests.** `PHP_CREDENTIAL_ROWS` stores one live and one revoked credential
+  the way PHP does, with digests computed outside native. After takeover the
+  live PHP token authenticates with its subject, roles, tenant and claims,
+  and the revoked one is refused:
+  - SQLite: `takeover_keeps_php_runtime_credentials_that_native_then_accepts`;
+  - PostgreSQL, MySQL and MariaDB: the 2.11.1 takeover qualification, for
+    both the fresh and the upgraded-from-2.9.7 layouts.
+- **Update validation tasks** carry over too (October 27). Native validates
+  updates in the same `workflow_update_validation_tasks` rows and keeps no
+  state of its own, so takeover leaves them as they are. A pending task whose
+  caller was waiting on PHP is then what a PHP restart leaves: a validator may
+  still claim it, and a caller's retry with the same request reads its
+  recorded outcome. `takeover_keeps_php_update_validation_tasks` covers it.
+- **Still refused.** Message streams and external payloads.

docs/rust-server-port.md

@@ -489,6 +489,16 @@ contract fixture before its correction.
   drops the three columns; the verified backup keeps them
   ([#325 decision](https://github.com/durable-workflow/server/issues/325#issuecomment-6306545520),
   [checkpoint](rust-server-port-checkpoints/2026-10-26-php-2111-takeover.md)).
+- **Server 2.11.3 (Core 2.14.2) takeover.** Takeover recognizes both
+  published 2.11.3 layouts, fresh and created by 2.9.x. It copies each active
+  session's recorded holder, drops full failure messages (counted; native
+  reads the labelled preview, as PHP's older readers do), the service-call
+  idempotency digest, completed dispatches and the stream reclaim deadline,
+  and refuses while a dispatch is pending
+  ([checkpoint](rust-server-port-checkpoints/2026-10-26-php-2113-takeover.md)).
+  The parity reference is still 2.11.1. Core 2.14's new behavior (full
+  failure-message storage, service-call scheduling and idempotency scope)
+  is native work for the move to 2.11.3.
 - **Stop signal in the published image.** The published Server image declares
   `STOPSIGNAL SIGWINCH`, Apache's graceful stop. Native stops gracefully on
   `SIGTERM` or `SIGINT` and ignores `SIGWINCH`, so `docker stop` on the

docs/rust-server-upgrade-takeover.md

@@ -77,6 +77,16 @@ inspection is `unqualified` and takeover refuses with
 `upgrade_source_external_payload_storage_unsupported` before any change. A
 disabled policy does not block takeover.
 
+PHP 2.11.3 (Core 2.14) schedules a service call's handler in the
+`workflow_service_call_dispatches` outbox and dispatches it in a later pass.
+Native runs the handler when it accepts the call, so it cannot resume a
+dispatch that is still scheduled or claimed. Every inspection reports
+`service_call_dispatch_inspection`: the number of dispatches and of pending
+ones, without naming calls. A pending dispatch makes the inspection
+`unqualified`, and takeover refuses with
+`upgrade_source_service_call_dispatches_pending` before any change. Let PHP
+finish them before the stop. Completed dispatches do not block takeover.
+
 ## Exclusive transaction per backend
 
 The marker, the fingerprint check and the schema changes must not interleave

rust/src/runtime/execution.rs

@@ -52,6 +52,19 @@ struct TimerScheduler {
     stop: watch::Sender<bool>,
     task: Mutex<Option<JoinHandle<()>>>,
     healthy: Arc<AtomicBool>,
+    /// Whether the ticker also runs the repair pass (`maintenance_repair_enabled`).
+    repair: Arc<AtomicBool>,
+}
+
+/// Whether the ticker runs the repair pass, as PHP's separate `scheduler`
+/// container does. A PHP server container never repairs on its own, so the
+/// native binary takes that role only when asked: `DW_RUST_MAINTENANCE_LOOP=1`
+/// (the native image sets it, having no separate scheduler).
+/// `DW_V2_MATCHING_ROLE_QUEUE_WAKE=false` or `0` still turns it off, as it
+/// turns off PHP's.
+fn maintenance_repair_enabled() -> bool {
+    matches!(std::env::var("DW_RUST_MAINTENANCE_LOOP").as_deref(), Ok("1" | "true"))
+        && !matches!(std::env::var("DW_V2_MATCHING_ROLE_QUEUE_WAKE").as_deref(), Ok("false" | "0"))
 }
 
 impl TimerScheduler {
@@ -61,11 +74,13 @@ impl TimerScheduler {
             stop,
             task: Mutex::new(None),
             healthy: Arc::new(AtomicBool::new(true)),
+            repair: Arc::new(AtomicBool::new(maintenance_repair_enabled())),
         }
     }
     fn start(&self, storage: Arc<Storage>) {
         let mut stop = self.stop.subscribe();
         let healthy = self.healthy.clone();
+        let repair = self.repair.clone();
         let task = tokio::spawn(async move {
             loop {
                 tokio::select! {
@@ -75,10 +90,15 @@ impl TimerScheduler {
                         // retried from durable state, without logging payloads.
                         // Timer and schedule transactions share the cross-node lock with
                         // completion. CPU/deadline performance is unqualified.
-                        let result = async { match &*storage {
-                            Storage::Sqlite(store) => { store.fire_due_timers().await?; store.fire_due_schedules().await },
-                            Storage::Postgres(store) => { store.fire_due_timers().await?; store.fire_due_schedules().await },
-                            Storage::MySql(store) => { store.fire_due_timers().await?; store.fire_due_schedules().await },
+                        // Then PHP's maintenance loop (`workflow:v2:repair-pass`),
+                        // at most once per repair throttle: run deadlines, task
+                        // repair, activity and service-call timeouts.
+                        let pass = repair.load(Ordering::Relaxed)
+                            .then(|| super::repair_pass::PassRequest { respect_throttle: true, ..Default::default() });
+                        let result: Result<()> = async { match &*storage {
+                            Storage::Sqlite(store) => { store.fire_due_timers().await?; store.fire_due_schedules().await?; if let Some(pass) = &pass { store.repair_pass(pass).await?; } Ok(()) },
+                            Storage::Postgres(store) => { store.fire_due_timers().await?; store.fire_due_schedules().await?; if let Some(pass) = &pass { store.repair_pass(pass).await?; } Ok(()) },
+                            Storage::MySql(store) => { store.fire_due_timers().await?; store.fire_due_schedules().await?; if let Some(pass) = &pass { store.repair_pass(pass).await?; } Ok(()) },
                         }}.await;
                         let success = result.is_ok();
                         if healthy.swap(success,Ordering::Relaxed) && !success {
@@ -188,6 +208,13 @@ impl Runtime {
         Ok(())
     }
 
+    /// Turns the maintenance loop's repair pass on or off for this runtime,
+    /// as `DW_V2_MATCHING_ROLE_QUEUE_WAKE` does at startup. Timers, schedules
+    /// and deadlines still fire.
+    pub fn set_maintenance_repair(&self, enabled: bool) {
+        self.scheduler.repair.store(enabled, Ordering::Relaxed);
+    }
+
     pub async fn open(database: &str, token: String) -> Result<Self> {
         Self::validate_token(&token)?;
         let external = super::ExternalPayloadConfig::from_env()?;