From d65bd8124fde22ee1d8bf1745d6f97fdf551f586 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:28:27 +0000 Subject: [PATCH 01/20] Fingerprint a stopped PHP source database The takeover design (#517) proves a backup by comparing a fingerprint of the stopped source with the same fingerprint of the backup restored into a separate database. This adds the read-only upgrade-fingerprint command. The digest covers the qualified source catalog and every row of every table in primary-key order, each value rendered by the database itself in one fixed form: SQLite quote() with 17-digit reals, PostgreSQL quote_nullable() under fixed time zone, date, interval, float and bytea settings, and MySQL QUOTE() in UTC with binary columns as hex. Values are length-framed before hashing. It reads in one snapshot through the same read-only source checks as inspection, and reports only the digest, the per-table row counts and the elapsed time. Every compatibility, backup, quiescence and takeover flag stays false. The SQLite source checks are factored out of upgrade-inspect unchanged. Unit tests cover equality for a byte copy and for rows inserted in another order, a change for each altered value including NULL against 0 and text against blob, a return to the original digest when values are restored, an unchanged source file, and refusal of an owned or unknown catalog. The upgrade source cohort now fingerprints the published PHP source twice, restores its backup into a separate database (mysqldump, pg_dump and pg_restore, or a file copy for SQLite), requires an equal digest there, and requires one changed row in the restored copy to change it. Co-Authored-By: Claude Opus 5.5 --- rust/src/main.rs | 32 ++ rust/src/runtime/upgrade.rs | 182 ++++++----- rust/src/runtime/upgrade/fingerprint.rs | 308 ++++++++++++++++++ rust/src/runtime/upgrade_tests.rs | 90 +++++ .../server-parity/upgrade-source-cleanup.sh | 15 + .../server-parity/upgrade-source-profile.sh | 64 +++- 6 files changed, 604 insertions(+), 87 deletions(-) create mode 100644 rust/src/runtime/upgrade/fingerprint.rs diff --git a/rust/src/main.rs b/rust/src/main.rs index 22abcb22c..10b77cafb 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -132,11 +132,43 @@ async fn main() -> Result<(), Box> { | "upgrade-inspect" | "upgrade-queue-inspect" | "upgrade-payload-inspect" + | "upgrade-fingerprint" ) { return Err("unknown development command".into()); } let backend = env::var("DB_CONNECTION").unwrap_or_default(); + if command == "upgrade-fingerprint" { + use durable_workflow_server::runtime::upgrade::fingerprint; + let result = match backend.as_str() { + "sqlite" => { + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + fingerprint::fingerprint_sqlite_cli(&path).await + } + "mysql" | "mariadb" => fingerprint::fingerprint_mysql(mysql_options()?).await, + "pgsql" => fingerprint::fingerprint_postgres(postgres_options()?).await, + _ => { + return Err( + "upgrade fingerprint requires DB_CONNECTION=sqlite, pgsql, mysql or mariadb" + .into(), + ); + } + }; + return match result { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_fingerprint_unavailable", + }; + eprintln!("{reason}"); + Err("upgrade source fingerprint refused".into()) + } + }; + } if matches!( command.as_str(), "upgrade-inspect" | "upgrade-queue-inspect" | "upgrade-payload-inspect" diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 6172da77c..a8c62ee70 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -1,5 +1,6 @@ //! Read-only PHP source inventory. This command never grants takeover authority. pub mod external_reference; +pub mod fingerprint; mod payload_scan; pub mod queue; pub mod queue_binding; @@ -189,65 +190,68 @@ pub async fn inspect_sqlite_cli( ) -> Result { timeout(BUDGET, async { queue_config(queues)?; - if path.is_empty() || path.contains(['?', ':']) { + let resolved = readonly_source(path).await?; + inspect_sqlite_inner(&resolved, queues, payloads).await + }) + .await + .map_err(|_| expired())? +} + +/// Resolves a CLI source file and requires that it and its directory are on a +/// kernel read-only mount. +pub(super) async fn readonly_source(path: &str) -> Result { + if path.is_empty() || path.contains(['?', ':']) { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_file_required", + )); + } + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + // SQLite resolves symlinks before locating its sidecars. Check the + // resolved parent as well: a file-only read-only bind mount can + // still have a writable directory beside it. + let resolved = tokio::fs::canonicalize(path).await?; + let mut options = tokio::fs::OpenOptions::new(); + options.read(true).custom_flags(libc::O_NONBLOCK); + let file = options.open(&resolved).await?; + if !file.metadata().await?.is_file() { return Err(refuse( StatusCode::BAD_REQUEST, - "upgrade_source_file_required", + "upgrade_source_regular_file_required", )); } - #[cfg(unix)] - { - use std::os::fd::AsRawFd; - // SQLite resolves symlinks before locating its sidecars. Check the - // resolved parent as well: a file-only read-only bind mount can - // still have a writable directory beside it. - let resolved = tokio::fs::canonicalize(path).await?; - let mut options = tokio::fs::OpenOptions::new(); - options.read(true).custom_flags(libc::O_NONBLOCK); - let file = options.open(&resolved).await?; - if !file.metadata().await?.is_file() { + let parent = options + .open(resolved.parent().ok_or_else(|| { + refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required") + })?) + .await?; + for descriptor in [&file, &parent] { + let mut filesystem = std::mem::MaybeUninit::::uninit(); + // fstatvfs uses this live descriptor's mount. Only read the + // returned structure after success. + if unsafe { libc::fstatvfs(descriptor.as_raw_fd(), filesystem.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error().into()); + } + let filesystem = unsafe { filesystem.assume_init() }; + if filesystem.f_flag & libc::ST_RDONLY == 0 { return Err(refuse( - StatusCode::BAD_REQUEST, - "upgrade_source_regular_file_required", + StatusCode::CONFLICT, + "upgrade_source_readonly_filesystem_required", )); } - let parent = options - .open(resolved.parent().ok_or_else(|| { - refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required") - })?) - .await?; - for descriptor in [&file, &parent] { - let mut filesystem = std::mem::MaybeUninit::::uninit(); - // fstatvfs uses this live descriptor's mount. Only read the - // returned structure after success. - if unsafe { libc::fstatvfs(descriptor.as_raw_fd(), filesystem.as_mut_ptr()) } != 0 { - return Err(std::io::Error::last_os_error().into()); - } - let filesystem = unsafe { filesystem.assume_init() }; - if filesystem.f_flag & libc::ST_RDONLY == 0 { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_readonly_filesystem_required", - )); - } - } - return inspect_sqlite_inner( - resolved.to_str().ok_or_else(|| { - refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required") - })?, - queues, - payloads, - ) - .await; } - #[cfg(not(unix))] - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_readonly_filesystem_unavailable", - )); - }) - .await - .map_err(|_| expired())? + return Ok(resolved + .to_str() + .ok_or_else(|| refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required"))? + .to_owned()); + } + #[cfg(not(unix))] + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_readonly_filesystem_unavailable", + )); } async fn inspect_sqlite_inner( @@ -257,6 +261,45 @@ async fn inspect_sqlite_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; + let mut connection = open_sqlite_source(path, deadline).await?; + let result = timeout_at(deadline, async { + let mut tx = connection.begin().await?; + let catalog = schema::php_source_catalog(&mut tx).await?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, _, _)| kind == "table") + .map(|(_, name, _, _)| name.clone()) + .collect(); + let counts = count_tables!(tx, tables, '"'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + tx.rollback().await?; + report( + "sqlite", + &catalog, + counts, + queue_inspection, + payload_inspection, + ) + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + close(connection.close()).await?; + result +} + +/// Opens a qualified rollback-journal source file read-only. +pub(super) async fn open_sqlite_source(path: &str, deadline: Instant) -> Result { if path.is_empty() || path.contains(['?', ':']) { return Err(refuse( StatusCode::BAD_REQUEST, @@ -307,7 +350,7 @@ async fn inspect_sqlite_inner( }) .await .map_err(|_| expired())??; - let mut connection = timeout_at( + let connection = timeout_at( deadline, SqliteConnection::connect_with( &SqliteConnectOptions::new() @@ -319,40 +362,7 @@ async fn inspect_sqlite_inner( ) .await .map_err(|_| expired())??; - let result = timeout_at(deadline, async { - let mut tx = connection.begin().await?; - let catalog = schema::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, _, _)| kind == "table") - .map(|(_, name, _, _)| name.clone()) - .collect(); - let counts = count_tables!(tx, tables, '"'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; - tx.rollback().await?; - report( - "sqlite", - &catalog, - counts, - queue_inspection, - payload_inspection, - ) - }) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result + Ok(connection) } pub async fn inspect_mysql(options: MySqlConnectOptions) -> Result { diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs new file mode 100644 index 000000000..d1bdc9b02 --- /dev/null +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -0,0 +1,308 @@ +//! Logical fingerprint of a stopped PHP source database. +//! +//! The digest covers the qualified source catalog and every row of every table +//! in primary-key order, with each value rendered by the database itself in one +//! fixed text form. A stopped source and its backup restored into a separate +//! database give equal digests only when the restore reproduces every row, and +//! takeover compares it again before writing anything. Only the digest and the +//! per-table row counts leave the database. The fingerprint grants nothing: +//! every compatibility, backup, quiescence and takeover flag stays false. +use super::{ + Result, close, expired, mysql, open_sqlite_source, postgres, readonly_source, refuse, schema, +}; +use axum::http::StatusCode; +use futures_util::TryStreamExt; +use serde::Serialize; +use serde_json::{Value, json}; +use sha2::{Digest as _, Sha256}; +use sqlx::{ + AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, mysql::MySqlConnectOptions, + postgres::PgConnectOptions, +}; +use std::{collections::BTreeMap, time::Duration}; +use tokio::time::{Instant, timeout, timeout_at}; + +/// A fingerprint reads every row, so it gets far longer than an inspection. +const BUDGET: Duration = Duration::from_secs(3600); + +struct Digest { + hasher: Sha256, + rows: BTreeMap, + started: Instant, +} + +impl Digest { + fn new(backend: &str, catalog: &impl Serialize) -> Result { + let mut digest = Self { + hasher: Sha256::new(), + rows: BTreeMap::new(), + started: Instant::now(), + }; + digest.frame(b"durable-workflow.rust-upgrade-source-fingerprint.v1"); + digest.frame(backend.as_bytes()); + digest.frame(&serde_json::to_vec(catalog)?); + Ok(digest) + } + + /// Length-prefixed, so no value can be mistaken for a boundary. + fn frame(&mut self, bytes: &[u8]) { + self.hasher.update((bytes.len() as u64).to_be_bytes()); + self.hasher.update(bytes); + } + + fn table(&mut self, name: &str, columns: &[String]) { + self.frame(b"table"); + self.frame(name.as_bytes()); + self.frame(&(columns.len() as u64).to_be_bytes()); + for column in columns { + self.frame(column.as_bytes()); + } + } + + fn report(self, backend: &str) -> Value { + let hex: String = self + .hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + json!({ + "schema":"durable-workflow.rust-upgrade-source-fingerprint.v1", + "status":"fingerprinted", "backend":backend, "fingerprint":format!("sha256:{hex}"), + "table_rows":self.rows, "elapsed_ms":self.started.elapsed().as_millis() as u64, + "source_mutated":false, "takeover_allowed":false, "backup_verified":false, + "php_quiescence_verified":false, "stored_representation_compatibility_verified":false + }) + } +} + +/// Names come from a catalog equal to a qualified PHP layout. Anything else is +/// refused rather than quoted. +fn identifier(name: &str) -> Result<&str> { + if name.is_empty() || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + Ok(name) +} + +/// Primary-key order, or every rendered column when a table has no key. +fn order(key: &[String], width: usize, quote: char) -> Result { + if key.is_empty() { + return Ok((1..=width).map(|n| n.to_string()).collect::>().join(",")); + } + Ok(key + .iter() + .map(|column| identifier(column).map(|column| format!("{quote}{column}{quote}"))) + .collect::>>()? + .join(",")) +} + +// Every rendered value is text, framed in column order. +macro_rules! hash_rows { + ($tx:ident, $digest:ident, $table:expr, $sql:expr, $width:expr) => {{ + let mut count = 0_i64; + let mut rows = sqlx::query(AssertSqlSafe($sql)).fetch(&mut *$tx); + while let Some(row) = rows.try_next().await? { + $digest.frame(b"row"); + for index in 0..$width { + let value: String = row.try_get(index)?; + $digest.frame(value.as_bytes()); + } + count += 1; + } + drop(rows); + $digest.rows.insert($table.to_owned(), count); + }}; +} + +/// Fingerprints a SQLite source the operator CLI found on a read-only mount. +pub async fn fingerprint_sqlite_cli(path: &str) -> Result { + timeout(BUDGET, async { + let resolved = readonly_source(path).await?; + fingerprint_sqlite(&resolved).await + }) + .await + .map_err(|_| expired())? +} + +pub async fn fingerprint_sqlite(path: &str) -> Result { + let deadline = Instant::now() + BUDGET; + let mut connection = open_sqlite_source(path, deadline).await?; + let result = timeout_at(deadline, async { + // One deferred read transaction keeps one snapshot of the file. + let mut tx = connection.begin().await?; + let catalog = schema::php_source_catalog(&mut tx).await?; + let mut digest = Digest::new("sqlite", &catalog)?; + for (_, table, _, _) in catalog.iter().filter(|(kind, _, _, _)| kind == "table") { + let table = identifier(table)?; + let columns: Vec = + sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) ORDER BY cid") + .bind(table) + .fetch_all(&mut *tx) + .await?; + let key: Vec = + sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) WHERE pk>0 ORDER BY pk") + .bind(table) + .fetch_all(&mut *tx) + .await?; + digest.table(table, &columns); + // quote() keeps the storage class: integers, text and blobs render + // exactly. Reals use 17 significant digits, which round-trip. + let select = columns + .iter() + .map(|column| { + identifier(column).map(|c| { + format!("CASE typeof(\"{c}\") WHEN 'real' THEN printf('%!.17g',\"{c}\") ELSE quote(\"{c}\") END") + }) + }) + .collect::>>()? + .join(","); + let sql = format!( + "SELECT {select} FROM \"{table}\" ORDER BY {}", + order(&key, columns.len(), '"')? + ); + hash_rows!(tx, digest, table, sql, columns.len()); + } + tx.rollback().await?; + Ok(digest.report("sqlite")) + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + close(connection.close()).await?; + result +} + +pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { + let deadline = Instant::now() + BUDGET; + let mut connection = timeout_at( + deadline, + PgConnection::connect_with(&postgres::source_options(options)), + ) + .await + .map_err(|_| expired())??; + let result = timeout_at(deadline, async { + let mut tx = connection + .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + .await?; + // Fixed text forms for timestamps, intervals, floats and bytes. + sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'") + .execute(&mut *tx) + .await?; + let catalog = postgres::php_source_catalog(&mut tx).await?; + let mut digest = Digest::new("pgsql", &catalog)?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + for table in &tables { + let table = identifier(table)?; + let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") + .bind(table).fetch_all(&mut *tx).await?; + digest.table(table, &columns); + let select = columns + .iter() + .map(|column| identifier(column).map(|c| format!("quote_nullable(\"{c}\")"))) + .collect::>>()? + .join(","); + let sql = format!( + "SELECT {select} FROM public.\"{table}\" ORDER BY {}", + order(&key, columns.len(), '"')? + ); + hash_rows!(tx, digest, table, sql, columns.len()); + } + tx.rollback().await?; + Ok(digest.report("pgsql")) + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + close(connection.close()).await?; + result +} + +pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { + let deadline = Instant::now() + BUDGET; + let mut connection = timeout_at( + deadline, + MySqlConnection::connect_with(&mysql::source_options(options)), + ) + .await + .map_err(|_| expired())??; + let result = timeout_at(deadline, async { + let version: String = sqlx::query_scalar("SELECT VERSION()") + .fetch_one(&mut connection) + .await?; + let backend = if version.contains("MariaDB") { + "mariadb" + } else { + "mysql" + }; + // TIMESTAMP values render in UTC. Long table reads are expected, but + // lock waits stay short; the client deadline bounds the whole read. + sqlx::query(if backend == "mariadb" { + "SET time_zone='+00:00',sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_statement_time=0" + } else { + "SET time_zone='+00:00',sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_execution_time=0" + }) + .execute(&mut connection) + .await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + .execute(&mut connection) + .await?; + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let catalog = mysql::php_source_catalog(&mut tx).await?; + let mut digest = Digest::new(backend, &catalog)?; + for (table, _) in &catalog { + let table = identifier(table)?; + let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let names: Vec = columns.iter().map(|(name, _)| name.clone()).collect(); + digest.table(table, &names); + // Binary values render as hex literals; everything else as its + // quoted string form. + let select = columns + .iter() + .map(|(column, kind)| { + identifier(column).map(|c| match kind.to_ascii_lowercase().as_str() { + "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( + "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" + ), + _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), + }) + }) + .collect::>>()? + .join(","); + let sql = format!( + "SELECT {select} FROM `{table}` ORDER BY {}", + order(&key, columns.len(), '`')? + ); + hash_rows!(tx, digest, table, sql, columns.len()); + } + // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that + // changed while the rows were read. + if mysql::php_source_catalog(&mut tx).await? != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + tx.rollback().await?; + Ok(digest.report(backend)) + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + close(connection.close()).await?; + result +} diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 08ea8309d..591181e4e 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -725,3 +725,93 @@ async fn read_only_source_file_can_be_inspected_without_write_permissions() { ); assert_eq!(std::fs::read(&path).unwrap(), before); } + +#[tokio::test] +async fn fingerprint_matches_a_byte_copy_and_changes_with_any_row() { + let (directory, path) = php_layout().await; + let before = tokio::fs::read(&path).await.unwrap(); + let first = fingerprint::fingerprint_sqlite(&path).await.unwrap(); + assert_eq!(first["status"], "fingerprinted"); + assert_eq!(first["table_rows"]["jobs"], 1); + for flag in [ + "source_mutated", + "takeover_allowed", + "backup_verified", + "php_quiescence_verified", + "stored_representation_compatibility_verified", + ] { + assert_eq!(first[flag], false, "{flag}"); + } + assert!(!first.to_string().contains("private-payload"), "{first}"); + assert_eq!(tokio::fs::read(&path).await.unwrap(), before); + assert_eq!( + fingerprint::fingerprint_sqlite(&path).await.unwrap()["fingerprint"], + first["fingerprint"] + ); + let copy = directory.path().join("copy.sqlite"); + tokio::fs::copy(&path, ©).await.unwrap(); + let copy = copy.to_str().unwrap(); + assert_eq!( + fingerprint::fingerprint_sqlite(copy).await.unwrap()["fingerprint"], + first["fingerprint"] + ); + let mut seen = vec![first["fingerprint"].clone()]; + for statement in [ + "UPDATE jobs SET attempts=1", + "UPDATE jobs SET reserved_at=7", + "UPDATE jobs SET reserved_at=0", + "UPDATE jobs SET payload='private-payload-λ '", + "INSERT INTO migrations(migration,batch) VALUES ('extra',1)", + ] { + mutate(copy, statement).await; + let changed = fingerprint::fingerprint_sqlite(copy).await.unwrap()["fingerprint"].clone(); + assert!(!seen.contains(&changed), "{statement}"); + seen.push(changed); + } + // The digest describes contents, so restoring earlier values restores it. + mutate( + copy, + "UPDATE jobs SET attempts=0,reserved_at=NULL,payload='private-payload-λ'; DELETE FROM migrations", + ) + .await; + assert_eq!( + fingerprint::fingerprint_sqlite(copy).await.unwrap()["fingerprint"], + first["fingerprint"] + ); +} + +#[tokio::test] +async fn fingerprint_ignores_physical_row_order_but_not_storage_class() { + let (_first_directory, first) = php_layout().await; + let (_second_directory, second) = php_layout().await; + mutate(&first, "INSERT INTO migrations(id,migration,batch) VALUES (5,'a',1),(3,'b',1)").await; + mutate(&second, "INSERT INTO migrations(id,migration,batch) VALUES (3,'b',1)").await; + mutate(&second, "INSERT INTO migrations(id,migration,batch) VALUES (5,'a',1)").await; + let one = fingerprint::fingerprint_sqlite(&first).await.unwrap(); + assert_eq!( + fingerprint::fingerprint_sqlite(&second).await.unwrap()["fingerprint"], + one["fingerprint"] + ); + // A text value is not the same row as an equal-looking blob. + mutate(&second, "UPDATE migrations SET migration=CAST('a' AS BLOB) WHERE id=5").await; + assert_ne!( + fingerprint::fingerprint_sqlite(&second).await.unwrap()["fingerprint"], + one["fingerprint"] + ); +} + +#[tokio::test] +async fn fingerprint_refuses_an_owned_or_unknown_source() { + let (_directory, path) = php_layout().await; + mutate(&path, "CREATE TABLE dw_server_schema(engine TEXT)").await; + reason( + fingerprint::fingerprint_sqlite(&path).await, + "upgrade_source_already_owned", + ); + let (_directory, path) = php_layout().await; + mutate(&path, "ALTER TABLE jobs ADD COLUMN unexpected TEXT").await; + reason( + fingerprint::fingerprint_sqlite(&path).await, + "upgrade_source_catalog_unqualified", + ); +} diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index 35b995418..400f04641 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -34,6 +34,21 @@ case "$backend" in *) exit 2 ;; esac fi +if test -f "$evidence/restore-owned.txt" && test "$(cat "$evidence/restore-owned.txt")" = "$RESOURCE_SCOPE"; then +case "$backend" in + mysql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ + -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ + -e 'DROP DATABASE IF EXISTS upgrade_restore_ref' \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + pgsql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c 'DROP DATABASE IF EXISTS upgrade_restore_ref' \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + *) ;; +esac +fi if test -f "$evidence/roles-owned.txt" && test "$(cat "$evidence/roles-owned.txt")" = "$RESOURCE_SCOPE"; then if test "$backend" = mysql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 2c7e58f5c..070277491 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -100,7 +100,8 @@ inspect_source() { } snapshot() { if test "$backend" = sqlite; then - find "$state" -maxdepth 1 -type f -print0 | sort -z | xargs -0 -r sha256sum + # The source file and any sidecar it gains; the restored copy lives beside it. + find "$state" -maxdepth 1 -type f -name 'database.sqlite*' -print0 | sort -z | xargs -0 -r sha256sum else local helper=mysql if test "$backend" = pgsql; then helper=postgres; fi @@ -132,6 +133,64 @@ jq -e '.status=="inspected" and (.payload_inspection.columns|length)==4 and .payload_inspection.registry_binding_verified==false and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/payload-inspection.json" >/dev/null +fingerprint() { + local target=$1 + docker run --rm --name "$resource-inspect" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network "$RESOURCE_SCOPE" "${reader[@]}" -e DB_DATABASE="$target" -e DW_RUST_EXPERIMENTAL=1 \ + -v "$PWD/$state:/state:ro" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-fingerprint +} +fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" +jq -e '.status=="fingerprinted" and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) + and (.table_rows|length)==50 and .table_rows.workflow_runs==2 and .table_rows.jobs==1 + and .source_mutated==false and .takeover_allowed==false and .backup_verified==false + and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null +source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") +fingerprint "$database" > "$evidence/fingerprint-repeat.json" 2>> "$evidence/fingerprint.stderr" +test "$(jq -r .fingerprint "$evidence/fingerprint-repeat.json")" = "$source_fingerprint" +# Back up the stopped source and restore it into a separate database, as the +# backup procedure does. The restored copy must fingerprint equal, and one +# changed row in it must change the fingerprint. +restored=upgrade_restore_ref +case "$backend" in + mysql) + mysql_admin() { + docker run --rm -i --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint "$1" "$MYSQL_IMAGE" \ + -h "$host" -uroot -p"$password" "${@:2}" + } + mysql_admin mysqldump --single-transaction --quick --no-tablespaces "$database" > "$state/backup.sql" 2> "$evidence/backup.stderr" + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/restore-owned.txt" + mysql_admin mysql -e "CREATE DATABASE $restored CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 2>> "$evidence/backup.stderr" + mysql_admin mysql "$restored" < "$state/backup.sql" 2>> "$evidence/backup.stderr" + mysql_admin mysql -e "GRANT SELECT ON $restored.* TO 'upgrade_source_reader'@'%'" 2>> "$evidence/backup.stderr" + change_restored() { mysql_admin mysql "$restored" -e 'UPDATE jobs SET attempts=attempts+1' 2>> "$evidence/backup.stderr"; } ;; + pgsql) + pg_admin() { + docker run --rm -i --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint "$1" "$POSTGRES_IMAGE" -h "$host" -U parity "${@:2}" + } + pg_admin pg_dump -Fc -d "$database" > "$state/backup.dump" 2> "$evidence/backup.stderr" + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/restore-owned.txt" + pg_admin createdb "$restored" 2>> "$evidence/backup.stderr" + pg_admin pg_restore --no-owner --exit-on-error -d "$restored" < "$state/backup.dump" 2>> "$evidence/backup.stderr" + pg_admin psql -d "$restored" -v ON_ERROR_STOP=1 -c "GRANT CONNECT ON DATABASE $restored TO upgrade_source_reader" \ + -c 'GRANT USAGE ON SCHEMA public TO upgrade_source_reader' \ + -c 'GRANT SELECT ON ALL TABLES IN SCHEMA public TO upgrade_source_reader' > /dev/null 2>> "$evidence/backup.stderr" + change_restored() { pg_admin psql -d "$restored" -v ON_ERROR_STOP=1 -c 'UPDATE jobs SET attempts=attempts+1' > /dev/null 2>> "$evidence/backup.stderr"; } ;; + sqlite) + # A stopped SQLite installation is backed up by copying its file. + cp "$state/database.sqlite" "$state/restored.sqlite" + restored=/state/restored.sqlite + change_restored() { + php_run -r '(new PDO("sqlite:/state/restored.sqlite"))->exec("UPDATE jobs SET attempts=attempts+1") === 1 || exit(1);' + } ;; +esac +fingerprint "$restored" > "$evidence/restored-fingerprint.json" 2>> "$evidence/fingerprint.stderr" +test "$(jq -r .fingerprint "$evidence/restored-fingerprint.json")" = "$source_fingerprint" +change_restored +fingerprint "$restored" > "$evidence/changed-fingerprint.json" 2>> "$evidence/fingerprint.stderr" +changed_fingerprint=$(jq -r .fingerprint "$evidence/changed-fingerprint.json") +[[ $changed_fingerprint =~ ^sha256:[0-9a-f]{64}$ && $changed_fingerprint != "$source_fingerprint" ]] snapshot > "$evidence/after.json" cmp "$evidence/before.json" "$evidence/after.json" refusal() { @@ -163,9 +222,12 @@ git diff --cached --quiet jq -n --arg revision "$revision" --arg native "$native_tree" \ --arg image "$image" --arg backend "$backend" --arg binary "$binary_sha256" \ --slurpfile inventory "$evidence/inspection.json" --slurpfile originals "$evidence/originals.json" \ + --slurpfile fingerprint "$evidence/fingerprint.json" \ '{outcome:"pass",scope:"read_only_source_inventory_and_refusals",backend:$backend, runner_revision:$revision,native_tree:$native,native_binary_sha256:$binary,php_image:$image, original_runs:2,source_state_unchanged_by_inspection:true, + source_fingerprint:$fingerprint[0].fingerprint,restored_backup_fingerprint_equal:true, + changed_restored_row_changes_fingerprint:true, catalog_sha256:$inventory[0].catalog_sha256,table_count:($inventory[0].table_rows|length), originals:($originals[0].originals|to_entries|map({kind:.key,workflow_id:.value.workflow_id, run_id:.value.run_id,history_events:(.value.history|length),activity_attempt_id:.value.activity_task.activity_attempt_id})), From ff5356bfdcaad77fb74f581eb62c2f4ea563c1c9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:39:50 +0000 Subject: [PATCH 02/20] Compare MySQL column character sets as their collation A PHP 2.8.0 MySQL source restored from mysqldump prints each column as CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, where the source printed COLLATE utf8mb4_unicode_ci. The schema is the same, since a collation names its character set, but the catalog check compared the text, so inspection refused every installation restored from a dump as unqualified. The live catalog, the frozen native catalogs and the published PHP variant deltas now all drop a character set that the following collation names. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/mysql.rs | 57 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 5ae70bc04..627a7b167 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -67,10 +67,10 @@ impl Flavor { }) } fn expected(self) -> Result { - Ok(serde_json::from_str(match self { + Ok(serde_json::from_str(&canonical(match self { Self::MySql => include_str!("../../migrations/mysql/0004_catalog.json"), Self::MariaDb => include_str!("../../migrations/mariadb/0004_catalog.json"), - })?) + }))?) } fn checksum(self) -> &'static [u8] { self.migrator().iter().next().unwrap().checksum.as_ref() @@ -106,6 +106,38 @@ fn table_sql(prefix: &str, table: &str, suffix: &str) -> Result String { + const CHARSET: &str = " CHARACTER SET "; + let mut result = String::with_capacity(sql.len()); + let mut rest = sql; + while let Some(at) = rest.find(CHARSET) { + let after = &rest[at + CHARSET.len()..]; + let length = after + .bytes() + .take_while(|b| b.is_ascii_alphanumeric() || *b == b'_') + .count(); + let (charset, tail) = after.split_at(length); + if length > 0 + && tail + .strip_prefix(" COLLATE ") + .is_some_and(|collation| { + collation.starts_with(charset) && collation[length..].starts_with('_') + }) + { + result.push_str(&rest[..at]); + } else { + result.push_str(&rest[..at + CHARSET.len() + length]); + } + rest = tail; + } + result.push_str(rest); + result +} + pub fn definition(sql: String) -> String { let Some(footer) = sql.rfind(") ENGINE=") else { return sql; @@ -144,7 +176,7 @@ async fn catalog(connection: &mut MySqlConnection) -> Result { let row = sqlx::query(table_sql("SHOW CREATE TABLE ", &name, "")?) .fetch_one(&mut *connection) .await?; - result.push((name, definition(row.try_get(1)?))); + result.push((name, canonical(&definition(row.try_get(1)?)))); } Ok(result) } @@ -189,7 +221,7 @@ pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Resu Flavor::MySql => include_str!("../../upgrade/php-2.8.0/mysql.json"), Flavor::MariaDb => include_str!("../../upgrade/php-2.8.0/mariadb.json"), }; - let mut current = super::upgrade::source_catalog_variant(&expected, delta)?; + let mut current = super::upgrade::source_catalog_variant(&expected, &canonical(delta))?; current.sort_by(|left, right| left.0.cmp(&right.0)); if actual != current { return Err(refuse( @@ -514,6 +546,23 @@ mod tests { sqlx::query("INSERT INTO _sqlx_migrations VALUES (?, 'interrupted qualification', CURRENT_TIMESTAMP, FALSE, ?, -1)").bind(VERSION).bind(flavor.checksum()).execute(connection).await.unwrap(); } + #[test] + fn restored_column_character_sets_compare_as_their_collation() { + let created = "CREATE TABLE `t` (\n `a` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `b` char(8) COLLATE ascii_bin\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; + let restored = "CREATE TABLE `t` (\n `a` varchar(26) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,\n `b` char(8) CHARACTER SET ascii COLLATE ascii_bin\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; + assert_eq!(canonical(restored), created); + assert_eq!(canonical(created), created); + // Only a character set its collation names is redundant. + for kept in [ + "`a` text CHARACTER SET utf8 COLLATE utf8mb4_bin", + "`a` text CHARACTER SET utf8mb4", + "`a` text CHARACTER SET latin1 COLLATE utf8mb4_bin", + "`a` text COMMENT ' CHARACTER SET '", + ] { + assert_eq!(canonical(kept), kept); + } + } + #[tokio::test] #[ignore = "requires disposable MySQL/MariaDB; run qualification_ explicitly"] async fn qualification_schedule_creation_keeps_original_audit_and_unsigned_counters() { From 7afa5c175f3d6259166775bbea4030a1912911f6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:17:05 +0000 Subject: [PATCH 03/20] Let the interrupted-bootstrap child publish its receipt atomically The SQLite, PostgreSQL and MySQL interruption tests wait for the child's receipt file to exist and then read it. std::fs::write creates the file before writing it, so a parent can read it empty: #508's MySQL job failed with left "" right "dirty-partial". The child now writes beside the receipt and renames it into place. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/mysql.rs | 6 +++++- rust/src/runtime/postgres.rs | 6 +++++- rust/src/runtime/schema.rs | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 627a7b167..8b1e4eebc 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -925,7 +925,11 @@ mod tests { connection.execute(LEDGER_SQL).await.unwrap(); flavor.migrator().run(&mut connection).await.unwrap(); } - std::fs::write(std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(), phase).unwrap(); + // The parent waits for the receipt to exist, so it must appear with + // its contents: write beside it and rename into place. + let receipt = std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(); + std::fs::write(format!("{receipt}.partial"), &phase).unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); loop { std::thread::park(); } diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index f7200b79b..98719af2d 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -658,7 +658,11 @@ mod tests { verify(&mut tx).await.unwrap(); } // An actual server acknowledgement at a still-uncommitted boundary. - std::fs::write(std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(), phase).unwrap(); + // The parent waits for the receipt to exist, so it must appear with + // its contents: write beside it and rename into place. + let receipt = std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(); + std::fs::write(format!("{receipt}.partial"), &phase).unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); loop { std::thread::park(); } diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 279e78d67..d9842f2df 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -180,7 +180,10 @@ mod tests { } // This receipt acknowledges an actual SQLite boundary inside the // still-open outer transaction. The parent sends a real SIGKILL. - std::fs::write(format!("{path}.boundary"), phase).unwrap(); + // The parent waits for the receipt to exist, so it must appear with + // its contents: write beside it and rename into place. + std::fs::write(format!("{path}.boundary.partial"), &phase).unwrap(); + std::fs::rename(format!("{path}.boundary.partial"), format!("{path}.boundary")).unwrap(); loop { std::thread::park(); } From 26328b81140bb76bec537c9dcc06474e9f211534 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:40:15 +0000 Subject: [PATCH 04/20] Recognize the current PHP 2.9.7 layout and name the matched layout Upgrade inspection knew only the frozen layout and PHP 2.8.0, so a source on the current release, Server 2.9.7 with Core 2.7.2, was refused as an unknown catalog: Core added workflow_tasks.queue_delivery_reference. - Deltas are now chained: each recorded release layout is the previous one plus its delta. rust/upgrade/php-2.9.7 records the 2.9.7 delta for SQLite, MySQL, MariaDB and PostgreSQL, captured from real published-image databases. - On MariaDB, an installation using Laravel's separate mariadb connection stores the backup hold owner as uuid. mariadb-driver.json records that, and every release layout is also tried with it. - Inspection and fingerprint reports name the matched layout (source_layout), which takeover will use to choose its conversion. - The upgrade source cohort gains a labeled run against published 2.9.7 in the MySQL/MariaDB, PostgreSQL and SQLite jobs, with separate state, evidence, containers and databases, and checks the reported layout. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 45 ++++++++++++++ rust/src/runtime/mysql.rs | 62 ++++++++++++++----- rust/src/runtime/postgres.rs | 24 +++---- rust/src/runtime/schema.rs | 24 +++---- rust/src/runtime/upgrade.rs | 51 +++++++++++++-- rust/src/runtime/upgrade/fingerprint.rs | 19 +++--- rust/src/runtime/upgrade_tests.rs | 40 ++++++++++++ rust/upgrade/mariadb-driver.json | 14 +++++ rust/upgrade/php-2.9.7/README.md | 29 +++++++++ rust/upgrade/php-2.9.7/mariadb.json | 14 +++++ rust/upgrade/php-2.9.7/mysql.json | 14 +++++ rust/upgrade/php-2.9.7/postgres.json | 18 ++++++ rust/upgrade/php-2.9.7/sqlite.json | 18 ++++++ .../server-parity/upgrade-source-cleanup.sh | 18 +++--- .../server-parity/upgrade-source-profile.sh | 24 ++++--- 15 files changed, 340 insertions(+), 74 deletions(-) create mode 100644 rust/upgrade/mariadb-driver.json create mode 100644 rust/upgrade/php-2.9.7/README.md create mode 100644 rust/upgrade/php-2.9.7/mariadb.json create mode 100644 rust/upgrade/php-2.9.7/mysql.json create mode 100644 rust/upgrade/php-2.9.7/postgres.json create mode 100644 rust/upgrade/php-2.9.7/sqlite.json diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index c4bb53b6c..90a4dbe41 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -670,6 +670,21 @@ jobs: if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql + - name: Inspect published PHP 2.9.7 upgrade source without takeover + timeout-minutes: 8 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-profile.sh mysql + - name: Retire PHP 2.9.7 upgrade source cohort + if: always() + timeout-minutes: 3 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql - name: Compare corrected markers, ranges, clocks and activity expiry if: ${{ !cancelled() && steps.prepare_runtime_comparison.outcome == 'success' }} # The preceding process-loss case deliberately kills native-a. @@ -1149,6 +1164,21 @@ jobs: if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql + - name: Inspect published PHP 2.9.7 upgrade source without takeover + timeout-minutes: 8 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-profile.sh pgsql + - name: Retire PHP 2.9.7 upgrade source cohort + if: always() + timeout-minutes: 3 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql - name: Compare corrected markers, ranges, clocks and activity expiry if: ${{ !cancelled() && steps.prepare_runtime_comparison.outcome == 'success' }} env: @@ -1623,6 +1653,21 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite + - name: Inspect published PHP 2.9.7 upgrade source without takeover + timeout-minutes: 8 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-profile.sh sqlite + - name: Retire PHP 2.9.7 upgrade source cohort + if: always() + timeout-minutes: 3 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite - name: Compare corrected markers, ranges, clocks and activity expiry if: ${{ !cancelled() && steps.prepare_runtime_comparison.outcome == 'success' }} env: diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 8b1e4eebc..35e68ac0d 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -181,7 +181,7 @@ async fn catalog(connection: &mut MySqlConnection) -> Result { Ok(result) } -pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Result { +pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Result<(Catalog, &'static str)> { let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(TABLE_NAME)='dw_server_schema'") .fetch_one(&mut *connection).await?; if marked != 0 { @@ -206,7 +206,7 @@ pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Resu } } let flavor = Flavor::detect(connection).await?; - let mut expected: Catalog = flavor.expected()?.into_iter() + let expected: Catalog = flavor.expected()?.into_iter() .filter(|(table, _)| !super::upgrade::native_relation(table)) .map(|(table, ddl)| (table, ddl.replace( ",\n KEY `dw_workflow_tasks_poll` (`namespace`(64),`queue`(64),`task_type`(16),`status`(16),`available_at`)", ""))) @@ -214,23 +214,51 @@ pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Resu let mut actual = catalog(connection).await?; // MariaDB's information_schema collation ignores some underscores. Only // canonicalize catalog presentation; preserve every full definition. - expected.sort_by(|left, right| left.0.cmp(&right.0)); actual.sort_by(|left, right| left.0.cmp(&right.0)); - if actual != expected { - let delta = match flavor { - Flavor::MySql => include_str!("../../upgrade/php-2.8.0/mysql.json"), - Flavor::MariaDb => include_str!("../../upgrade/php-2.8.0/mariadb.json"), - }; - let mut current = super::upgrade::source_catalog_variant(&expected, &canonical(delta))?; - current.sort_by(|left, right| left.0.cmp(&right.0)); - if actual != current { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_unqualified", - )); + let deltas = match flavor { + Flavor::MySql => [ + canonical(include_str!("../../upgrade/php-2.8.0/mysql.json")), + canonical(include_str!("../../upgrade/php-2.9.7/mysql.json")), + ], + Flavor::MariaDb => [ + canonical(include_str!("../../upgrade/php-2.8.0/mariadb.json")), + canonical(include_str!("../../upgrade/php-2.9.7/mariadb.json")), + ], + }; + let layout = super::upgrade::source_layout( + &actual, + expected.clone(), + [&deltas[0], &deltas[1]], + |row| row.0.clone(), + ); + // Laravel's separate mariadb connection stores the backup hold owner as + // uuid rather than char(36); every release layout is otherwise the same. + let layout = match (layout, flavor) { + (Err(_), Flavor::MariaDb) => { + let driver = super::upgrade::source_catalog_variant( + &expected, + &canonical(include_str!("../../upgrade/mariadb-driver.json")), + )?; + match super::upgrade::source_layout( + &actual, + driver, + [&deltas[0], &deltas[1]], + |row| row.0.clone(), + )? { + "frozen" => "frozen+mariadb-driver", + "php-2.8.0" => "php-2.8.0+mariadb-driver", + "php-2.9.7" => "php-2.9.7+mariadb-driver", + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + } } - } - Ok(actual) + (layout, _) => layout?, + }; + Ok((actual, layout)) } pub(super) fn source_options(options: MySqlConnectOptions) -> MySqlConnectOptions { diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index 98719af2d..fa763e307 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -54,7 +54,7 @@ async fn catalog(connection: &mut PgConnection) -> Result { Ok(sqlx::query_as(CATALOG_QUERY).fetch_all(connection).await?) } -pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result { +pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result<(Catalog, &'static str)> { verify_scope(connection).await?; if marked(connection).await? { return Err(refuse(StatusCode::CONFLICT, "upgrade_source_already_owned")); @@ -80,20 +80,16 @@ pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result< }) .collect(); let actual = catalog(connection).await?; - if actual != expected { - let mut current = super::upgrade::source_catalog_variant( - &expected, + let layout = super::upgrade::source_layout( + &actual, + expected, + [ include_str!("../../upgrade/php-2.8.0/postgres.json"), - )?; - current.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); - if actual != current { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_unqualified", - )); - } - } - Ok(actual) + include_str!("../../upgrade/php-2.9.7/postgres.json"), + ], + |row| (row.0.clone(), row.1.clone()), + )?; + Ok((actual, layout)) } pub(super) fn source_options(options: PgConnectOptions) -> PgConnectOptions { diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index d9842f2df..9085600a2 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -67,7 +67,7 @@ async fn verify_catalog(connection: &mut SqliteConnection) -> Result<()> { Ok(()) } -pub(super) async fn php_source_catalog(connection: &mut SqliteConnection) -> Result { +pub(super) async fn php_source_catalog(connection: &mut SqliteConnection) -> Result<(Catalog, &'static str)> { let owned: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name='dw_server_schema' COLLATE NOCASE)", ) @@ -85,20 +85,16 @@ pub(super) async fn php_source_catalog(connection: &mut SqliteConnection) -> Res .cloned() .collect(); let actual = catalog(connection).await?; - if actual != expected { - let mut current = super::upgrade::source_catalog_variant( - &expected, + let layout = super::upgrade::source_layout( + &actual, + expected, + [ include_str!("../../upgrade/php-2.8.0/sqlite.json"), - )?; - current.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); - if actual != current { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_unqualified", - )); - } - } - Ok(actual) + include_str!("../../upgrade/php-2.9.7/sqlite.json"), + ], + |row| (row.0.clone(), row.1.clone()), + )?; + Ok((actual, layout)) } pub(super) async fn verify(connection: &mut SqliteConnection) -> Result<()> { diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index a8c62ee70..aef6b3be5 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -21,6 +21,44 @@ use tokio::time::{Instant, timeout, timeout_at}; const BUDGET: Duration = Duration::from_secs(30); +/// Published PHP layouts after the frozen one, oldest first. Each backend +/// records one delta per layout, applied to the layout before it. +pub(super) const SOURCE_LAYOUTS: [&str; 2] = ["php-2.8.0", "php-2.9.7"]; + +/// Names the qualified layout `actual` matches exactly: the frozen layout, or +/// one reached by applying the recorded deltas in order. Rows are compared in +/// `order`, so only presentation order, never a definition, is normalized. +pub(super) fn source_layout( + actual: &[T], + frozen: Vec, + deltas: [&str; 2], + order: impl Fn(&T) -> K, +) -> Result<&'static str> +where + T: Clone + PartialEq + DeserializeOwned, +{ + let sorted = |rows: &[T]| { + let mut rows = rows.to_vec(); + rows.sort_by_key(&order); + rows + }; + let actual = sorted(actual); + let mut layout = sorted(&frozen); + if actual == layout { + return Ok("frozen"); + } + for (name, delta) in SOURCE_LAYOUTS.into_iter().zip(deltas) { + layout = sorted(&source_catalog_variant(&layout, delta)?); + if actual == layout { + return Ok(name); + } + } + Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )) +} + /// Exact published-catalog additions/replacements, applied only in memory. /// Neither unknown objects nor unmatched old definitions are discarded. pub(super) fn source_catalog_variant(baseline: &[T], delta: &str) -> Result> @@ -101,6 +139,7 @@ async fn close(future: impl Future fn report( backend: &str, catalog: &impl Serialize, + layout: &str, counts: BTreeMap, queue_inspection: Option, payload_inspection: Option, @@ -112,6 +151,7 @@ fn report( let mut report = json!({ "schema":"durable-workflow.rust-upgrade-source-inspection.v1", "status":"inspected", "backend":backend, "catalog_visibility":"configured_role", + "source_layout":layout, "catalog_sha256":digest, "table_rows":counts, "source_mutated":false, "takeover_allowed":false, "backup_verified":false, "php_quiescence_verified":false, "stored_representation_compatibility_verified":false @@ -264,7 +304,7 @@ async fn inspect_sqlite_inner( let mut connection = open_sqlite_source(path, deadline).await?; let result = timeout_at(deadline, async { let mut tx = connection.begin().await?; - let catalog = schema::php_source_catalog(&mut tx).await?; + let (catalog, layout) = schema::php_source_catalog(&mut tx).await?; let tables: Vec = catalog .iter() .filter(|(kind, _, _, _)| kind == "table") @@ -286,6 +326,7 @@ async fn inspect_sqlite_inner( report( "sqlite", &catalog, + layout, counts, queue_inspection, payload_inspection, @@ -415,7 +456,7 @@ async fn inspect_mysql_inner( .execute(&mut connection) .await?; let mut tx = connection.begin_with("START TRANSACTION READ ONLY").await?; - let catalog = mysql::php_source_catalog(&mut tx).await?; + let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); let counts = count_tables!(tx, tables, '`'); let queue_inspection = match queues { @@ -431,7 +472,7 @@ async fn inspect_mysql_inner( }; // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible // catalog rather than joining counts to a different schema. - if mysql::php_source_catalog(&mut tx).await? != catalog { + if mysql::php_source_catalog(&mut tx).await?.0 != catalog { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_catalog_changed", @@ -441,6 +482,7 @@ async fn inspect_mysql_inner( report( backend, &catalog, + layout, counts, queue_inspection, payload_inspection, @@ -485,7 +527,7 @@ async fn inspect_postgres_inner( let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - let catalog = postgres::php_source_catalog(&mut tx).await?; + let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; let tables: Vec = catalog .iter() .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") @@ -507,6 +549,7 @@ async fn inspect_postgres_inner( report( "pgsql", &catalog, + layout, counts, queue_inspection, payload_inspection, diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index d1bdc9b02..29fac63b5 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -59,7 +59,7 @@ impl Digest { } } - fn report(self, backend: &str) -> Value { + fn report(self, backend: &str, layout: &str) -> Value { let hex: String = self .hasher .finalize() @@ -68,7 +68,8 @@ impl Digest { .collect(); json!({ "schema":"durable-workflow.rust-upgrade-source-fingerprint.v1", - "status":"fingerprinted", "backend":backend, "fingerprint":format!("sha256:{hex}"), + "status":"fingerprinted", "backend":backend, "source_layout":layout, + "fingerprint":format!("sha256:{hex}"), "table_rows":self.rows, "elapsed_ms":self.started.elapsed().as_millis() as u64, "source_mutated":false, "takeover_allowed":false, "backup_verified":false, "php_quiescence_verified":false, "stored_representation_compatibility_verified":false @@ -134,7 +135,7 @@ pub async fn fingerprint_sqlite(path: &str) -> Result { let result = timeout_at(deadline, async { // One deferred read transaction keeps one snapshot of the file. let mut tx = connection.begin().await?; - let catalog = schema::php_source_catalog(&mut tx).await?; + let (catalog, layout) = schema::php_source_catalog(&mut tx).await?; let mut digest = Digest::new("sqlite", &catalog)?; for (_, table, _, _) in catalog.iter().filter(|(kind, _, _, _)| kind == "table") { let table = identifier(table)?; @@ -167,7 +168,7 @@ pub async fn fingerprint_sqlite(path: &str) -> Result { hash_rows!(tx, digest, table, sql, columns.len()); } tx.rollback().await?; - Ok(digest.report("sqlite")) + Ok(digest.report("sqlite", layout)) }) .await .map_err(|_| expired()) @@ -192,7 +193,7 @@ pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'") .execute(&mut *tx) .await?; - let catalog = postgres::php_source_catalog(&mut tx).await?; + let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; let mut digest = Digest::new("pgsql", &catalog)?; let tables: Vec = catalog .iter() @@ -218,7 +219,7 @@ pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { hash_rows!(tx, digest, table, sql, columns.len()); } tx.rollback().await?; - Ok(digest.report("pgsql")) + Ok(digest.report("pgsql", layout)) }) .await .map_err(|_| expired()) @@ -259,7 +260,7 @@ pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { let mut tx = connection .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") .await?; - let catalog = mysql::php_source_catalog(&mut tx).await?; + let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; let mut digest = Digest::new(backend, &catalog)?; for (table, _) in &catalog { let table = identifier(table)?; @@ -291,14 +292,14 @@ pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { } // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that // changed while the rows were read. - if mysql::php_source_catalog(&mut tx).await? != catalog { + if mysql::php_source_catalog(&mut tx).await?.0 != catalog { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_catalog_changed", )); } tx.rollback().await?; - Ok(digest.report(backend)) + Ok(digest.report(backend, layout)) }) .await .map_err(|_| expired()) diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 591181e4e..3f5d79c30 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -815,3 +815,43 @@ async fn fingerprint_refuses_an_owned_or_unknown_source() { "upgrade_source_catalog_unqualified", ); } + +#[test] +fn source_layout_names_the_release_each_recorded_delta_reaches() { + let rows = |rows: &[(&str, &str)]| -> Vec<(String, String)> { + rows.iter() + .map(|(name, ddl)| ((*name).to_owned(), (*ddl).to_owned())) + .collect() + }; + let frozen = rows(&[("a", "1"), ("b", "2")]); + let deltas = [ + r#"{"remove":[["a","1"]],"add":[["a","1 changed"]]}"#, + r#"{"remove":[],"add":[["c","3"]]}"#, + ]; + let layout = |actual: &[(&str, &str)]| { + source_layout(&rows(actual), frozen.clone(), deltas, |row| row.0.clone()) + .map(|layout| Value::from(layout)) + }; + assert_eq!(layout(&[("b", "2"), ("a", "1")]).unwrap(), "frozen"); + assert_eq!(layout(&[("a", "1 changed"), ("b", "2")]).unwrap(), "php-2.8.0"); + assert_eq!( + layout(&[("c", "3"), ("a", "1 changed"), ("b", "2")]).unwrap(), + "php-2.9.7" + ); + // A later release's addition without the earlier change is no recorded layout. + reason( + layout(&[("a", "1"), ("b", "2"), ("c", "3")]), + "upgrade_source_catalog_unqualified", + ); + reason(layout(&[("a", "1")]), "upgrade_source_catalog_unqualified"); +} + +#[tokio::test] +async fn frozen_layout_is_reported_by_inspection_and_fingerprint() { + let (_directory, path) = php_layout().await; + assert_eq!(inspect_sqlite(&path).await.unwrap()["source_layout"], "frozen"); + assert_eq!( + fingerprint::fingerprint_sqlite(&path).await.unwrap()["source_layout"], + "frozen" + ); +} diff --git a/rust/upgrade/mariadb-driver.json b/rust/upgrade/mariadb-driver.json new file mode 100644 index 000000000..4f4a74900 --- /dev/null +++ b/rust/upgrade/mariadb-driver.json @@ -0,0 +1,14 @@ +{ + "remove": [ + [ + "runtime_external_payload_backup_hold", + "CREATE TABLE `runtime_external_payload_backup_hold` (\n `id` tinyint(3) unsigned NOT NULL,\n `owner` char(36) DEFAULT NULL,\n `acquired_at` timestamp NULL DEFAULT NULL,\n `expires_at` timestamp NULL DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ], + "add": [ + [ + "runtime_external_payload_backup_hold", + "CREATE TABLE `runtime_external_payload_backup_hold` (\n `id` tinyint(3) unsigned NOT NULL,\n `owner` uuid DEFAULT NULL,\n `acquired_at` timestamp NULL DEFAULT NULL,\n `expires_at` timestamp NULL DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/README.md b/rust/upgrade/php-2.9.7/README.md new file mode 100644 index 000000000..8874b00d2 --- /dev/null +++ b/rust/upgrade/php-2.9.7/README.md @@ -0,0 +1,29 @@ +# Exact PHP source-catalog variant: Server 2.9.7 + +These deltas apply to the [PHP 2.8.0 layout](../php-2.8.0/README.md), not to +the frozen layout. They come from normal bootstrap of published Server 2.9.7, +tag `2.9.7`, image index +`sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50` +(amd64 `sha256:40e969d0b6b240b835dcdb2fc531507266216ec34a83f59f561674f41e35f461`), +which pins Core (Workflow package) 2.7.2. + +The only physical change from 2.8.0 is Core's nullable JSON column +`workflow_tasks.queue_delivery_reference`, added by its +`2026_10_16_000100_add_queue_delivery_reference_to_workflow_tasks` migration. +PHP records there the database queue job that delivers a task. Server's own +migrations are unchanged from 2.8.0. + +Definitions were read from actual published-image databases: SQLite, MySQL +8.0, MariaDB 10.11 through the `mysql` connection, and PostgreSQL 17. MySQL +8.4 was checked against the same definitions. No application row values or +credentials are recorded. + +[`../mariadb-driver.json`](../mariadb-driver.json) records the one difference +when PHP uses Laravel's separate `mariadb` connection instead of `mysql`: +`runtime_external_payload_backup_hold.owner` is `uuid` rather than `char(36)`. +Inspection applies it to every MariaDB release layout and reports, for +example, `php-2.9.7+mariadb-driver`. + +Like the 2.8.0 variant, this recognizes physical layout for read-only +inventory only. It neither identifies the installed package nor qualifies +stored representations, backup, writer quiescence or takeover of that layout. diff --git a/rust/upgrade/php-2.9.7/mariadb.json b/rust/upgrade/php-2.9.7/mariadb.json new file mode 100644 index 000000000..de626b086 --- /dev/null +++ b/rust/upgrade/php-2.9.7/mariadb.json @@ -0,0 +1,14 @@ +{ + "remove": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) NOT NULL,\n `workflow_run_id` varchar(26) NOT NULL,\n `namespace` varchar(255) DEFAULT NULL,\n `task_type` varchar(255) NOT NULL,\n `status` varchar(255) NOT NULL,\n `compatibility` varchar(255) DEFAULT NULL,\n `payload` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`payload`)),\n `connection` varchar(255) DEFAULT NULL,\n `queue` varchar(255) DEFAULT NULL,\n `priority` tinyint(3) unsigned NOT NULL DEFAULT 5,\n `fairness_key` varchar(64) DEFAULT NULL,\n `fairness_weight` smallint(5) unsigned NOT NULL DEFAULT 1,\n `sticky_worker_id` varchar(255) DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int(10) unsigned NOT NULL DEFAULT 0,\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text DEFAULT NULL,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text DEFAULT NULL,\n `repair_count` int(10) unsigned NOT NULL DEFAULT 0,\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text DEFAULT NULL,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ], + "add": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) NOT NULL,\n `workflow_run_id` varchar(26) NOT NULL,\n `namespace` varchar(255) DEFAULT NULL,\n `task_type` varchar(255) NOT NULL,\n `status` varchar(255) NOT NULL,\n `compatibility` varchar(255) DEFAULT NULL,\n `payload` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`payload`)),\n `connection` varchar(255) DEFAULT NULL,\n `queue` varchar(255) DEFAULT NULL,\n `priority` tinyint(3) unsigned NOT NULL DEFAULT 5,\n `fairness_key` varchar(64) DEFAULT NULL,\n `fairness_weight` smallint(5) unsigned NOT NULL DEFAULT 1,\n `sticky_worker_id` varchar(255) DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int(10) unsigned NOT NULL DEFAULT 0,\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text DEFAULT NULL,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text DEFAULT NULL,\n `repair_count` int(10) unsigned NOT NULL DEFAULT 0,\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text DEFAULT NULL,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n `queue_delivery_reference` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`queue_delivery_reference`)),\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/mysql.json b/rust/upgrade/php-2.9.7/mysql.json new file mode 100644 index 000000000..dfd764add --- /dev/null +++ b/rust/upgrade/php-2.9.7/mysql.json @@ -0,0 +1,14 @@ +{ + "remove": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `workflow_run_id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `namespace` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `task_type` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `status` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `compatibility` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `payload` json DEFAULT NULL,\n `connection` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `queue` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `priority` tinyint unsigned NOT NULL DEFAULT '5',\n `fairness_key` varchar(64) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `fairness_weight` smallint unsigned NOT NULL DEFAULT '1',\n `sticky_worker_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int unsigned NOT NULL DEFAULT '0',\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text COLLATE utf8mb4_unicode_ci,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text COLLATE utf8mb4_unicode_ci,\n `repair_count` int unsigned NOT NULL DEFAULT '0',\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text COLLATE utf8mb4_unicode_ci,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ], + "add": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `workflow_run_id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `namespace` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `task_type` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `status` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `compatibility` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `payload` json DEFAULT NULL,\n `connection` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `queue` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `priority` tinyint unsigned NOT NULL DEFAULT '5',\n `fairness_key` varchar(64) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `fairness_weight` smallint unsigned NOT NULL DEFAULT '1',\n `sticky_worker_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int unsigned NOT NULL DEFAULT '0',\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text COLLATE utf8mb4_unicode_ci,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text COLLATE utf8mb4_unicode_ci,\n `repair_count` int unsigned NOT NULL DEFAULT '0',\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text COLLATE utf8mb4_unicode_ci,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n `queue_delivery_reference` json DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/postgres.json b/rust/upgrade/php-2.9.7/postgres.json new file mode 100644 index 000000000..ddfd2bbb6 --- /dev/null +++ b/rust/upgrade/php-2.9.7/postgres.json @@ -0,0 +1,18 @@ +{ + "remove": [], + "add": [ + [ + "column", + "workflow_tasks.32", + { + "name": "queue_delivery_reference", + "type": "json", + "default": null, + "identity": "", + "not_null": false, + "collation": null, + "generated": "" + } + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/sqlite.json b/rust/upgrade/php-2.9.7/sqlite.json new file mode 100644 index 000000000..525aa789b --- /dev/null +++ b/rust/upgrade/php-2.9.7/sqlite.json @@ -0,0 +1,18 @@ +{ + "remove": [ + [ + "table", + "workflow_tasks", + "workflow_tasks", + "CREATE TABLE \"workflow_tasks\" (\"id\" varchar not null, \"workflow_run_id\" varchar not null, \"namespace\" varchar, \"task_type\" varchar not null, \"status\" varchar not null, \"compatibility\" varchar, \"payload\" text, \"connection\" varchar, \"queue\" varchar, \"sticky_worker_id\" varchar, \"sticky_until\" datetime, \"sticky_replay_mode\" varchar, \"sticky_claimed_at\" datetime, \"available_at\" datetime, \"leased_at\" datetime, \"lease_owner\" varchar, \"lease_expires_at\" datetime, \"attempt_count\" integer not null default '0', \"last_dispatch_attempt_at\" datetime, \"last_dispatched_at\" datetime, \"last_dispatch_error\" text, \"last_claim_failed_at\" datetime, \"last_claim_error\" text, \"repair_count\" integer not null default '0', \"repair_available_at\" datetime, \"last_error\" text, \"created_at\" datetime, \"updated_at\" datetime, \"priority\" integer not null default '5', \"fairness_key\" varchar, \"fairness_weight\" integer not null default '1', primary key (\"id\"))" + ] + ], + "add": [ + [ + "table", + "workflow_tasks", + "workflow_tasks", + "CREATE TABLE \"workflow_tasks\" (\"id\" varchar not null, \"workflow_run_id\" varchar not null, \"namespace\" varchar, \"task_type\" varchar not null, \"status\" varchar not null, \"compatibility\" varchar, \"payload\" text, \"connection\" varchar, \"queue\" varchar, \"sticky_worker_id\" varchar, \"sticky_until\" datetime, \"sticky_replay_mode\" varchar, \"sticky_claimed_at\" datetime, \"available_at\" datetime, \"leased_at\" datetime, \"lease_owner\" varchar, \"lease_expires_at\" datetime, \"attempt_count\" integer not null default '0', \"last_dispatch_attempt_at\" datetime, \"last_dispatched_at\" datetime, \"last_dispatch_error\" text, \"last_claim_failed_at\" datetime, \"last_claim_error\" text, \"repair_count\" integer not null default '0', \"repair_available_at\" datetime, \"last_error\" text, \"created_at\" datetime, \"updated_at\" datetime, \"priority\" integer not null default '5', \"fairness_key\" varchar, \"fairness_weight\" integer not null default '1', \"queue_delivery_reference\" text, primary key (\"id\"))" + ] + ] +} diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index 400f04641..f4952c161 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -4,11 +4,15 @@ set -euo pipefail test "$#" = 1 || exit 2 backend=$1 test -n "${RESOURCE_SCOPE:?Set the owning isolated network}" -evidence=parity-evidence/upgrade-source -state=parity-upgrade-source-state +label=${DW_PARITY_UPGRADE_LABEL:-} +[[ $label =~ ^[a-z0-9]*$ ]] || exit 2 +evidence=parity-evidence/upgrade-source${label:+-$label} +state=parity-upgrade-source-state${label:+-$label} test -f "$evidence/owner.txt" || exit 0 test "$(cat "$evidence/owner.txt")" = "$RESOURCE_SCOPE" || exit 2 -resource="$RESOURCE_SCOPE-upgrade" +resource="$RESOURCE_SCOPE-upgrade${label:+-$label}" +source=upgrade_source_ref${label:+_$label} +restored=upgrade_restore_ref${label:+_$label} image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 for role in author control inspect php; do @@ -23,12 +27,12 @@ case "$backend" in mysql) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ - -e 'DROP DATABASE IF EXISTS upgrade_source_ref' \ + -e "DROP DATABASE IF EXISTS $source" \ > "$evidence/database-cleanup.txt" 2>&1 ;; pgsql) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ - -c 'DROP DATABASE IF EXISTS upgrade_source_ref' \ + -c "DROP DATABASE IF EXISTS $source" \ > "$evidence/database-cleanup.txt" 2>&1 ;; sqlite) ;; *) exit 2 ;; @@ -39,12 +43,12 @@ case "$backend" in mysql) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ - -e 'DROP DATABASE IF EXISTS upgrade_restore_ref' \ + -e "DROP DATABASE IF EXISTS $restored" \ >> "$evidence/database-cleanup.txt" 2>&1 ;; pgsql) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ - -c 'DROP DATABASE IF EXISTS upgrade_restore_ref' \ + -c "DROP DATABASE IF EXISTS $restored" \ >> "$evidence/database-cleanup.txt" 2>&1 ;; *) ;; esac diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 070277491..e939fb647 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -11,9 +11,15 @@ docker network inspect "$RESOURCE_SCOPE" >/dev/null image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 rust_image=mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 -resource="$RESOURCE_SCOPE-upgrade" -state=parity-upgrade-source-state -evidence=parity-evidence/upgrade-source +# A labeled run observes another release beside the default one. Its state, +# evidence, containers and databases are separate, and it names the layout +# inspection must report for that release. +label=${DW_PARITY_UPGRADE_LABEL:-} +[[ $label =~ ^[a-z0-9]*$ ]] || exit 2 +layout=${DW_PARITY_UPGRADE_EXPECTED_LAYOUT:-php-2.8.0} +resource="$RESOURCE_SCOPE-upgrade${label:+-$label}" +state=parity-upgrade-source-state${label:+-$label} +evidence=parity-evidence/upgrade-source${label:+-$label} test -f rust-target/debug/durable-workflow-server git diff --quiet git diff --cached --quiet @@ -26,7 +32,7 @@ mkdir -p "$state" "$evidence" chmod a+rwx "$state" "$evidence" printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/owner.txt" trap 'printf "%s\n" "$?" > "$evidence/profile-status.txt"' EXIT -database=upgrade_source_ref +database=upgrade_source_ref${label:+_$label} password=${DW_PARITY_UPGRADE_DB_PASSWORD:-parity-disposable-password} case "$backend" in mysql) host=${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}; admin=(-e DB_CONNECTION=mysql -e DB_HOST="$host" -e DB_USERNAME=root -e DB_PASSWORD="$password") ;; @@ -37,7 +43,7 @@ esac if test "$backend" = mysql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ -h "$host" -uroot -p"$password" \ - -e 'CREATE DATABASE upgrade_source_ref CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' > "$evidence/create.txt" 2>&1 + -e "CREATE DATABASE $database CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" > "$evidence/create.txt" 2>&1 elif test "$backend" = pgsql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ --entrypoint createdb "$POSTGRES_IMAGE" -h "$host" -U parity "$database" > "$evidence/create.txt" 2>&1 @@ -110,7 +116,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e '.status=="inspected" and (.table_rows|length)==50 and .table_rows.workflow_runs==2 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==2 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -141,7 +147,7 @@ fingerprint() { --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-fingerprint } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" -jq -e '.status=="fingerprinted" and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) +jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) and (.table_rows|length)==50 and .table_rows.workflow_runs==2 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null @@ -151,7 +157,7 @@ test "$(jq -r .fingerprint "$evidence/fingerprint-repeat.json")" = "$source_fing # Back up the stopped source and restore it into a separate database, as the # backup procedure does. The restored copy must fingerprint equal, and one # changed row in it must change the fingerprint. -restored=upgrade_restore_ref +restored=upgrade_restore_ref${label:+_$label} case "$backend" in mysql) mysql_admin() { @@ -225,7 +231,7 @@ jq -n --arg revision "$revision" --arg native "$native_tree" \ --slurpfile fingerprint "$evidence/fingerprint.json" \ '{outcome:"pass",scope:"read_only_source_inventory_and_refusals",backend:$backend, runner_revision:$revision,native_tree:$native,native_binary_sha256:$binary,php_image:$image, - original_runs:2,source_state_unchanged_by_inspection:true, + original_runs:2,source_state_unchanged_by_inspection:true,source_layout:$inventory[0].source_layout, source_fingerprint:$fingerprint[0].fingerprint,restored_backup_fingerprint_equal:true, changed_restored_row_changes_fingerprint:true, catalog_sha256:$inventory[0].catalog_sha256,table_count:($inventory[0].table_rows|length), From def78ff7e61000a15419b353c34d64aedbe1c3e3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:18:22 +0000 Subject: [PATCH 05/20] Present stored history payloads the way PHP does Published PHP stores encoded values in history payloads as bare strings beside their codec and wraps them into {codec, blob} envelopes when it reads them (ExternalPayloadEnvelopeService::historyPayload). It also lifts the command's principal to each public history event and keeps only the neutral fields of an activity failure. Native stores the envelope itself, so history written by PHP read differently through native: command payloads and activity arguments came back as strings, and principals were missing. Native now applies the same rules to every history event it returns, in the public history and in worker task history. Only bare strings are wrapped, so native's own envelopes are unchanged; an unsupported codec or a PHP external reference is left as stored. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/history_presentation.rs | 182 +++++++++++++++++++++++ rust/src/runtime/mod.rs | 1 + rust/src/runtime/store.rs | 8 +- 3 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 rust/src/runtime/history_presentation.rs diff --git a/rust/src/runtime/history_presentation.rs b/rust/src/runtime/history_presentation.rs new file mode 100644 index 000000000..e46d0abac --- /dev/null +++ b/rust/src/runtime/history_presentation.rs @@ -0,0 +1,182 @@ +//! Presents stored history payloads the way published PHP Server does +//! (`ExternalPayloadEnvelopeService::historyPayload` and +//! `HistoryController::eventPrincipal`). PHP stores encoded values as bare +//! strings beside their codec and wraps them while reading; native stores +//! the envelope itself. With the same rules, history written by PHP before a +//! takeover reads as PHP returned it, and native's own envelopes pass through +//! unchanged. +use serde_json::{Map, Value, json}; + +/// Kept from a PHP activity failure; PHP redacts its runtime internals. +const NEUTRAL_EXCEPTION_KEYS: [&str; 12] = [ + "type", + "message", + "code", + "kind", + "classification", + "retryable", + "non_retryable", + "timeout_type", + "cancelled", + "malformed_output", + "details", + "details_payload_codec", +]; +const DIAGNOSTIC_KEYS: [&str; 2] = ["diagnostics", "runtime_diagnostics"]; +/// PHP's stored provider reference, which only PHP's external payload store +/// can resolve. Takeover refuses sources that hold any, so it is left as is. +const PHP_EXTERNAL_REFERENCE: &str = "dw-external-payload:v1:"; + +pub(super) fn payload(mut payload: Value, event_type: &str) -> Value { + let Value::Object(fields) = &mut payload else { + return payload; + }; + let codec = codec(fields, "payload_codec", Some("avro")); + for field in ["arguments", "result", "output"] { + wrap(fields, field, codec.as_deref()); + } + if let Some(Value::Object(command)) = fields.get_mut("command") { + let codec = self::codec(command, "payload_codec", codec.as_deref()); + wrap(command, "payload", codec.as_deref()); + } + if let Some(Value::Object(activity)) = fields.get_mut("activity") { + let codec = self::codec(activity, "payload_codec", codec.as_deref()); + for field in ["arguments", "result"] { + wrap(activity, field, codec.as_deref()); + } + } + let activity_failure = event_type == "ActivityFailed"; + let opaque_legacy = activity_failure + && fields + .get("migration_projection") + .and_then(|projection| projection.get("source_record")) + == Some(&json!("workflow_exception")); + if let Some(Value::Object(exception)) = fields.get_mut("exception") { + let codec = self::codec(exception, "details_payload_codec", codec.as_deref()); + wrap(exception, "details", codec.as_deref()); + if activity_failure { + let mut neutral = Map::new(); + for key in NEUTRAL_EXCEPTION_KEYS { + if let Some(value) = exception.get(key) { + neutral.insert(key.to_owned(), value.clone()); + } + } + for key in DIAGNOSTIC_KEYS { + if let Some(value @ (Value::Object(_) | Value::Array(_))) = exception.get(key) { + neutral.insert(key.to_owned(), value.clone()); + } + } + if opaque_legacy + && let Some(value) = exception.get("value").filter(|value| opaque_descriptor(value)) + { + neutral.insert("value".to_owned(), value.clone()); + } + *exception = neutral; + } + } + payload +} + +/// The actor PHP surfaces at the top of a public history event: the +/// principal recorded on the event's command, or null. +pub(super) fn principal(payload: &Value) -> Value { + let Some(command) = payload.get("command").filter(|command| command.is_object()) else { + return Value::Null; + }; + let mut principal = Map::new(); + for (key, field) in [ + ("type", "principal_type"), + ("id", "principal_id"), + ("label", "principal_label"), + ] { + if let Some(value) = command[field].as_str().filter(|value| !value.is_empty()) { + principal.insert(key.to_owned(), json!(value)); + } + } + if principal.is_empty() { + Value::Null + } else { + Value::Object(principal) + } +} + +/// The section's own codec, or the enclosing one. `None` means a codec PHP +/// rejects, so nothing in that section is wrapped. +fn codec(section: &Map, field: &str, fallback: Option<&str>) -> Option { + match section.get(field).and_then(Value::as_str).filter(|codec| !codec.is_empty()) { + Some("avro") => Some("avro".to_owned()), + Some(_) => None, + None => fallback.map(str::to_owned), + } +} + +fn wrap(section: &mut Map, field: &str, codec: Option<&str>) { + let Some(codec) = codec else { + return; + }; + if let Some(Value::String(blob)) = section.get(field) + && !blob.starts_with(PHP_EXTERNAL_REFERENCE) + { + let envelope = json!({"codec": codec, "blob": blob}); + section.insert(field.to_owned(), envelope); + } +} + +fn opaque_descriptor(value: &Value) -> bool { + value["available"].is_boolean() + && value["encoding"] == "php-serialize" + && value.get("value").is_some_and(|inner| inner.is_string() || inner.is_null()) + && value["portable"] == false + && value["unsupported_reason"] == "legacy_php_serialization_not_portable" + && value["remediation"] + == "Treat the preserved value as opaque; export decoded JSON from the source application if portable payload access is required." +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn php_stored_strings_become_envelopes_and_native_envelopes_stay() { + let stored = json!({"payload_codec":"avro","arguments":"AAA=","output":null, + "command":{"payload_codec":"avro","payload":"BBB=","principal_type":"auth:token", + "principal_id":"legacy-token","principal_label":"Admin"}, + "activity":{"arguments":"CCC=","result":{"codec":"avro","blob":"DDD="}}}); + let presented = payload(stored.clone(), "ActivityScheduled"); + assert_eq!(presented["arguments"], json!({"codec":"avro","blob":"AAA="})); + assert_eq!(presented["output"], Value::Null); + assert_eq!(presented["command"]["payload"], json!({"codec":"avro","blob":"BBB="})); + assert_eq!(presented["activity"]["arguments"], json!({"codec":"avro","blob":"CCC="})); + assert_eq!(presented["activity"]["result"], json!({"codec":"avro","blob":"DDD="})); + assert_eq!(payload(presented.clone(), "ActivityScheduled"), presented); + assert_eq!( + principal(&stored), + json!({"type":"auth:token","id":"legacy-token","label":"Admin"}) + ); + assert_eq!(principal(&json!({"command":{"type":"start"}})), Value::Null); + assert_eq!(principal(&json!({})), Value::Null); + } + + #[test] + fn unsupported_codecs_and_php_references_are_left_unchanged() { + let stored = json!({"payload_codec":"json","result":"plain", + "command":{"payload":"dw-external-payload:v1:eyJ9"}}); + let presented = payload(stored.clone(), "ActivityCompleted"); + assert_eq!(presented["result"], "plain"); + assert_eq!(presented["command"]["payload"], "dw-external-payload:v1:eyJ9"); + } + + #[test] + fn activity_failures_keep_only_neutral_fields() { + let stored = json!({"exception":{"type":"RuntimeException","message":"boom","file":"/app/x.php", + "line":3,"trace":["frame"],"details":"EEE=","diagnostics":{"hint":"retry"},"runtime_diagnostics":"text"}}); + let presented = payload(stored.clone(), "ActivityFailed"); + assert_eq!( + presented["exception"], + json!({"type":"RuntimeException","message":"boom","details":{"codec":"avro","blob":"EEE="}, + "diagnostics":{"hint":"retry"}}) + ); + // Other failures keep every field, as PHP presents them. + assert_eq!(payload(stored, "WorkflowFailed")["exception"]["file"], "/app/x.php"); + } +} diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 0d051c048..6eb8fb731 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -28,6 +28,7 @@ mod external_transport; mod external_wait_summary; mod failure_commands; mod history_budget; +mod history_presentation; mod http; mod memo_json; mod memo_limits; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f95..0ce48aabc 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -387,7 +387,8 @@ where .into_iter() .map(|e| { json!({"sequence": e["sequence"], - "event_type": e["event_type"], "timestamp": e["recorded_at"], "payload": e["payload"]}) + "event_type": e["event_type"], "timestamp": e["recorded_at"], + "principal": super::history_presentation::principal(&e["payload"]), "payload": e["payload"]}) }) .collect(); Ok( @@ -2541,9 +2542,12 @@ where } fn event(row: DB::Row) -> Result { + let event_type = DB::string(&row, "event_type")?; + let payload = + super::history_presentation::payload(DB::document_row(&row, "payload")?, &event_type); Ok( json!({"id": DB::string(&row,"id")?, "sequence": DB::number(&row,"sequence")?, "event_type": DB::string(&row,"event_type")?, - "payload": DB::document_row(&row,"payload")?, + "payload": payload, "recorded_at": DB::instant(&row,"recorded_at")?.to_rfc3339_opts(chrono::SecondsFormat::Micros, true), "workflow_task_id": DB::optional_string(&row,"workflow_task_id")?, "workflow_command_id": DB::optional_string(&row,"workflow_command_id")?}), ) From 834f74ab91dfe1b038ec8a36267c0bee39c6f44c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:18:22 +0000 Subject: [PATCH 06/20] Take over a stopped PHP SQLite database backup-first upgrade-takeover implements the SQLite part of the takeover design (#517). In one BEGIN EXCLUSIVE transaction it: - runs the inventory, queue and payload inspections and requires their qualified results, refusing external payloads and service calls; - recomputes the fingerprint and refuses unless it equals the operator's verified backup (DW_UPGRADE_BACKUP_FINGERPRINT); - adds the native relations, converts the recorded PHP release layout (PHP's registration incarnations and tokens copied into native's, PHP-only columns and table dropped), records the native migration and runs native's own startup verification. A refusal or interruption leaves the PHP file unchanged; a committed takeover is a database native opens and PHP 2.5.14+ refuses. The SQLite upgrade source cohort now takes over a copy of the stopped published-PHP source, starts native on it and resumes the originals: the PHP-leased activity completes with its original attempt, the PHP-completed workflow task is refused as stale, the PHP-scheduled timer fires at its original deadline, both workflows complete on their original history, and PHP refuses the upgraded file. Co-Authored-By: Claude Opus 5.5 --- rust/README.md | 29 ++- rust/src/main.rs | 41 ++++ rust/src/runtime/schema.rs | 36 +++ rust/src/runtime/upgrade.rs | 76 +++--- rust/src/runtime/upgrade/fingerprint.rs | 75 +++--- rust/src/runtime/upgrade/takeover.rs | 226 ++++++++++++++++++ rust/src/runtime/upgrade_tests.rs | 134 +++++++++++ .../server-parity/upgrade-source-author.php | 5 +- .../server-parity/upgrade-source-cleanup.sh | 2 +- .../server-parity/upgrade-source-profile.sh | 50 +++- .../server-parity/upgrade-takeover-resume.php | 113 +++++++++ 11 files changed, 719 insertions(+), 68 deletions(-) create mode 100644 rust/src/runtime/upgrade/takeover.rs create mode 100644 scripts/conformance/server-parity/upgrade-takeover-resume.php diff --git a/rust/README.md b/rust/README.md index d94d2400e..1e4aa7dbb 100644 --- a/rust/README.md +++ b/rust/README.md @@ -412,10 +412,31 @@ version-1/version-2/version-3 development schemas are refused without conversion needed old development data separately; this unpublished slice supplies no older-development converter. Use a new isolated file for version 4. -This refusal is not the upgrade mechanism. Backup-first conversion of PHP's -stored representations and interrupted takeover remain required on all three -database families. The PHP ownership fence is merged in source, while the frozen -published PHP image still lacks it. **Never connect PHP to a Rust development database.** +This refusal is not the upgrade mechanism. A stopped PHP SQLite database can +be taken over with the development `upgrade-takeover` command, following the +[takeover design](../docs/rust-server-upgrade-takeover.md): + +1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and + stop every PHP role. +2. Run `upgrade-fingerprint` on the source, mounted read-only. Back the file + up, restore the backup to a separate path, and fingerprint that too. The + digests must be equal. +3. Run `upgrade-takeover` with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that + digest and the original queue settings (`DW_RUST_UPGRADE_QUEUE_*`, as for + `upgrade-queue-inspect`). + +Inspection, the fingerprint check, the conversion of the recorded PHP release +layout, the native relations, the migration ledger and native's own startup +verification all happen in one exclusive transaction. A refusal or an +interruption leaves the PHP file unchanged. The command refuses: + +- a digest that differs, so any write after the backup stops it; +- unrecognized queued jobs, external payloads and service calls; +- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7, or a database + native already owns. + +PostgreSQL and MySQL/MariaDB takeover, nested payload columns and multi-node +qualification remain open. **Never connect PHP to a Rust development database.** Named namespace isolation and configured worker/operator/admin roles have representative shared qualification. Principal/custom-provider credentials and the full authorization contract remain open. Broader timer/signal semantics, failure/retry policy, cancellation and the rest of the API also diff --git a/rust/src/main.rs b/rust/src/main.rs index 10b77cafb..79365e07a 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -133,11 +133,52 @@ async fn main() -> Result<(), Box> { | "upgrade-queue-inspect" | "upgrade-payload-inspect" | "upgrade-fingerprint" + | "upgrade-takeover" ) { return Err("unknown development command".into()); } let backend = env::var("DB_CONNECTION").unwrap_or_default(); + if command == "upgrade-takeover" { + use durable_workflow_server::runtime::upgrade::{ + queue_binding::QueueSourceConfig, takeover, + }; + if backend != "sqlite" { + eprintln!("upgrade_takeover_backend_unqualified"); + return Err("upgrade takeover refused".into()); + } + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + let fingerprint = env::var("DW_UPGRADE_BACKUP_FINGERPRINT") + .map_err(|_| "DW_UPGRADE_BACKUP_FINGERPRINT is required")?; + // The original Laravel queue transport, as for upgrade-queue-inspect. + let (connection, default_connection, default_queue) = ( + env::var("DW_RUST_UPGRADE_QUEUE_CONNECTION") + .map_err(|_| "original queue connection is required")?, + env::var("DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION") + .map_err(|_| "original default queue connection is required")?, + env::var("DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE") + .map_err(|_| "original default transport queue is required")?, + ); + let queues = QueueSourceConfig { + connection: &connection, + default_connection: &default_connection, + default_queue: &default_queue, + }; + return match takeover::take_over_sqlite(&path, &fingerprint, &queues).await { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_takeover_unavailable", + }; + eprintln!("{reason}"); + Err("upgrade takeover refused".into()) + } + }; + } if command == "upgrade-fingerprint" { use durable_workflow_server::runtime::upgrade::fingerprint; let result = match backend.as_str() { diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 9085600a2..6b6ebaf0e 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -31,6 +31,42 @@ fn migrator() -> &'static Migrator { }) } +/// The native relations, indexes and ownership marker that a fresh +/// bootstrap adds after the frozen PHP layout. Takeover applies them to a +/// converted PHP database instead of running the whole migration. +pub(super) fn native_relations_sql() -> Result { + let frozen = include_str!("../../migrations/sqlite/0003_full_schema.sql"); + let start = frozen + .find("CREATE TABLE dw_server_schema ") + .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; + Ok(format!( + "{}\n{}", + &frozen[start..], + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") + )) +} + +/// Records the native migration as applied in the caller's transaction, for a +/// takeover whose schema came from conversion. `verify` then checks the +/// marker, this ledger and the complete catalog as for any native database. +pub(super) async fn record_migration(connection: &mut SqliteConnection) -> Result<()> { + use sqlx::migrate::Migrate; + connection + .ensure_migrations_table("_sqlx_migrations") + .await?; + let migration = migrator() + .iter() + .next() + .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_migration_history_mismatch"))?; + sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *connection) + .await?; + Ok(()) +} + type Catalog = Vec<(String, String, String, String)>; static EXPECTED: OnceCell = OnceCell::const_new(); diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index aef6b3be5..656637f12 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -1,6 +1,7 @@ //! Read-only PHP source inventory. This command never grants takeover authority. pub mod external_reference; pub mod fingerprint; +pub mod takeover; mod payload_scan; pub mod queue; pub mod queue_binding; @@ -304,33 +305,9 @@ async fn inspect_sqlite_inner( let mut connection = open_sqlite_source(path, deadline).await?; let result = timeout_at(deadline, async { let mut tx = connection.begin().await?; - let (catalog, layout) = schema::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, _, _)| kind == "table") - .map(|(_, name, _, _)| name.clone()) - .collect(); - let counts = count_tables!(tx, tables, '"'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; + let report = sqlite_inventory(&mut tx, queues, payloads).await; tx.rollback().await?; - report( - "sqlite", - &catalog, - layout, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await .map_err(|_| expired()) @@ -339,8 +316,52 @@ async fn inspect_sqlite_inner( result } +/// The SQLite inspection report, read inside the caller's transaction. +pub(super) async fn sqlite_inventory( + tx: &mut SqliteConnection, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, +) -> Result { + let (catalog, layout) = schema::php_source_catalog(&mut *tx).await?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, _, _)| kind == "table") + .map(|(_, name, _, _)| name.clone()) + .collect(); + let counts = count_tables!(tx, tables, '"'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + report( + "sqlite", + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) +} + /// Opens a qualified rollback-journal source file read-only. pub(super) async fn open_sqlite_source(path: &str, deadline: Instant) -> Result { + open_sqlite_file(path, deadline, false).await +} + +/// Opens a qualified rollback-journal source file, writable only for takeover. +/// It keeps the rollback journal: native enables WAL itself after takeover. +pub(super) async fn open_sqlite_file( + path: &str, + deadline: Instant, + writable: bool, +) -> Result { if path.is_empty() || path.contains(['?', ':']) { return Err(refuse( StatusCode::BAD_REQUEST, @@ -396,7 +417,8 @@ pub(super) async fn open_sqlite_source(path: &str, deadline: Instant) -> Result< SqliteConnection::connect_with( &SqliteConnectOptions::new() .filename(path) - .read_only(true) + .read_only(!writable) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete) .create_if_missing(false) .foreign_keys(false), ), diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index 29fac63b5..0ee534766 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -16,7 +16,8 @@ use serde::Serialize; use serde_json::{Value, json}; use sha2::{Digest as _, Sha256}; use sqlx::{ - AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, mysql::MySqlConnectOptions, + AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, SqliteConnection, + mysql::MySqlConnectOptions, postgres::PgConnectOptions, }; use std::{collections::BTreeMap, time::Duration}; @@ -135,40 +136,9 @@ pub async fn fingerprint_sqlite(path: &str) -> Result { let result = timeout_at(deadline, async { // One deferred read transaction keeps one snapshot of the file. let mut tx = connection.begin().await?; - let (catalog, layout) = schema::php_source_catalog(&mut tx).await?; - let mut digest = Digest::new("sqlite", &catalog)?; - for (_, table, _, _) in catalog.iter().filter(|(kind, _, _, _)| kind == "table") { - let table = identifier(table)?; - let columns: Vec = - sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) ORDER BY cid") - .bind(table) - .fetch_all(&mut *tx) - .await?; - let key: Vec = - sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) WHERE pk>0 ORDER BY pk") - .bind(table) - .fetch_all(&mut *tx) - .await?; - digest.table(table, &columns); - // quote() keeps the storage class: integers, text and blobs render - // exactly. Reals use 17 significant digits, which round-trip. - let select = columns - .iter() - .map(|column| { - identifier(column).map(|c| { - format!("CASE typeof(\"{c}\") WHEN 'real' THEN printf('%!.17g',\"{c}\") ELSE quote(\"{c}\") END") - }) - }) - .collect::>>()? - .join(","); - let sql = format!( - "SELECT {select} FROM \"{table}\" ORDER BY {}", - order(&key, columns.len(), '"')? - ); - hash_rows!(tx, digest, table, sql, columns.len()); - } + let report = sqlite_digest(&mut tx).await; tx.rollback().await?; - Ok(digest.report("sqlite", layout)) + report }) .await .map_err(|_| expired()) @@ -177,6 +147,43 @@ pub async fn fingerprint_sqlite(path: &str) -> Result { result } +/// The SQLite fingerprint report, read inside the caller's transaction. +pub(super) async fn sqlite_digest(tx: &mut SqliteConnection) -> Result { + let (catalog, layout) = schema::php_source_catalog(&mut *tx).await?; + let mut digest = Digest::new("sqlite", &catalog)?; + for (_, table, _, _) in catalog.iter().filter(|(kind, _, _, _)| kind == "table") { + let table = identifier(table)?; + let columns: Vec = + sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) ORDER BY cid") + .bind(table) + .fetch_all(&mut *tx) + .await?; + let key: Vec = + sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) WHERE pk>0 ORDER BY pk") + .bind(table) + .fetch_all(&mut *tx) + .await?; + digest.table(table, &columns); + // quote() keeps the storage class: integers, text and blobs render + // exactly. Reals use 17 significant digits, which round-trip. + let select = columns + .iter() + .map(|column| { + identifier(column).map(|c| { + format!("CASE typeof(\"{c}\") WHEN 'real' THEN printf('%!.17g',\"{c}\") ELSE quote(\"{c}\") END") + }) + }) + .collect::>>()? + .join(","); + let sql = format!( + "SELECT {select} FROM \"{table}\" ORDER BY {}", + order(&key, columns.len(), '"')? + ); + hash_rows!(tx, digest, table, sql, columns.len()); + } + Ok(digest.report("sqlite", layout)) +} + pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { let deadline = Instant::now() + BUDGET; let mut connection = timeout_at( diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs new file mode 100644 index 000000000..a0e71b9b9 --- /dev/null +++ b/rust/src/runtime/upgrade/takeover.rs @@ -0,0 +1,226 @@ +//! Backup-first, one-way takeover of a stopped PHP SQLite database, following +//! docs/rust-server-upgrade-takeover.md. +//! +//! Everything happens in one `BEGIN EXCLUSIVE` transaction: the inspections, +//! the fingerprint check against the operator's verified backup, the layout +//! conversion, the native relations, the migration ledger and native's own +//! startup verification. An interrupted or refused takeover therefore leaves +//! the PHP database exactly as it was, and a committed one is a native +//! database that PHP 2.5.14 and later refuse to open. +use super::{ + Result, close, expired, fingerprint, open_sqlite_file, queue_binding::QueueSourceConfig, + queue_config, refuse, schema, sqlite_inventory, +}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{AssertSqlSafe, Connection, SqliteConnection}; +use std::time::Duration; +use tokio::time::{Instant, timeout_at}; + +/// Takeover reads every row twice (inspection and fingerprint). +const BUDGET: Duration = Duration::from_secs(3600); + +pub async fn take_over_sqlite( + path: &str, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + queue_config(Some(queues))?; + if !backup_fingerprint.starts_with("sha256:") || backup_fingerprint.len() != 71 { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_backup_fingerprint_required", + )); + } + let deadline = Instant::now() + BUDGET; + let mut connection = open_sqlite_file(path, deadline, true).await?; + let result = timeout_at(deadline, async { + let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; + match take_over(&mut tx, backup_fingerprint, queues).await { + Ok(report) => { + tx.commit().await?; + Ok(report) + } + Err(error) => { + tx.rollback().await?; + Err(error) + } + } + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + close(connection.close()).await?; + result +} + +async fn take_over( + tx: &mut SqliteConnection, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + let inventory = sqlite_inventory(&mut *tx, Some(queues), true).await?; + qualified(&inventory)?; + // The same digest the operator compared with the restored backup. Any + // write after that backup changes it, so nothing below can run. + let digest = fingerprint::sqlite_digest(&mut *tx).await?; + if digest["fingerprint"] != backup_fingerprint { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + let layout = inventory["source_layout"] + .as_str() + .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified"))?; + sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + .execute(&mut *tx) + .await?; + let converted = convert(&mut *tx, layout).await?; + schema::record_migration(&mut *tx).await?; + // Exactly the check native startup makes: marker, ledger and catalog. + schema::verify(&mut *tx).await?; + Ok(json!({ + "schema":"durable-workflow.rust-upgrade-takeover.v1", + "status":"taken_over", "backend":"sqlite", "source_layout":layout, + "fingerprint":digest["fingerprint"], "table_rows":inventory["table_rows"], + "converted":converted, "backup_fingerprint_matched":true, + // What the inspections cannot yet establish stays explicit. + "nested_payloads_verified":false, "other_root_payload_columns_verified":false, + "stored_representation_compatibility_verified":false + })) +} + +/// Takeover needs every inspection result it relies on, and refuses what +/// native cannot represent yet: external payloads and service calls. +fn qualified(inventory: &Value) -> Result<()> { + if inventory["queue_inspection"]["status"] != "known_subset_bound" { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_queue_binding_unqualified", + )); + } + let payloads = &inventory["payload_inspection"]; + if payloads["references_metadata_supported"] != true || payloads["inline_values_verified"] != true + { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_payload_metadata_unqualified", + )); + } + let total = |field: &str| { + payloads["columns"].as_array().map_or(0, |columns| { + columns + .iter() + .map(|column| column["counts"][field].as_i64().unwrap_or(0)) + .sum::() + }) + }; + if total("php_v1_references") != 0 + || total("unqualified_references") != 0 + || total("other_unvalidated") != 0 + || inventory["table_rows"]["runtime_external_payloads"] != 0 + { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_external_payloads_unsupported", + )); + } + Ok(()) +} + +/// Converts a recorded PHP release layout to native's (the design's recipes). +async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { + let (incarnation_table, token_column) = match layout { + "frozen" => (false, false), + "php-2.8.0" | "php-2.9.7" => (true, true), + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_layout_unconverted", + )); + } + }; + let calls: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM workflow_service_calls") + .fetch_one(&mut *tx) + .await?; + if calls != 0 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_service_calls_unsupported", + )); + } + let now = super::super::store::now() + .format("%Y-%m-%d %H:%M:%S%.6f") + .to_string(); + let mut copied = 0; + if incarnation_table { + // Same statuses as native. Native keeps the recovered count only on + // deregistration and a finish time on every retired incarnation. + copied = sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$1),COALESCE(updated_at,created_at,$1) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") + .bind(&now) + .execute(&mut *tx) + .await? + .rows_affected(); + let total: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM workflow_worker_registration_incarnations") + .fetch_one(&mut *tx) + .await?; + if copied != total as u64 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_incarnation_unqualified", + )); + } + } + // Every active registration gets the authority native expects. A PHP + // 2.8.0+ worker keeps the token it was given; an older one gets a new token. + let missing: Vec<(String, String, Option)> = sqlx::query_as(AssertSqlSafe(format!( + "SELECT r.namespace,r.worker_id,{} FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id", + if token_column { "r.registration_token" } else { "NULL" } + ))) + .fetch_all(&mut *tx) + .await?; + for (namespace, worker_id, token) in &missing { + let token = match token { + Some(token) => token.clone(), + None => super::super::worker_incarnations::token().await?, + }; + sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") + .bind(token) + .bind(namespace) + .bind(worker_id) + .bind(&now) + .bind(&now) + .execute(&mut *tx) + .await?; + } + if incarnation_table { + sqlx::raw_sql( + "DROP TABLE workflow_worker_registration_incarnations; \ + ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ + DROP INDEX wf_service_calls_deadline_idx; \ + ALTER TABLE workflow_service_calls DROP COLUMN call_deadline_at;", + ) + .execute(&mut *tx) + .await?; + } + let delivery_references = if layout == "php-2.9.7" { + // PHP's own queue delivery bookkeeping; native does not use the jobs. + let references: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", + ) + .fetch_one(&mut *tx) + .await?; + sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .execute(&mut *tx) + .await?; + references + } else { + 0 + }; + Ok(json!({ + "incarnations_copied":copied, "incarnations_created":missing.len(), + "queue_delivery_references_removed":delivery_references + })) +} diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 3f5d79c30..d45f60094 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -855,3 +855,137 @@ async fn frozen_layout_is_reported_by_inspection_and_fingerprint() { "frozen" ); } + +/// The frozen test layout moved to PHP 2.9.7's, by the same column, table and +/// index changes PHP's migrations make. Inspection must call it php-2.9.7. +async fn php297_layout() -> (tempfile::TempDir, String) { + let (directory, path) = php_layout().await; + for statement in [ + // The frozen test layout's private job is not a recognized delivery. + "DELETE FROM jobs", + r#"ALTER TABLE "workflow_worker_registrations" ADD COLUMN "registration_token" varchar"#, + r#"CREATE TABLE "workflow_worker_registration_incarnations" ("token" varchar not null, "namespace" varchar not null, "worker_id" varchar not null, "status" varchar not null, "recovered_workflow_task_count" integer not null default '0', "finished_at" datetime, "created_at" datetime, "updated_at" datetime, primary key ("token"))"#, + r#"CREATE INDEX "workflow_worker_registration_incarnations_finished_at_index" on "workflow_worker_registration_incarnations" ("finished_at")"#, + r#"CREATE INDEX "dw_worker_incarnation_scope_idx" on "workflow_worker_registration_incarnations" ("namespace", "worker_id")"#, + r#"ALTER TABLE "workflow_service_calls" ADD COLUMN "call_deadline_at" datetime"#, + r#"CREATE INDEX "wf_service_calls_deadline_idx" on "workflow_service_calls" ("status", "call_deadline_at")"#, + r#"ALTER TABLE "workflow_tasks" ADD COLUMN "queue_delivery_reference" text"#, + ] { + mutate(&path, statement).await; + } + (directory, path) +} + +async fn backup_fingerprint(path: &str) -> String { + fingerprint::fingerprint_sqlite(path).await.unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() +} + +#[tokio::test] +async fn takeover_converts_a_php_297_database_that_native_then_opens() { + let (_directory, path) = php297_layout().await; + assert_eq!(inspect_sqlite(&path).await.unwrap()["source_layout"], "php-2.9.7"); + for statement in [ + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00')", + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00')", + "INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00.000000','2026-10-20 09:00:00.000000')", + "INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00.000000','2026-10-20 07:00:00.000000','2026-10-20 08:00:00.000000')", + "INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00.000000','2026-10-20 07:00:00.000000','2026-10-20 08:30:00.000000')", + ] { + mutate(&path, statement).await; + } + let fingerprint = backup_fingerprint(&path).await; + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}")).await.unwrap(); + let incarnations: Vec<(String, String, String, Option, Option)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + // The current worker keeps its PHP token; the retired ones keep their state. + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + None + ))); + assert!(incarnations.iter().any(|row| row.1 == "gone" && row.2 == "deregistered" && row.3 == Some(3))); + assert!(incarnations.iter().any(|row| row.1 == "current" && row.2 == "superseded" && row.3.is_none() && row.4.is_some())); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + connection.close().await.unwrap(); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}")).await.unwrap(); + let marker: (String, i64) = sqlx::query_as("SELECT engine,version FROM dw_server_schema") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(marker, ("rust-development".to_owned(), 4)); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn takeover_of_the_frozen_layout_adds_only_native_relations() { + let (_directory, path) = php_layout().await; + mutate(&path, "DELETE FROM jobs").await; + let fingerprint = backup_fingerprint(&path).await; + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; +} + +#[tokio::test] +async fn takeover_refusals_leave_the_php_database_unchanged() { + let (_directory, path) = php297_layout().await; + let fingerprint = backup_fingerprint(&path).await; + let original = std::fs::read(&path).unwrap(); + reason( + takeover::take_over_sqlite(&path, &format!("sha256:{}", "0".repeat(64)), &queue_config()).await, + "upgrade_backup_fingerprint_mismatch", + ); + reason( + takeover::take_over_sqlite(&path, "not-a-fingerprint", &queue_config()).await, + "upgrade_backup_fingerprint_required", + ); + assert_eq!(std::fs::read(&path).unwrap(), original); + // A write after the backup changes the digest, so the old one is refused. + mutate(&path, "INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)").await; + reason( + takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await, + "upgrade_backup_fingerprint_mismatch", + ); + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let (_directory, path) = php297_layout().await; + mutate(&path, statement).await; + let fingerprint = backup_fingerprint(&path).await; + let before = std::fs::read(&path).unwrap(); + reason( + takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await, + expected, + ); + assert_eq!(std::fs::read(&path).unwrap(), before, "{expected}"); + } +} diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index a2524546d..cae370f95 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -50,6 +50,9 @@ $value = [9007199254740993, null, 'upgradeλ']; $envelope = ['codec' => 'avro', 'blob' => Avro::serialize([$value])]; $poll = ['worker_id' => $worker, 'task_queue' => $prefix, 'timeout_seconds' => 0]; +// A takeover run resumes this timer under native, so it may choose a shorter delay. +$timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); +$check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; foreach (['activity', 'timer'] as $kind) { $id = $prefix.'-'.$kind; @@ -64,7 +67,7 @@ ? ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900] - : ['type' => 'start_timer', 'delay_seconds' => 600]; + : ['type' => 'start_timer', 'delay_seconds' => $timerSeconds]; $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'commands' => [$command], diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index f4952c161..27fce947c 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -15,7 +15,7 @@ source=upgrade_source_ref${label:+_$label} restored=upgrade_restore_ref${label:+_$label} image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 -for role in author control inspect php; do +for role in author control inspect php takeover native resume; do if docker inspect "$resource-$role" >/dev/null 2>&1; then docker stop --signal SIGTERM --time 5 "$resource-$role" >/dev/null docker rm "$resource-$role" >/dev/null diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index e939fb647..86d8140ff 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -52,6 +52,9 @@ else chmod 0666 "$state/database.sqlite" fi printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" +# SQLite also takes over a copy and resumes the originals, so its timer is short. +timer_seconds=600 +if test "$backend" = sqlite; then timer_seconds=60; fi configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) php_run() { @@ -71,7 +74,7 @@ for attempt in $(seq 1 30); do done test "$ready" = 1 docker run --rm --name "$resource-author" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ - --network "$RESOURCE_SCOPE" -e DW_PARITY_TOKEN=upgrade-fixture-token -v "$PWD:/source:ro" --entrypoint php "$image" \ + --network "$RESOURCE_SCOPE" -e DW_PARITY_TOKEN=upgrade-fixture-token -e DW_PARITY_UPGRADE_TIMER_SECONDS="$timer_seconds" -v "$PWD:/source:ro" --entrypoint php "$image" \ /source/scripts/conformance/server-parity/upgrade-source-author.php http://parity-upgrade-php:8080 upgrade-source \ > "$evidence/originals.json" 2> "$evidence/author.stderr" docker stop --signal SIGTERM --time 10 "$resource-php" >/dev/null @@ -197,6 +200,51 @@ change_restored fingerprint "$restored" > "$evidence/changed-fingerprint.json" 2>> "$evidence/fingerprint.stderr" changed_fingerprint=$(jq -r .fingerprint "$evidence/changed-fingerprint.json") [[ $changed_fingerprint =~ ^sha256:[0-9a-f]{64}$ && $changed_fingerprint != "$source_fingerprint" ]] +if test "$backend" = sqlite; then + # Backup-first takeover of a copy of the stopped source, as an operator's + # live database: its digest is the one the restored backup matched. Native + # then resumes the originals, and PHP must refuse the upgraded file. + cp "$state/database.sqlite" "$state/takeover.sqlite" + docker run --rm --name "$resource-takeover" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + -e DW_UPGRADE_BACKUP_FINGERPRINT="$source_fingerprint" \ + -e DW_RUST_UPGRADE_QUEUE_CONNECTION=database -e DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION=database \ + -e DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE=default \ + -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-takeover \ + > "$evidence/takeover.json" 2> "$evidence/takeover.stderr" + jq -e --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" + and .source_layout==$layout and .fingerprint==$fingerprint and .backup_fingerprint_matched==true + and .converted.incarnations_copied==(if $layout=="frozen" then 0 else 1 end) + and .stored_representation_compatibility_verified==false' "$evidence/takeover.json" >/dev/null + docker run -d --name "$resource-native" --network "$RESOURCE_SCOPE" --network-alias parity-upgrade-native \ + --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m -e DW_RUST_EXPERIMENTAL=1 \ + -e DW_BIND_ADDRESS=0.0.0.0:8080 -e DW_AUTH_DRIVER=token -e DW_AUTH_TOKEN=upgrade-fixture-token \ + -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite \ + -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" > "$evidence/native-container.txt" + ready=0 + for attempt in $(seq 1 30); do + if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ + -fsS http://parity-upgrade-native:8080/api/ready > "$evidence/native-readiness.json"; then ready=1; break; fi + sleep 1 + done + [[ $ready == 1 ]] + docker run --rm --name "$resource-resume" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ + --network "$RESOURCE_SCOPE" -e DW_PARITY_TOKEN=upgrade-fixture-token -v "$PWD:/source:ro" --entrypoint php "$image" \ + /source/scripts/conformance/server-parity/upgrade-takeover-resume.php http://parity-upgrade-native:8080 \ + "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" + jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 + and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 + and .timer.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null + if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ + --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_AUTH_TOKEN=upgrade-fixture-token \ + -e CACHE_STORE=file -e QUEUE_CONNECTION=database -v "$PWD/$state:/state" --entrypoint php "$image" \ + artisan server:assert-php-database > "$evidence/php-refusal.txt" 2>&1; then exit 1; fi + grep -q php_database_refused "$evidence/php-refusal.txt" + sqlite_takeover=true +fi snapshot > "$evidence/after.json" cmp "$evidence/before.json" "$evidence/after.json" refusal() { diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php new file mode 100644 index 000000000..55492f72f --- /dev/null +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -0,0 +1,113 @@ + rtrim($url, '/'), 'http_errors' => false, 'connect_timeout' => 1, 'timeout' => 10]); +$headers = ['Authorization' => 'Bearer '.getenv('DW_PARITY_TOKEN'), 'X-Namespace' => 'default', + 'X-Durable-Workflow-Control-Plane-Version' => '2', 'X-Durable-Workflow-Protocol-Version' => '1.20']; +$request = static function (string $method, string $path, array $body = []) use ($http, $headers): array { + $response = $http->request($method, $path, ['headers' => $headers] + ($body === [] ? [] : ['json' => $body])); + + return [$response->getStatusCode(), json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; +}; +$check = static function (bool $condition, string $message): void { + if (! $condition) { + throw new RuntimeException($message); + } +}; +$expect = static function (array $response, int $status, string $operation) use ($check): array { + $check($response[0] === $status, $operation.': '.json_encode(['status' => $response[0], + 'reason' => $response[1]['reason'] ?? null], JSON_THROW_ON_ERROR)); + + return $response[1]; +}; +// Complete events, keys sorted at every level: PHP-written history must read +// exactly as PHP returned it before the takeover. +$sorted = static function (mixed $value) use (&$sorted): mixed { + if (! is_array($value)) { + return $value; + } + if (! array_is_list($value)) { + ksort($value); + } + + return array_map($sorted, $value); +}; +$keep = static fn (array $original, array $current): bool => json_encode($sorted($original), JSON_THROW_ON_ERROR) + === json_encode($sorted(array_slice($current, 0, count($original))), JSON_THROW_ON_ERROR); +$types = static fn (array $history): array => array_column($history, 'event_type'); +$history = static function (array $original) use ($request, $expect): array { + return $expect($request('GET', '/api/workflows/'.$original['workflow_id'].'/runs/'.$original['run_id'].'/history'), + 200, 'history')['events']; +}; +$worker = $originals['activity']['activity_task']['lease_owner']; +$queue = $originals['activity']['workflow_task']['task_queue'] ?? 'upgrade-source'; +$result = ['codec' => 'avro', 'blob' => Avro::serialize(['resumed under native'])]; +$outcome = []; + +// The activity leased under PHP completes with its original attempt. +$activity = $originals['activity']; +$task = $activity['activity_task']; +$completion = ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], 'result' => $result]; +$completed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'original activity completion'); +$check(($completed['outcome'] ?? null) === 'completed' && ($completed['recorded'] ?? null) === true, 'Original activity completion was not recorded'); +$replayed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'activity completion replay'); +$check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); +$resume = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, + 'timeout_seconds' => 5]), 200, 'resume claim')['task']; +$check(is_array($resume) && $resume['run_id'] === $activity['run_id'], 'The resumed workflow task was not claimed'); +$original = $activity['history']; +$check(array_slice($types($resume['history_events']), 0, count($original)) === $types($original), 'Original activity history changed in the resumed task'); +$check(end($resume['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); +$expect($request('POST', '/api/worker/workflow-tasks/'.$resume['task_id'].'/complete', ['lease_owner' => $resume['lease_owner'], + 'workflow_task_attempt' => $resume['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, 'resumed workflow completion'); +// The workflow task PHP already completed is a stale attempt. +$stale = $activity['workflow_task']; +$refused = $request('POST', '/api/worker/workflow-tasks/'.$stale['task_id'].'/complete', ['lease_owner' => $stale['lease_owner'], + 'workflow_task_attempt' => $stale['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]); +$check($refused[0] === 409, 'A stale PHP workflow task attempt was accepted'); +$closed = $history($activity); +$check($keep($original, $closed) && end($closed)['event_type'] === 'WorkflowCompleted', + 'The resumed activity workflow did not complete on its original history'); +$outcome['activity'] = ['run_id' => $activity['run_id'], 'events' => array_column($closed, 'event_type'), + 'stale_attempt_status' => $refused[0], 'stale_attempt_reason' => $refused[1]['reason'] ?? null]; + +// The timer scheduled under PHP keeps its deadline and fires under native. +$timer = $originals['timer']; +$scheduled = array_values(array_filter($timer['history'], static fn (array $event): bool => $event['event_type'] === 'TimerScheduled'))[0]; +$fireAt = $scheduled['payload']['fire_at']; +$current = $history($timer); +$check($keep($timer['history'], $current), 'Original timer history changed'); +$deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); +$check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); +$fired = null; +while ($fired === null && microtime(true) < $deadline + 30) { + $poll = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, + 'timeout_seconds' => 5]), 200, 'timer claim'); + $fired = $poll['task'] ?? null; +} +$check(is_array($fired) && $fired['run_id'] === $timer['run_id'], 'The original timer did not fire under native'); +$firedEvent = end($fired['history_events']); +$check($firedEvent['event_type'] === 'TimerFired', 'Timer firing is missing from history'); +$firedPublic = array_values(array_filter($history($timer), static fn (array $event): bool => $event['event_type'] === 'TimerFired')); +$check(count($firedPublic) === 1, 'The original timer did not fire exactly once'); +$firedAt = (float) (new DateTimeImmutable($firedPublic[0]['timestamp']))->format('U.u'); +$check($firedAt >= $deadline, 'The original timer fired before its deadline'); +$expect($request('POST', '/api/worker/workflow-tasks/'.$fired['task_id'].'/complete', ['lease_owner' => $fired['lease_owner'], + 'workflow_task_attempt' => $fired['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, 'timer workflow completion'); +$outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, + 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($history($timer), 'event_type')]; +echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, + JSON_THROW_ON_ERROR)."\n"; From b7b125c1828acde1ba4ca36b3ab5d79d4e711725 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:35:23 +0000 Subject: [PATCH 07/20] Expect the command principal in projected control history Control history now presents each event's command principal as PHP does. These tests rebuild control events from worker events, so the projection includes the principal, null for their unauthenticated commands. Co-Authored-By: Claude Opus 5.5 --- rust/tests/execution.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 8c2b7ba54..19d45b684 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -2346,7 +2346,8 @@ async fn queries_fence_recovered_leases_across_pools_without_mutating_the_run() let claim = await_query(&b, "query-a").await; assert_eq!(claim["run_status"], "completed"); // Worker snapshots retain durable row IDs/links; the control-plane - // history endpoint exposes its documented timestamp projection. + // history endpoint exposes its documented timestamp projection and the + // command principal, which these unauthenticated commands do not record. let projected = Value::Array( claim["history_events"] .as_array() @@ -2354,7 +2355,7 @@ async fn queries_fence_recovered_leases_across_pools_without_mutating_the_run() .iter() .map(|event| { json!({"sequence":event["sequence"],"event_type":event["event_type"], - "payload":event["payload"],"timestamp":event["recorded_at"]}) + "payload":event["payload"],"timestamp":event["recorded_at"],"principal":null}) }) .collect(), ); @@ -4622,13 +4623,14 @@ async fn terminal_activity_failures_resume_once_and_preserve_original_failure() .collect(); assert_eq!(event_ids.len(), worker_events.len()); // Worker history retains event IDs/attribution and recorded_at; - // the control API explicitly returns these four common fields. + // the control API returns these common fields plus the command + // principal, which these unauthenticated commands do not record. let control_events: Vec<_> = worker_events .iter() .map(|event| { json!({ "sequence":event["sequence"],"event_type":event["event_type"], - "timestamp":event["recorded_at"],"payload":event["payload"] + "timestamp":event["recorded_at"],"payload":event["payload"],"principal":null }) }) .collect(); From 094fc7589fce0c17abf0228f5129f4c0796e978d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:37:42 +0000 Subject: [PATCH 08/20] Share the source connection and row reads across fingerprint backends The MySQL inspection and fingerprint now open their source through one helper that detects the backend and bounds lock waits and statements. Each backend's fingerprint builds its row select and closes its connection through shared helpers, keeping only its own value rendering. The cohort script uses [[ ]] tests, names its tool argument and refuses an unknown backend. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade.rs | 79 +++++++--- rust/src/runtime/upgrade/fingerprint.rs | 148 ++++++++---------- .../server-parity/upgrade-source-cleanup.sh | 2 +- .../server-parity/upgrade-source-profile.sh | 15 +- 4 files changed, 127 insertions(+), 117 deletions(-) diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index a8c62ee70..8c885305c 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -98,6 +98,58 @@ async fn close(future: impl Future Ok(()) } +/// A dedicated MySQL or MariaDB source connection, with its backend name. +/// Lock waits stay short and transactions are repeatable reads. The client +/// deadline drops the socket, but the server would keep a running statement, +/// its metadata locks and its snapshot, so `statement_seconds` bounds each +/// statement server-side; zero leaves that to the client deadline. With `utc`, +/// TIMESTAMP values render in UTC. +async fn mysql_source( + options: MySqlConnectOptions, + deadline: Instant, + utc: bool, + statement_seconds: u32, +) -> Result<(MySqlConnection, &'static str)> { + let mut connection = timeout_at( + deadline, + MySqlConnection::connect_with(&mysql::source_options(options)), + ) + .await + .map_err(|_| expired())??; + let setup = timeout_at(deadline, async { + let version: String = sqlx::query_scalar("SELECT VERSION()") + .fetch_one(&mut connection) + .await?; + let (backend, limit) = if version.contains("MariaDB") { + ("mariadb", format!("max_statement_time={statement_seconds}")) + } else { + let milliseconds = u64::from(statement_seconds) * 1000; + ("mysql", format!("max_execution_time={milliseconds}")) + }; + let zone = if utc { "time_zone='+00:00'," } else { "" }; + sqlx::query(AssertSqlSafe(format!( + "SET {zone}sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,{limit}" + ))) + .execute(&mut connection) + .await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + .execute(&mut connection) + .await?; + Ok(backend) + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + match setup { + Ok(backend) => Ok((connection, backend)), + Err(error) => { + // The setup failure explains more than a failed close would. + let _ = close(connection.close()).await; + Err(error) + } + } +} + fn report( backend: &str, catalog: &impl Serialize, @@ -387,33 +439,8 @@ async fn inspect_mysql_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( - deadline, - MySqlConnection::connect_with(&mysql::source_options(options)), - ) - .await - .map_err(|_| expired())??; + let (mut connection, backend) = mysql_source(options, deadline, false, 30).await?; let result = timeout_at(deadline, async { - let version: String = sqlx::query_scalar("SELECT VERSION()") - .fetch_one(&mut connection) - .await?; - let backend = if version.contains("MariaDB") { - "mariadb" - } else { - "mysql" - }; - // The client deadline drops the socket, but the server would keep a running - // statement, its metadata locks and its snapshot. Bound them server-side. - sqlx::query(if backend == "mariadb" { - "SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_statement_time=30" - } else { - "SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_execution_time=30000" - }) - .execute(&mut connection) - .await?; - sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - .execute(&mut connection) - .await?; let mut tx = connection.begin_with("START TRANSACTION READ ONLY").await?; let catalog = mysql::php_source_catalog(&mut tx).await?; let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index d1bdc9b02..a4a83775d 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -8,7 +8,8 @@ //! per-table row counts leave the database. The fingerprint grants nothing: //! every compatibility, backup, quiescence and takeover flag stays false. use super::{ - Result, close, expired, mysql, open_sqlite_source, postgres, readonly_source, refuse, schema, + Result, close, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, + refuse, schema, }; use axum::http::StatusCode; use futures_util::TryStreamExt; @@ -16,7 +17,7 @@ use serde::Serialize; use serde_json::{Value, json}; use sha2::{Digest as _, Sha256}; use sqlx::{ - AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, mysql::MySqlConnectOptions, + AssertSqlSafe, Connection, PgConnection, Row, mysql::MySqlConnectOptions, postgres::PgConnectOptions, }; use std::{collections::BTreeMap, time::Duration}; @@ -100,6 +101,42 @@ fn order(key: &[String], width: usize, quote: char) -> Result { .join(",")) } +/// Every row of one table in primary-key order, with each column rendered by +/// `render(column, position)`. +fn row_select( + from: &str, + columns: &[String], + key: &[String], + quote: char, + render: impl Fn(&str, usize) -> String, +) -> Result { + let select = columns + .iter() + .enumerate() + .map(|(position, column)| identifier(column).map(|column| render(column, position))) + .collect::>>()? + .join(","); + Ok(format!( + "SELECT {select} FROM {from} ORDER BY {}", + order(key, columns.len(), quote)? + )) +} + +/// Runs a read on a dedicated source connection within the deadline, then +/// closes the connection whatever the read returned. +async fn bounded_read( + mut connection: C, + deadline: Instant, + read: impl AsyncFnOnce(&mut C) -> Result, +) -> Result { + let result = timeout_at(deadline, read(&mut connection)) + .await + .map_err(|_| expired()) + .and_then(|result| result); + close(connection.close()).await?; + result +} + // Every rendered value is text, framed in column order. macro_rules! hash_rows { ($tx:ident, $digest:ident, $table:expr, $sql:expr, $width:expr) => {{ @@ -130,8 +167,8 @@ pub async fn fingerprint_sqlite_cli(path: &str) -> Result { pub async fn fingerprint_sqlite(path: &str) -> Result { let deadline = Instant::now() + BUDGET; - let mut connection = open_sqlite_source(path, deadline).await?; - let result = timeout_at(deadline, async { + let connection = open_sqlite_source(path, deadline).await?; + bounded_read(connection, deadline, async |connection| { // One deferred read transaction keeps one snapshot of the file. let mut tx = connection.begin().await?; let catalog = schema::php_source_catalog(&mut tx).await?; @@ -151,40 +188,26 @@ pub async fn fingerprint_sqlite(path: &str) -> Result { digest.table(table, &columns); // quote() keeps the storage class: integers, text and blobs render // exactly. Reals use 17 significant digits, which round-trip. - let select = columns - .iter() - .map(|column| { - identifier(column).map(|c| { - format!("CASE typeof(\"{c}\") WHEN 'real' THEN printf('%!.17g',\"{c}\") ELSE quote(\"{c}\") END") - }) - }) - .collect::>>()? - .join(","); - let sql = format!( - "SELECT {select} FROM \"{table}\" ORDER BY {}", - order(&key, columns.len(), '"')? - ); + let sql = row_select(&format!("\"{table}\""), &columns, &key, '"', |c, _| { + format!("CASE typeof(\"{c}\") WHEN 'real' THEN printf('%!.17g',\"{c}\") ELSE quote(\"{c}\") END") + })?; hash_rows!(tx, digest, table, sql, columns.len()); } tx.rollback().await?; Ok(digest.report("sqlite")) }) .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result } pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( + let connection = timeout_at( deadline, PgConnection::connect_with(&postgres::source_options(options)), ) .await .map_err(|_| expired())??; - let result = timeout_at(deadline, async { + bounded_read(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; @@ -206,56 +229,23 @@ pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") .bind(table).fetch_all(&mut *tx).await?; digest.table(table, &columns); - let select = columns - .iter() - .map(|column| identifier(column).map(|c| format!("quote_nullable(\"{c}\")"))) - .collect::>>()? - .join(","); - let sql = format!( - "SELECT {select} FROM public.\"{table}\" ORDER BY {}", - order(&key, columns.len(), '"')? - ); + let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { + format!("quote_nullable(\"{c}\")") + })?; hash_rows!(tx, digest, table, sql, columns.len()); } tx.rollback().await?; Ok(digest.report("pgsql")) }) .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result } pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( - deadline, - MySqlConnection::connect_with(&mysql::source_options(options)), - ) - .await - .map_err(|_| expired())??; - let result = timeout_at(deadline, async { - let version: String = sqlx::query_scalar("SELECT VERSION()") - .fetch_one(&mut connection) - .await?; - let backend = if version.contains("MariaDB") { - "mariadb" - } else { - "mysql" - }; - // TIMESTAMP values render in UTC. Long table reads are expected, but - // lock waits stay short; the client deadline bounds the whole read. - sqlx::query(if backend == "mariadb" { - "SET time_zone='+00:00',sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_statement_time=0" - } else { - "SET time_zone='+00:00',sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_execution_time=0" - }) - .execute(&mut connection) - .await?; - sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - .execute(&mut connection) - .await?; + // TIMESTAMP values render in UTC. Long table reads are expected, but lock + // waits stay short; the client deadline bounds the whole read. + let (connection, backend) = mysql_source(options, deadline, true, 0).await?; + bounded_read(connection, deadline, async |connection| { let mut tx = connection .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") .await?; @@ -267,27 +257,19 @@ pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { .bind(table).fetch_all(&mut *tx).await?; let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") .bind(table).fetch_all(&mut *tx).await?; - let names: Vec = columns.iter().map(|(name, _)| name.clone()).collect(); + let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); digest.table(table, &names); // Binary values render as hex literals; everything else as its // quoted string form. - let select = columns - .iter() - .map(|(column, kind)| { - identifier(column).map(|c| match kind.to_ascii_lowercase().as_str() { - "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( - "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" - ), - _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), - }) - }) - .collect::>>()? - .join(","); - let sql = format!( - "SELECT {select} FROM `{table}` ORDER BY {}", - order(&key, columns.len(), '`')? - ); - hash_rows!(tx, digest, table, sql, columns.len()); + let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { + match kinds[index].to_ascii_lowercase().as_str() { + "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( + "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" + ), + _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), + } + })?; + hash_rows!(tx, digest, table, sql, names.len()); } // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that // changed while the rows were read. @@ -301,8 +283,4 @@ pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { Ok(digest.report(backend)) }) .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result } diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index 400f04641..bd0829b1c 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -34,7 +34,7 @@ case "$backend" in *) exit 2 ;; esac fi -if test -f "$evidence/restore-owned.txt" && test "$(cat "$evidence/restore-owned.txt")" = "$RESOURCE_SCOPE"; then +if [[ -f "$evidence/restore-owned.txt" && "$(cat "$evidence/restore-owned.txt")" == "$RESOURCE_SCOPE" ]]; then case "$backend" in mysql) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 070277491..5f06e41d5 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -147,7 +147,7 @@ jq -e '.status=="fingerprinted" and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") fingerprint "$database" > "$evidence/fingerprint-repeat.json" 2>> "$evidence/fingerprint.stderr" -test "$(jq -r .fingerprint "$evidence/fingerprint-repeat.json")" = "$source_fingerprint" +[[ "$(jq -r .fingerprint "$evidence/fingerprint-repeat.json")" == "$source_fingerprint" ]] # Back up the stopped source and restore it into a separate database, as the # backup procedure does. The restored copy must fingerprint equal, and one # changed row in it must change the fingerprint. @@ -155,8 +155,10 @@ restored=upgrade_restore_ref case "$backend" in mysql) mysql_admin() { - docker run --rm -i --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint "$1" "$MYSQL_IMAGE" \ - -h "$host" -uroot -p"$password" "${@:2}" + local tool=$1 + shift + docker run --rm -i --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint "$tool" "$MYSQL_IMAGE" \ + -h "$host" -uroot -p"$password" "$@" } mysql_admin mysqldump --single-transaction --quick --no-tablespaces "$database" > "$state/backup.sql" 2> "$evidence/backup.stderr" printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/restore-owned.txt" @@ -166,8 +168,10 @@ case "$backend" in change_restored() { mysql_admin mysql "$restored" -e 'UPDATE jobs SET attempts=attempts+1' 2>> "$evidence/backup.stderr"; } ;; pgsql) pg_admin() { + local tool=$1 + shift docker run --rm -i --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ - --entrypoint "$1" "$POSTGRES_IMAGE" -h "$host" -U parity "${@:2}" + --entrypoint "$tool" "$POSTGRES_IMAGE" -h "$host" -U parity "$@" } pg_admin pg_dump -Fc -d "$database" > "$state/backup.dump" 2> "$evidence/backup.stderr" printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/restore-owned.txt" @@ -184,9 +188,10 @@ case "$backend" in change_restored() { php_run -r '(new PDO("sqlite:/state/restored.sqlite"))->exec("UPDATE jobs SET attempts=attempts+1") === 1 || exit(1);' } ;; + *) exit 2 ;; esac fingerprint "$restored" > "$evidence/restored-fingerprint.json" 2>> "$evidence/fingerprint.stderr" -test "$(jq -r .fingerprint "$evidence/restored-fingerprint.json")" = "$source_fingerprint" +[[ "$(jq -r .fingerprint "$evidence/restored-fingerprint.json")" == "$source_fingerprint" ]] change_restored fingerprint "$restored" > "$evidence/changed-fingerprint.json" 2>> "$evidence/fingerprint.stderr" changed_fingerprint=$(jq -r .fingerprint "$evidence/changed-fingerprint.json") From c14032cea3fb36538de2abd80a0df60aab207e7e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:01:31 +0000 Subject: [PATCH 09/20] Take over a stopped PHP PostgreSQL database backup-first upgrade-takeover now also accepts DB_CONNECTION=pgsql. One read-committed transaction takes native's bootstrap lock, locks every public table in ACCESS EXCLUSIVE mode (refusing with upgrade_source_in_use when a session still holds one after the 5 s lock wait), runs the inspections and the fingerprint against the operator's verified backup digest, adds the native relations, converts the recorded PHP release layout, records the native migration and runs native's startup verification before committing. The layout conversion and ledger record are now shared by SQLite and PostgreSQL through the Backend trait. Inspection and fingerprint reads can run inside the takeover's transaction, and PostgreSQL fingerprint reads are bounded by the client deadline instead of the 30 s statement limit. A takeover killed before its commit leaves nothing committed. On SQLite it leaves a rollback journal that read-only inspection refuses; a takeover rerun with the same digest lets SQLite roll it back, and the fingerprint check then proves the file is the backed-up one. Both backends have a test that SIGKILLs a real takeover process just before its commit and reruns it. The upgrade source cohort takes over a restored copy of the published-PHP PostgreSQL source too, resumes the originals under native and checks PHP's refusal, as it does for SQLite. Co-Authored-By: Claude Opus 5.5 --- rust/README.md | 29 +- rust/src/main.rs | 11 +- rust/src/runtime/postgres.rs | 304 +++++++++++++++- rust/src/runtime/schema.rs | 37 +- rust/src/runtime/upgrade.rs | 101 ++++-- rust/src/runtime/upgrade/fingerprint.rs | 81 ++--- rust/src/runtime/upgrade/takeover.rs | 334 +++++++++++++----- rust/src/runtime/upgrade_tests.rs | 65 ++++ .../server-parity/upgrade-source-cleanup.sh | 6 + .../server-parity/upgrade-source-profile.sh | 44 ++- 10 files changed, 797 insertions(+), 215 deletions(-) diff --git a/rust/README.md b/rust/README.md index 1e4aa7dbb..0876ce23b 100644 --- a/rust/README.md +++ b/rust/README.md @@ -412,31 +412,38 @@ version-1/version-2/version-3 development schemas are refused without conversion needed old development data separately; this unpublished slice supplies no older-development converter. Use a new isolated file for version 4. -This refusal is not the upgrade mechanism. A stopped PHP SQLite database can -be taken over with the development `upgrade-takeover` command, following the -[takeover design](../docs/rust-server-upgrade-takeover.md): +This refusal is not the upgrade mechanism. A stopped PHP SQLite or PostgreSQL +database can be taken over with the development `upgrade-takeover` command, +following the [takeover design](../docs/rust-server-upgrade-takeover.md): 1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and stop every PHP role. -2. Run `upgrade-fingerprint` on the source, mounted read-only. Back the file - up, restore the backup to a separate path, and fingerprint that too. The - digests must be equal. +2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, + or PostgreSQL through a read-only role. Back the database up (a file copy, + or `pg_dump`), restore the backup separately, and fingerprint that too. + The digests must be equal. 3. Run `upgrade-takeover` with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that digest and the original queue settings (`DW_RUST_UPGRADE_QUEUE_*`, as for - `upgrade-queue-inspect`). + `upgrade-queue-inspect`). PostgreSQL uses the owner's `DB_*` settings. Inspection, the fingerprint check, the conversion of the recorded PHP release layout, the native relations, the migration ledger and native's own startup -verification all happen in one exclusive transaction. A refusal or an -interruption leaves the PHP file unchanged. The command refuses: +verification all happen in one transaction. SQLite holds the file +exclusively; PostgreSQL takes native's bootstrap lock and locks every table +before reading, and refuses with `upgrade_source_in_use` if a session still +holds one after five seconds. A refusal leaves the PHP database unchanged. An +interrupted takeover commits nothing: rerun it with the same digest. On +SQLite, the rerun lets SQLite roll back the journal the interrupted run left, +and the fingerprint check then proves the file is the backed-up one. The +command refuses: - a digest that differs, so any write after the backup stops it; - unrecognized queued jobs, external payloads and service calls; - a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7, or a database native already owns. -PostgreSQL and MySQL/MariaDB takeover, nested payload columns and multi-node -qualification remain open. **Never connect PHP to a Rust development database.** +MySQL/MariaDB takeover, nested payload columns and multi-node qualification +remain open. **Never connect PHP to a Rust development database.** Named namespace isolation and configured worker/operator/admin roles have representative shared qualification. Principal/custom-provider credentials and the full authorization contract remain open. Broader timer/signal semantics, failure/retry policy, cancellation and the rest of the API also diff --git a/rust/src/main.rs b/rust/src/main.rs index 79365e07a..22c7793f8 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -143,11 +143,10 @@ async fn main() -> Result<(), Box> { use durable_workflow_server::runtime::upgrade::{ queue_binding::QueueSourceConfig, takeover, }; - if backend != "sqlite" { + if backend != "sqlite" && backend != "pgsql" { eprintln!("upgrade_takeover_backend_unqualified"); return Err("upgrade takeover refused".into()); } - let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; let fingerprint = env::var("DW_UPGRADE_BACKUP_FINGERPRINT") .map_err(|_| "DW_UPGRADE_BACKUP_FINGERPRINT is required")?; // The original Laravel queue transport, as for upgrade-queue-inspect. @@ -164,7 +163,13 @@ async fn main() -> Result<(), Box> { default_connection: &default_connection, default_queue: &default_queue, }; - return match takeover::take_over_sqlite(&path, &fingerprint, &queues).await { + let result = if backend == "pgsql" { + takeover::take_over_postgres(postgres_options()?, &fingerprint, &queues).await + } else { + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + takeover::take_over_sqlite(&path, &fingerprint, &queues).await + }; + return match result { Ok(report) => { println!("{report}"); Ok(()) diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index fa763e307..06af1a9ce 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -39,6 +39,23 @@ fn migrator() -> &'static Migrator { }) } +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native PostgreSQL migration") +} + +/// The native relations, indexes and ownership marker that a fresh +/// bootstrap adds after the frozen PHP layout. Takeover applies them to a +/// converted PHP database instead of running the whole migration. +pub(super) fn native_relations_sql() -> Result { + super::upgrade::native_relations( + include_str!("../../migrations/postgres/0003_full_schema.sql"), + "CREATE TABLE public.dw_server_schema ", + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + ) +} + fn options(options: PgConnectOptions) -> PgConnectOptions { options .application_name("durable-workflow-rust-development") @@ -156,7 +173,7 @@ pub(super) async fn verify_history(connection: &mut PgConnection) -> Result<()> Ok(()) } -async fn verify(connection: &mut PgConnection) -> Result<()> { +pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { verify_scope(connection).await?; verify_history(connection).await?; // A canonical catalog is captured with PostgreSQL itself, not inferred @@ -712,4 +729,289 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout moved to PHP 2.9.7's by the changes PHP's own + /// migrations make (rust/upgrade/php-2.8.0 and php-2.9.7). Inspection + /// must call it php-2.9.7. + async fn php_297(db: &Database) -> PgConnection { + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let php = &frozen[..frozen.find("-- Native bookkeeping").unwrap()]; + let mut connection = db.connection().await; + sqlx::raw_sql(php).execute(&mut connection).await.unwrap(); + sqlx::raw_sql( + "ALTER TABLE workflow_worker_registrations ADD COLUMN registration_token varchar(32); + CREATE TABLE workflow_worker_registration_incarnations (token varchar(32) NOT NULL, namespace varchar(255) NOT NULL, worker_id varchar(255) NOT NULL, status varchar(32) NOT NULL, recovered_workflow_task_count integer NOT NULL DEFAULT 0, finished_at timestamp(6) without time zone, created_at timestamp(6) without time zone, updated_at timestamp(6) without time zone, PRIMARY KEY (token)); + CREATE INDEX workflow_worker_registration_incarnations_finished_at_index ON workflow_worker_registration_incarnations (finished_at); + CREATE INDEX dw_worker_incarnation_scope_idx ON workflow_worker_registration_incarnations (namespace, worker_id); + ALTER TABLE workflow_service_calls ADD COLUMN call_deadline_at timestamp(6) without time zone; + CREATE INDEX wf_service_calls_deadline_idx ON workflow_service_calls (status, call_deadline_at); + ALTER TABLE workflow_tasks ADD COLUMN queue_delivery_reference json;", + ) + .execute(&mut connection) + .await + .unwrap(); + connection + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_postgres(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// Every row and the physical schema, for byte-for-byte comparisons. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let state = ( + catalog(&mut connection).await.unwrap(), + backup_fingerprint(db).await, + ); + connection.close().await.unwrap(); + state + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["backend"], "pgsql"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + // The current worker keeps its PHP token; the retired ones keep their state. + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "current" && row.2 == "superseded" && row.3.is_none() && row.4)); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + connection.close().await.unwrap(); + let storage = PostgresStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + PostgresStorage::check(db.options.clone()).await.unwrap(); + // A second takeover finds a native database. + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let mut connection = db.connection().await; + sqlx::raw_sql(&frozen[..frozen.find("-- Native bookkeeping").unwrap()]) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_refusals_leave_the_php_database_unchanged() { + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), "not-a-fingerprint", &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + assert_eq!(source_state(&db).await, original); + // A PHP connection still holding a table refuses the takeover within + // the lock wait instead of stalling it. + let mut php = db.connection().await; + sqlx::raw_sql("BEGIN; LOCK TABLE jobs IN ROW EXCLUSIVE MODE") + .execute(&mut php) + .await + .unwrap(); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + sqlx::raw_sql("ROLLBACK").execute(&mut php).await.unwrap(); + // A write after the backup changes the digest, so the old one is refused. + sqlx::raw_sql("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .execute(&mut php) + .await + .unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + php.close().await.unwrap(); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql(statement).execute(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + #[tokio::test] + #[ignore = "only invoked as a real subprocess by the takeover interruption qualification"] + async fn takeover_child() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_postgres( + PgConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_killed_before_commit_leaves_php_and_retries() { + use std::os::unix::process::ExitStatusExt; + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::postgres::tests::takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + assert_eq!(std::fs::read_to_string(&receipt).unwrap(), "before-commit"); + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + // PostgreSQL ends the killed session's transaction and its locks. + let released = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let sessions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if sessions == 0 { + break; + } + assert!(tokio::time::Instant::now() < released, "killed takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert_eq!(source_state(&db).await, original, "partial takeover survived"); + // The same verified backup fingerprint still takes the database over. + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } } diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 6b6ebaf0e..9b602f319 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -35,36 +35,17 @@ fn migrator() -> &'static Migrator { /// bootstrap adds after the frozen PHP layout. Takeover applies them to a /// converted PHP database instead of running the whole migration. pub(super) fn native_relations_sql() -> Result { - let frozen = include_str!("../../migrations/sqlite/0003_full_schema.sql"); - let start = frozen - .find("CREATE TABLE dw_server_schema ") - .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; - Ok(format!( - "{}\n{}", - &frozen[start..], - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") - )) + super::upgrade::native_relations( + include_str!("../../migrations/sqlite/0003_full_schema.sql"), + "CREATE TABLE dw_server_schema ", + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + ) } -/// Records the native migration as applied in the caller's transaction, for a -/// takeover whose schema came from conversion. `verify` then checks the -/// marker, this ledger and the complete catalog as for any native database. -pub(super) async fn record_migration(connection: &mut SqliteConnection) -> Result<()> { - use sqlx::migrate::Migrate; - connection - .ensure_migrations_table("_sqlx_migrations") - .await?; - let migration = migrator() - .iter() - .next() - .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_migration_history_mismatch"))?; - sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") - .bind(migration.version) - .bind(migration.description.as_ref()) - .bind(migration.checksum.as_ref()) - .execute(&mut *connection) - .await?; - Ok(()) +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native SQLite migration") } type Catalog = Vec<(String, String, String, String)>; diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 068fa2014..d9029d1ca 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -452,7 +452,16 @@ pub(super) async fn open_sqlite_file( "upgrade_source_sqlite_journal_unqualified", )); } - for suffix in ["-wal", "-shm", "-journal"] { + // A killed takeover leaves its rollback journal. Takeover opens the + // file writable, so SQLite rolls that journal back before anything is + // read, and the fingerprint then proves the result is the backed-up + // PHP database. Read-only opens never attempt that recovery. + let sidecars: &[&str] = if writable { + &["-wal", "-shm"] + } else { + &["-wal", "-shm", "-journal"] + }; + for suffix in sidecars { if tokio::fs::try_exists(format!("{path}{suffix}")).await? { return Err(refuse( StatusCode::CONFLICT, @@ -566,47 +575,77 @@ async fn inspect_postgres_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( + let connection = timeout_at( deadline, PgConnection::connect_with(&postgres::source_options(options)), ) .await .map_err(|_| expired())??; - let result = timeout_at(deadline, async { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") - .map(|(_, name, _)| name.clone()) - .collect(); - let counts = count_tables!(tx, tables, '"'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; + let report = postgres_inventory(&mut tx, queues, payloads).await; tx.rollback().await?; - report( - "pgsql", - &catalog, - layout, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); +} + +/// The PostgreSQL inspection report, read inside the caller's transaction. +pub(super) async fn postgres_inventory( + tx: &mut PgConnection, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, +) -> Result { + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + let counts = count_tables!(tx, tables, '"'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + report( + "pgsql", + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) +} + +/// The frozen migration from its ownership marker on, then the incarnation +/// migration: what a fresh native bootstrap adds after the frozen PHP layout. +pub(super) fn native_relations(frozen: &str, marker: &str, incarnations: &str) -> Result { + let start = frozen + .find(marker) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; + Ok(format!("{}\n{incarnations}", &frozen[start..])) +} + +/// Runs work on a dedicated source connection within the deadline, then +/// closes the connection whatever the work returned. +pub(super) async fn bounded( + mut connection: C, + deadline: Instant, + work: impl AsyncFnOnce(&mut C) -> Result, +) -> Result { + let result = timeout_at(deadline, work(&mut connection)) + .await + .map_err(|_| expired()) + .and_then(|result| result); close(connection.close()).await?; result } diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index a13ed89ed..384995ff2 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -8,7 +8,7 @@ //! per-table row counts leave the database. The fingerprint grants nothing: //! every compatibility, backup, quiescence and takeover flag stays false. use super::{ - Result, close, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, + Result, bounded, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, refuse, schema, }; use axum::http::StatusCode; @@ -80,7 +80,7 @@ impl Digest { /// Names come from a catalog equal to a qualified PHP layout. Anything else is /// refused rather than quoted. -fn identifier(name: &str) -> Result<&str> { +pub(super) fn identifier(name: &str) -> Result<&str> { if name.is_empty() || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { return Err(refuse( StatusCode::CONFLICT, @@ -123,21 +123,6 @@ fn row_select( )) } -/// Runs a read on a dedicated source connection within the deadline, then -/// closes the connection whatever the read returned. -async fn bounded_read( - mut connection: C, - deadline: Instant, - read: impl AsyncFnOnce(&mut C) -> Result, -) -> Result { - let result = timeout_at(deadline, read(&mut connection)) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result -} - // Every rendered value is text, framed in column order. macro_rules! hash_rows { ($tx:ident, $digest:ident, $table:expr, $sql:expr, $width:expr) => {{ @@ -169,7 +154,7 @@ pub async fn fingerprint_sqlite_cli(path: &str) -> Result { pub async fn fingerprint_sqlite(path: &str) -> Result { let deadline = Instant::now() + BUDGET; let connection = open_sqlite_source(path, deadline).await?; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { // One deferred read transaction keeps one snapshot of the file. let mut tx = connection.begin().await?; let report = sqlite_digest(&mut tx).await; @@ -214,45 +199,53 @@ pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { ) .await .map_err(|_| expired())??; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - // Fixed text forms for timestamps, intervals, floats and bytes. - sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'") - .execute(&mut *tx) - .await?; - let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; - let mut digest = Digest::new("pgsql", &catalog)?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") - .map(|(_, name, _)| name.clone()) - .collect(); - for table in &tables { - let table = identifier(table)?; - let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") - .bind(table).fetch_all(&mut *tx).await?; - let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") - .bind(table).fetch_all(&mut *tx).await?; - digest.table(table, &columns); - let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { - format!("quote_nullable(\"{c}\")") - })?; - hash_rows!(tx, digest, table, sql, columns.len()); - } + let report = postgres_digest(&mut tx).await; tx.rollback().await?; - Ok(digest.report("pgsql", layout)) + report }) .await } +/// The PostgreSQL fingerprint report, read inside the caller's transaction. +/// Its settings last until that transaction ends. +pub(super) async fn postgres_digest(tx: &mut PgConnection) -> Result { + // Fixed text forms for timestamps, intervals, floats and bytes. Long table + // reads are expected, so only the client deadline bounds them. + sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'; SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let mut digest = Digest::new("pgsql", &catalog)?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + for table in &tables { + let table = identifier(table)?; + let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") + .bind(table).fetch_all(&mut *tx).await?; + digest.table(table, &columns); + let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { + format!("quote_nullable(\"{c}\")") + })?; + hash_rows!(tx, digest, table, sql, columns.len()); + } + Ok(digest.report("pgsql", layout)) +} + pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { let deadline = Instant::now() + BUDGET; // TIMESTAMP values render in UTC. Long table reads are expected, but lock // waits stay short; the client deadline bounds the whole read. let (connection, backend) = mysql_source(options, deadline, true, 0).await?; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") .await?; diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index a0e71b9b9..b7dbcb1cd 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -1,30 +1,78 @@ -//! Backup-first, one-way takeover of a stopped PHP SQLite database, following +//! Backup-first, one-way takeover of a stopped PHP database, following //! docs/rust-server-upgrade-takeover.md. //! -//! Everything happens in one `BEGIN EXCLUSIVE` transaction: the inspections, -//! the fingerprint check against the operator's verified backup, the layout -//! conversion, the native relations, the migration ledger and native's own -//! startup verification. An interrupted or refused takeover therefore leaves -//! the PHP database exactly as it was, and a committed one is a native -//! database that PHP 2.5.14 and later refuse to open. +//! On SQLite and PostgreSQL everything happens in one transaction that no +//! other writer can interleave with: the inspections, the fingerprint check +//! against the operator's verified backup, the layout conversion, the native +//! relations, the migration ledger and native's own startup verification. An +//! interrupted or refused takeover therefore leaves the PHP database exactly +//! as it was, and a committed one is a native database that PHP 2.5.14 and +//! later refuse to open. use super::{ - Result, close, expired, fingerprint, open_sqlite_file, queue_binding::QueueSourceConfig, - queue_config, refuse, schema, sqlite_inventory, + Result, bounded, fingerprint, open_sqlite_file, postgres, postgres_inventory, + queue_binding::QueueSourceConfig, queue_config, refuse, schema, sqlite_inventory, }; +use crate::runtime::backend::Backend; use axum::http::StatusCode; use serde_json::{Value, json}; -use sqlx::{AssertSqlSafe, Connection, SqliteConnection}; +use sqlx::{ + AssertSqlSafe, Connection, Encode, Executor, IntoArguments, PgConnection, Postgres, Sqlite, + SqliteConnection, Transaction, Type, + migrate::{Migrate, Migration}, + postgres::PgConnectOptions, +}; use std::time::Duration; use tokio::time::{Instant, timeout_at}; /// Takeover reads every row twice (inspection and fingerprint). const BUDGET: Duration = Duration::from_secs(3600); +/// Native's PostgreSQL bootstrap key, so a native node cannot open the +/// database while it is taken over. +const POSTGRES_BOOTSTRAP_KEY: i64 = 4924501473315676162; pub async fn take_over_sqlite( path: &str, backup_fingerprint: &str, queues: &QueueSourceConfig<'_>, ) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = open_sqlite_file(path, deadline, true).await?; + bounded(connection, deadline, async |connection| { + let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; + let outcome = take_over_sqlite_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +/// Takes over with the owner's credentials. The connection keeps native's +/// short lock wait, so a PHP process still holding a table refuses the +/// takeover instead of stalling it. +pub async fn take_over_postgres( + options: PgConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = timeout_at( + deadline, + PgConnection::connect_with(&postgres::source_options(options)), + ) + .await + .map_err(|_| super::expired())??; + bounded(connection, deadline, async |connection| { + // Read committed: once every table is locked, each statement reads + // the latest committed state and nothing else can change it. + let mut tx = connection.begin().await?; + let outcome = take_over_postgres_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +fn preconditions(backup_fingerprint: &str, queues: &QueueSourceConfig<'_>) -> Result<()> { queue_config(Some(queues))?; if !backup_fingerprint.starts_with("sha256:") || backup_fingerprint.len() != 71 { return Err(refuse( @@ -32,68 +80,141 @@ pub async fn take_over_sqlite( "upgrade_backup_fingerprint_required", )); } - let deadline = Instant::now() + BUDGET; - let mut connection = open_sqlite_file(path, deadline, true).await?; - let result = timeout_at(deadline, async { - let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; - match take_over(&mut tx, backup_fingerprint, queues).await { - Ok(report) => { - tx.commit().await?; - Ok(report) - } - Err(error) => { - tx.rollback().await?; - Err(error) - } + Ok(()) +} + +/// Commits a completed takeover, and rolls back anything else. +async fn settle(tx: Transaction<'_, DB>, outcome: Result) -> Result { + match outcome { + Ok(report) => { + tx.commit().await?; + Ok(report) } - }) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result + Err(error) => { + tx.rollback().await?; + Err(error) + } + } } -async fn take_over( +async fn take_over_sqlite_in( tx: &mut SqliteConnection, backup_fingerprint: &str, queues: &QueueSourceConfig<'_>, ) -> Result { let inventory = sqlite_inventory(&mut *tx, Some(queues), true).await?; - qualified(&inventory)?; + let layout = qualified(&inventory)?; // The same digest the operator compared with the restored backup. Any // write after that backup changes it, so nothing below can run. let digest = fingerprint::sqlite_digest(&mut *tx).await?; - if digest["fingerprint"] != backup_fingerprint { + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + .execute(&mut *tx) + .await?; + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, schema::migration()).await?; + // Exactly the check native startup makes: marker, ledger and catalog. + schema::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point(); + Ok(report("sqlite", layout, &digest, &inventory, converted)) +} + +async fn take_over_postgres_in( + tx: &mut PgConnection, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + // Whole-table reads and index builds are expected; the client deadline + // bounds them, while lock waits stay short. + sqlx::raw_sql("SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(POSTGRES_BOOTSTRAP_KEY) + .execute(&mut *tx) + .await?; + // The later DDL needs ACCESS EXCLUSIVE anyway; taking it first means no + // lock is ever upgraded while another session waits. + let tables: Vec = sqlx::query_scalar("SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname") + .fetch_all(&mut *tx) + .await?; + if tables.is_empty() { return Err(refuse( StatusCode::CONFLICT, - "upgrade_backup_fingerprint_mismatch", + "upgrade_source_catalog_unqualified", )); } - let layout = inventory["source_layout"] - .as_str() - .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified"))?; - sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + let tables = tables + .iter() + .map(|table| fingerprint::identifier(table).map(|table| format!("public.\"{table}\""))) + .collect::>>()? + .join(","); + sqlx::raw_sql(AssertSqlSafe(format!( + "LOCK TABLE {tables} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(&mut *tx) + .await + .map_err(|error| match &error { + sqlx::Error::Database(database) if database.code().as_deref() == Some("55P03") => { + refuse(StatusCode::CONFLICT, "upgrade_source_in_use") + } + _ => error.into(), + })?; + let inventory = postgres_inventory(&mut *tx, Some(queues), true).await?; + let layout = qualified(&inventory)?; + let digest = fingerprint::postgres_digest(&mut *tx).await?; + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(postgres::native_relations_sql()?)) .execute(&mut *tx) .await?; - let converted = convert(&mut *tx, layout).await?; - schema::record_migration(&mut *tx).await?; - // Exactly the check native startup makes: marker, ledger and catalog. - schema::verify(&mut *tx).await?; - Ok(json!({ + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, postgres::migration()).await?; + postgres::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point(); + Ok(report("pgsql", layout, &digest, &inventory, converted)) +} + +/// Lets the interruption qualification kill a real takeover process after +/// every step has run, just before the commit. +#[cfg(test)] +fn interruption_point() { + if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") { + std::fs::write(format!("{receipt}.partial"), "before-commit").unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); + loop { + std::thread::park(); + } + } +} + +fn matches_backup(digest: &Value, backup_fingerprint: &str) -> Result<()> { + if digest["fingerprint"] != backup_fingerprint { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + Ok(()) +} + +fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, converted: Value) -> Value { + json!({ "schema":"durable-workflow.rust-upgrade-takeover.v1", - "status":"taken_over", "backend":"sqlite", "source_layout":layout, + "status":"taken_over", "backend":backend, "source_layout":layout, "fingerprint":digest["fingerprint"], "table_rows":inventory["table_rows"], "converted":converted, "backup_fingerprint_matched":true, // What the inspections cannot yet establish stays explicit. "nested_payloads_verified":false, "other_root_payload_columns_verified":false, "stored_representation_compatibility_verified":false - })) + }) } /// Takeover needs every inspection result it relies on, and refuses what /// native cannot represent yet: external payloads and service calls. -fn qualified(inventory: &Value) -> Result<()> { +/// Returns the qualified source layout. +fn qualified(inventory: &Value) -> Result<&str> { if inventory["queue_inspection"]["status"] != "known_subset_bound" { return Err(refuse( StatusCode::CONFLICT, @@ -126,14 +247,52 @@ fn qualified(inventory: &Value) -> Result<()> { "upgrade_source_external_payloads_unsupported", )); } + inventory["source_layout"] + .as_str() + .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified")) +} + +fn query(sql: &'static str) -> sqlx::query::Query<'static, DB, DB::Arguments> +where + DB::Arguments: IntoArguments, +{ + sqlx::query(AssertSqlSafe(DB::statement(sql))) +} + +/// Records the native migration as applied, for a schema that came from +/// conversion. Native verification then checks this ledger like any other. +async fn record_migration( + tx: &mut DB::Connection, + migration: &'static Migration, +) -> Result<()> +where + DB::Connection: Migrate, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, +{ + tx.ensure_migrations_table("_sqlx_migrations").await?; + query::("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) + .await?; Ok(()) } /// Converts a recorded PHP release layout to native's (the design's recipes). -async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { - let (incarnation_table, token_column) = match layout { - "frozen" => (false, false), - "php-2.8.0" | "php-2.9.7" => (true, true), +async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let php_incarnations = match layout { + "frozen" => false, + "php-2.8.0" | "php-2.9.7" => true, _ => { return Err(refuse( StatusCode::CONFLICT, @@ -141,31 +300,30 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { )); } }; - let calls: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM workflow_service_calls") - .fetch_one(&mut *tx) - .await?; - if calls != 0 { + if count::(tx, "SELECT COUNT(*) AS n FROM workflow_service_calls").await? != 0 { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_service_calls_unsupported", )); } - let now = super::super::store::now() - .format("%Y-%m-%d %H:%M:%S%.6f") - .to_string(); + let now = super::super::store::now(); let mut copied = 0; - if incarnation_table { + if php_incarnations { // Same statuses as native. Native keeps the recovered count only on // deregistration and a finish time on every retired incarnation. - copied = sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$1),COALESCE(updated_at,created_at,$1) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") - .bind(&now) - .execute(&mut *tx) - .await? - .rows_affected(); - let total: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM workflow_worker_registration_incarnations") - .fetch_one(&mut *tx) - .await?; + copied = DB::affected( + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$2),COALESCE(updated_at,created_at,$3) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .execute(&mut *tx) + .await?, + ); + let total = count::( + tx, + "SELECT COUNT(*) AS n FROM workflow_worker_registration_incarnations", + ) + .await?; if copied != total as u64 { return Err(refuse( StatusCode::CONFLICT, @@ -175,27 +333,28 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { } // Every active registration gets the authority native expects. A PHP // 2.8.0+ worker keeps the token it was given; an older one gets a new token. - let missing: Vec<(String, String, Option)> = sqlx::query_as(AssertSqlSafe(format!( - "SELECT r.namespace,r.worker_id,{} FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id", - if token_column { "r.registration_token" } else { "NULL" } - ))) + let missing = query::(if php_incarnations { + "SELECT r.namespace,r.worker_id,r.registration_token AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + } else { + "SELECT r.namespace,r.worker_id,NULL AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + }) .fetch_all(&mut *tx) .await?; - for (namespace, worker_id, token) in &missing { - let token = match token { - Some(token) => token.clone(), + for row in &missing { + let token = match DB::optional_string(row, "token")? { + Some(token) => token, None => super::super::worker_incarnations::token().await?, }; - sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") .bind(token) - .bind(namespace) - .bind(worker_id) - .bind(&now) - .bind(&now) + .bind(DB::string(row, "namespace")?) + .bind(DB::string(row, "worker_id")?) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) .execute(&mut *tx) .await?; } - if incarnation_table { + if php_incarnations { sqlx::raw_sql( "DROP TABLE workflow_worker_registration_incarnations; \ ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ @@ -206,11 +365,12 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { .await?; } let delivery_references = if layout == "php-2.9.7" { - // PHP's own queue delivery bookkeeping; native does not use the jobs. - let references: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", + // PHP's own queue delivery lookup metadata; native does not use PHP's + // queue, and the queue inspection bound every job to its task row. + let references = count::( + tx, + "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", ) - .fetch_one(&mut *tx) .await?; sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") .execute(&mut *tx) @@ -224,3 +384,11 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { "queue_delivery_references_removed":delivery_references })) } + +async fn count(tx: &mut DB::Connection, sql: &'static str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, +{ + DB::number(&query::(sql).fetch_one(&mut *tx).await?, "n") +} diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index d45f60094..a140ec2ba 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -935,6 +935,71 @@ async fn takeover_converts_a_php_297_database_that_native_then_opens() { connection.close().await.unwrap(); } +#[tokio::test] +#[ignore = "only invoked as a real subprocess by the takeover interruption test"] +async fn sqlite_takeover_child() { + let path = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await; + panic!("takeover returned instead of parking: {report:?}"); +} + +#[cfg(unix)] +#[tokio::test] +async fn takeover_killed_before_commit_leaves_php_data_and_can_be_rerun() { + use std::os::unix::process::ExitStatusExt; + let (directory, path) = php297_layout().await; + let fingerprint = backup_fingerprint(&path).await; + let original = std::fs::read(&path).unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::upgrade::tests::sqlite_takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &path) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + // Either SQLite had not written the file yet, or its rollback journal + // still holds the original pages. Read-only inspection never recovers it. + if Path::new(&format!("{path}-journal")).exists() { + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + } else { + assert_eq!(std::fs::read(&path).unwrap(), original); + } + // Rerunning with the same verified fingerprint rolls the journal back, + // finds the backed-up PHP database and takes it over. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["fingerprint"], fingerprint.as_str()); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; +} + #[tokio::test] async fn takeover_of_the_frozen_layout_adds_only_native_relations() { let (_directory, path) = php_layout().await; diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index fb8f53b31..418448a34 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -53,6 +53,12 @@ case "$backend" in *) ;; esac fi +if [[ $backend == pgsql && -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 +fi if test -f "$evidence/roles-owned.txt" && test "$(cat "$evidence/roles-owned.txt")" = "$RESOURCE_SCOPE"; then if test "$backend" = mysql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 409a3b690..b50f7c585 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -52,9 +52,10 @@ else chmod 0666 "$state/database.sqlite" fi printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" -# SQLite also takes over a copy and resumes the originals, so its timer is short. +# SQLite and PostgreSQL also take over a copy and resume the originals, so +# their timer is short. timer_seconds=600 -if test "$backend" = sqlite; then timer_seconds=60; fi +if [[ $backend == sqlite || $backend == pgsql ]]; then timer_seconds=60; fi configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) php_run() { @@ -205,28 +206,44 @@ change_restored fingerprint "$restored" > "$evidence/changed-fingerprint.json" 2>> "$evidence/fingerprint.stderr" changed_fingerprint=$(jq -r .fingerprint "$evidence/changed-fingerprint.json") [[ $changed_fingerprint =~ ^sha256:[0-9a-f]{64}$ && $changed_fingerprint != "$source_fingerprint" ]] -if test "$backend" = sqlite; then - # Backup-first takeover of a copy of the stopped source, as an operator's - # live database: its digest is the one the restored backup matched. Native - # then resumes the originals, and PHP must refuse the upgraded file. - cp "$state/database.sqlite" "$state/takeover.sqlite" +# Backup-first takeover of a copy of the stopped source, as an operator's +# live database: its digest is the one the restored backup matched. Native +# then resumes the originals, and PHP must refuse the upgraded database. +target=() +case "$backend" in + sqlite) + cp "$state/database.sqlite" "$state/takeover.sqlite" + target=(-e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite) + target_network=none ;; + pgsql) + # The same backup restored once more, taken over with the owner role. + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + pg_admin createdb "$takeover_database" 2>> "$evidence/backup.stderr" + pg_admin pg_restore --no-owner --exit-on-error -d "$takeover_database" < "$state/backup.dump" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=pgsql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=parity -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; + *) ;; +esac +if (( ${#target[@]} )); then docker run --rm --name "$resource-takeover" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ - --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + --network "$target_network" "${target[@]}" -e DW_RUST_EXPERIMENTAL=1 \ -e DW_UPGRADE_BACKUP_FINGERPRINT="$source_fingerprint" \ -e DW_RUST_UPGRADE_QUEUE_CONNECTION=database -e DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION=database \ -e DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE=default \ -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-takeover \ > "$evidence/takeover.json" 2> "$evidence/takeover.stderr" - jq -e --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" - and .source_layout==$layout and .fingerprint==$fingerprint and .backup_fingerprint_matched==true + jq -e --arg backend "$backend" --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" + and .backend==$backend and .source_layout==$layout and .fingerprint==$fingerprint + and .backup_fingerprint_matched==true and .converted.incarnations_copied==(if $layout=="frozen" then 0 else 1 end) and .stored_representation_compatibility_verified==false' "$evidence/takeover.json" >/dev/null docker run -d --name "$resource-native" --network "$RESOURCE_SCOPE" --network-alias parity-upgrade-native \ --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m -e DW_RUST_EXPERIMENTAL=1 \ -e DW_BIND_ADDRESS=0.0.0.0:8080 -e DW_AUTH_DRIVER=token -e DW_AUTH_TOKEN=upgrade-fixture-token \ - -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite \ - -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + "${target[@]}" -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ --entrypoint /target/debug/durable-workflow-server "$rust_image" > "$evidence/native-container.txt" ready=0 for attempt in $(seq 1 30); do @@ -244,11 +261,10 @@ if test "$backend" = sqlite; then and .timer.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ - --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_AUTH_TOKEN=upgrade-fixture-token \ + --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -v "$PWD/$state:/state" --entrypoint php "$image" \ artisan server:assert-php-database > "$evidence/php-refusal.txt" 2>&1; then exit 1; fi grep -q php_database_refused "$evidence/php-refusal.txt" - sqlite_takeover=true fi snapshot > "$evidence/after.json" cmp "$evidence/before.json" "$evidence/after.json" From 3c1cc665678b262b2e59b615b233b28984b49998 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:42:46 +0000 Subject: [PATCH 10/20] Take over a stopped PHP MySQL or MariaDB database in recorded steps MySQL and MariaDB commit DDL implicitly, and LOCK TABLES refuses to create or drop tables it does not hold, so their takeover cannot be one transaction. Under native's bootstrap lock, upgrade-takeover now: - refuses while any other session uses the database (which needs the PROCESS privilege to see), then verifies the inventory and fingerprint in one consistent snapshot against the backup digest; - writes native's marker with a rust-takeover: row holding the digest. PHP refuses the database from then on, and native startup refuses it with native_takeover_incomplete; - checks the sessions and the fingerprint again with the marker set aside, then runs each conversion step, recording its completion in the row. Row changes commit with their step; DDL steps check what is left to do, so repeating an interrupted one is harmless; - commits the migration ledger with native's final marker. A rerun with the same digest resumes after the last recorded step; a different digest is refused. Laravel's mariadb driver layout, whose backup hold owner is a uuid, is converted too. The upgrade source cohort now takes over a restored copy on every backend, and its steps are named for what they do. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 12 +- rust/README.md | 43 +- rust/src/main.rs | 4 +- rust/src/runtime/mysql.rs | 432 +++++++++++++++++- rust/src/runtime/upgrade.rs | 94 ++-- rust/src/runtime/upgrade/fingerprint.rs | 78 ++-- rust/src/runtime/upgrade/takeover.rs | 139 +++--- .../runtime/upgrade/takeover/mysql_journal.rs | 397 ++++++++++++++++ .../server-parity/upgrade-source-cleanup.sh | 19 +- .../server-parity/upgrade-source-profile.sh | 20 +- 10 files changed, 1077 insertions(+), 161 deletions(-) create mode 100644 rust/src/runtime/upgrade/takeover/mysql_journal.rs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 90a4dbe41..aee48b1b8 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -663,14 +663,14 @@ jobs: cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json - name: Compare separate configured role-token cohort run: bash scripts/conformance/server-parity/role-profile.sh mysql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh mysql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 @@ -1157,14 +1157,14 @@ jobs: env: POSTGRES_IMAGE: ${{ matrix.image }} run: bash scripts/conformance/server-parity/role-profile.sh pgsql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh pgsql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 @@ -1642,7 +1642,7 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/role-profile.sh sqlite - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} @@ -1653,7 +1653,7 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 diff --git a/rust/README.md b/rust/README.md index 0876ce23b..36b9eee29 100644 --- a/rust/README.md +++ b/rust/README.md @@ -412,38 +412,51 @@ version-1/version-2/version-3 development schemas are refused without conversion needed old development data separately; this unpublished slice supplies no older-development converter. Use a new isolated file for version 4. -This refusal is not the upgrade mechanism. A stopped PHP SQLite or PostgreSQL -database can be taken over with the development `upgrade-takeover` command, -following the [takeover design](../docs/rust-server-upgrade-takeover.md): +This refusal is not the upgrade mechanism. A stopped PHP database can be taken +over with the development `upgrade-takeover` command, following the +[takeover design](../docs/rust-server-upgrade-takeover.md): 1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and stop every PHP role. 2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, - or PostgreSQL through a read-only role. Back the database up (a file copy, - or `pg_dump`), restore the backup separately, and fingerprint that too. - The digests must be equal. + or the database through a read-only account. Back the database up (a file + copy, `pg_dump` or `mysqldump --single-transaction`), restore the backup + separately, and fingerprint that too. The digests must be equal. 3. Run `upgrade-takeover` with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that digest and the original queue settings (`DW_RUST_UPGRADE_QUEUE_*`, as for `upgrade-queue-inspect`). PostgreSQL uses the owner's `DB_*` settings. + MySQL and MariaDB use an account that owns the database and has the + `PROCESS` privilege, so the takeover can see every other session. -Inspection, the fingerprint check, the conversion of the recorded PHP release -layout, the native relations, the migration ledger and native's own startup -verification all happen in one transaction. SQLite holds the file +On SQLite and PostgreSQL, inspection, the fingerprint check, the conversion +of the recorded PHP release layout, the native relations, the migration ledger +and native's own startup verification all happen in one transaction. SQLite +holds the file exclusively; PostgreSQL takes native's bootstrap lock and locks every table before reading, and refuses with `upgrade_source_in_use` if a session still holds one after five seconds. A refusal leaves the PHP database unchanged. An interrupted takeover commits nothing: rerun it with the same digest. On SQLite, the rerun lets SQLite roll back the journal the interrupted run left, -and the fingerprint check then proves the file is the backed-up one. The -command refuses: +and the fingerprint check then proves the file is the backed-up one. + +MySQL and MariaDB commit DDL implicitly, so their takeover runs in steps under +native's bootstrap lock. It refuses with `upgrade_source_in_use` while any +other session uses the database. Once the source is verified, it first writes +native's marker with a `rust-takeover:` row: PHP refuses the database +from then on, and native refuses it with `native_takeover_incomplete` until +the takeover finishes. It then checks the sessions and the fingerprint again, +and runs each conversion step, recording each in that row. An interrupted +takeover is rerun with the same digest and continues after the last recorded +step; a different digest is refused. + +The command refuses: - a digest that differs, so any write after the backup stops it; - unrecognized queued jobs, external payloads and service calls; -- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7, or a database - native already owns. +- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7 (with either + Laravel MariaDB driver), or a database native already owns. -MySQL/MariaDB takeover, nested payload columns and multi-node qualification -remain open. **Never connect PHP to a Rust development database.** +Nested payload columns and multi-node qualification remain open. **Never connect PHP to a Rust development database.** Named namespace isolation and configured worker/operator/admin roles have representative shared qualification. Principal/custom-provider credentials and the full authorization contract remain open. Broader timer/signal semantics, failure/retry policy, cancellation and the rest of the API also diff --git a/rust/src/main.rs b/rust/src/main.rs index 22c7793f8..56a8fe465 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -143,7 +143,7 @@ async fn main() -> Result<(), Box> { use durable_workflow_server::runtime::upgrade::{ queue_binding::QueueSourceConfig, takeover, }; - if backend != "sqlite" && backend != "pgsql" { + if !matches!(backend.as_str(), "sqlite" | "pgsql" | "mysql" | "mariadb") { eprintln!("upgrade_takeover_backend_unqualified"); return Err("upgrade takeover refused".into()); } @@ -165,6 +165,8 @@ async fn main() -> Result<(), Box> { }; let result = if backend == "pgsql" { takeover::take_over_postgres(postgres_options()?, &fingerprint, &queues).await + } else if backend != "sqlite" { + takeover::take_over_mysql(mysql_options()?, &fingerprint, &queues).await } else { let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; takeover::take_over_sqlite(&path, &fingerprint, &queues).await diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 35e68ac0d..a4a76dbf5 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -13,15 +13,19 @@ pub const VERSION: i64 = 4; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; -pub const LEDGER_SQL: &str = "CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; +/// The marker row of a takeover in progress: `rust-takeover:`, with +/// the completed step count as its version and the backup fingerprint's +/// digest as its checksum. Native startup refuses it and PHP sees the marker. +pub(super) const TAKEOVER_ENGINE: &str = "rust-takeover:"; +pub const LEDGER_SQL: &str ="CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; #[derive(Clone, Copy)] -enum Flavor { +pub(super) enum Flavor { MySql, MariaDb, } impl Flavor { - async fn detect(connection: &mut MySqlConnection) -> Result { + pub(super) async fn detect(connection: &mut MySqlConnection) -> Result { let version: String = sqlx::query_scalar("SELECT VERSION()") .fetch_one(connection) .await?; @@ -72,7 +76,35 @@ impl Flavor { Self::MariaDb => include_str!("../../migrations/mariadb/0004_catalog.json"), }))?) } - fn checksum(self) -> &'static [u8] { + /// The single native migration, which a takeover records as applied. + pub(super) fn migration(self) -> &'static Migration { + self.migrator().iter().next().expect("one native MySQL migration") + } + /// The native relations a fresh bootstrap adds after the frozen PHP + /// layout, each created only if missing so a resumed takeover can repeat + /// them. The poll index on `workflow_tasks` is added separately. + pub(super) fn native_relations_sql(self) -> Result { + let (frozen, incarnations) = match self { + Self::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + ), + Self::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + ), + }; + Ok(super::upgrade::native_relations( + frozen, + "-- Native receipts are separate", + incarnations, + )? + .replace( + "CREATE TABLE dw_worker_registration_incarnations", + "CREATE TABLE IF NOT EXISTS dw_worker_registration_incarnations", + )) + } + pub(super) fn checksum(self) -> &'static [u8] { self.migrator().iter().next().unwrap().checksum.as_ref() } } @@ -83,7 +115,7 @@ fn options(options: MySqlConnectOptions) -> MySqlConnectOptions { .collation("utf8mb4_unicode_ci") .timezone(Some("+00:00".into())) } -async fn configure(connection: &mut MySqlConnection) -> Result<()> { +pub(super) async fn configure(connection: &mut MySqlConnection) -> Result<()> { sqlx::query("SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5") .execute(connection) .await?; @@ -205,13 +237,44 @@ pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Resu )); } } + let actual = catalog(connection).await?; + source_layout(connection, actual).await +} + +/// The PHP source catalog of a database whose takeover has written its +/// marker and nothing else yet: native's exact marker table, set aside. +pub(super) async fn fenced_source_catalog(connection: &mut MySqlConnection) -> Result<(Catalog, &'static str)> { + let flavor = Flavor::detect(connection).await?; + let marker = flavor + .expected()? + .into_iter() + .find(|(table, _)| table == "dw_server_schema"); + let mut actual = catalog(connection).await?; + let position = actual.iter().position(|(table, _)| table == "dw_server_schema"); + match position { + Some(position) if Some(&actual[position]) == marker.as_ref() => { + actual.remove(position); + } + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_marker_unqualified", + )); + } + } + source_layout(connection, actual).await +} + +async fn source_layout( + connection: &mut MySqlConnection, + mut actual: Catalog, +) -> Result<(Catalog, &'static str)> { let flavor = Flavor::detect(connection).await?; let expected: Catalog = flavor.expected()?.into_iter() .filter(|(table, _)| !super::upgrade::native_relation(table)) .map(|(table, ddl)| (table, ddl.replace( ",\n KEY `dw_workflow_tasks_poll` (`namespace`(64),`queue`(64),`task_type`(16),`status`(16),`available_at`)", ""))) .collect(); - let mut actual = catalog(connection).await?; // MariaDB's information_schema collation ignores some underscores. Only // canonicalize catalog presentation; preserve every full definition. actual.sort_by(|left, right| left.0.cmp(&right.0)); @@ -322,7 +385,7 @@ pub(super) async fn verify_history(connection: &mut MySqlConnection) -> Result<( Ok(()) } -async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { +pub(super) async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { history(connection, flavor, false).await?; if catalog(connection).await? != flavor.expected()? { return Err(refuse( @@ -388,6 +451,9 @@ async fn preflight(connection: &mut MySqlConnection, flavor: Flavor) -> Result<( if engines == ["rust-initializing"] { verify_initializing(connection, flavor).await?; Ok(()) + } else if engines.iter().any(|engine| engine.starts_with(TAKEOVER_ENGINE)) { + // An interrupted takeover resumes only through upgrade-takeover. + Err(refuse(StatusCode::CONFLICT, "native_takeover_incomplete")) } else { verify(connection, flavor).await } @@ -1038,4 +1104,356 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout, or a release layout rebuilt from the recorded + /// deltas (each changed table exactly as that release's PHP created it). + async fn php_layout(db: &Database, release: Option<&str>) -> Flavor { + let mut connection = db.connection().await; + let flavor = Flavor::detect(&mut connection).await.unwrap(); + let (frozen, deltas) = match flavor { + Flavor::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mysql.json"), + include_str!("../../upgrade/php-2.9.7/mysql.json"), + ], + ), + Flavor::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mariadb.json"), + include_str!("../../upgrade/php-2.9.7/mariadb.json"), + ], + ), + }; + let php = &frozen[..frozen.find("-- Native receipts are separate").unwrap()]; + connection.execute(php).await.unwrap(); + connection + .execute("SET FOREIGN_KEY_CHECKS=1; ALTER TABLE workflow_tasks DROP KEY dw_workflow_tasks_poll") + .await + .unwrap(); + let releases = match release { + None => 0, + Some("php-2.8.0") => 1, + Some(_) => 2, + }; + for delta in &deltas[..releases] { + let delta: Value = serde_json::from_str(delta).unwrap(); + for added in delta["add"].as_array().unwrap() { + let (table, ddl) = (added[0].as_str().unwrap(), added[1].as_str().unwrap()); + connection + .execute(table_sql("DROP TABLE IF EXISTS ", table, "").unwrap()) + .await + .unwrap(); + connection.execute(AssertSqlSafe(ddl.to_owned())).await.unwrap(); + } + } + connection.close().await.unwrap(); + flavor + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_mysql(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// The physical schema and the fingerprint of every PHP row. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let schema = catalog(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + (schema, backup_fingerprint(db).await) + } + + async fn sessions_ended(db: &Database) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if others == 0 { + return; + } + assert!(tokio::time::Instant::now() < deadline, "a takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + async fn registrations(db: &Database) { + let mut connection = db.connection().await; + connection.execute( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .await + .unwrap(); + connection.close().await.unwrap(); + } + + /// Native opens the database and holds the converted incarnations. + async fn converted(db: &Database) { + let storage = MySqlStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + MySqlStorage::check(db.options.clone()).await.unwrap(); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + converted(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_the_mariadb_driver_layout() { + let db = Database::new().await; + if let Flavor::MySql = php_layout(&db, Some("php-2.9.7")).await { + db.remove().await; + return; + } + // Laravel's mariadb connection stores the backup hold owner as uuid. + let mut connection = db.connection().await; + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner uuid DEFAULT NULL") + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "php-2.9.7+mariadb-driver", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + php_layout(&db, None).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_refusals_before_the_marker_leave_php_unchanged() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "z".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + // Any other session on the database refuses the takeover. + let php = db.connection().await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + php.close().await.unwrap(); + assert_eq!(source_state(&db).await, original); + // A write after the backup changes the digest, so the old one is refused. + let mut php = db.connection().await; + php.execute("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .await + .unwrap(); + php.close().await.unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let mut connection = db.connection().await; + connection.execute(statement).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + // Named under bootstrap_child so the qualification run's existing skip of + // that subprocess helper also skips this one. + #[tokio::test] + #[ignore = "invoked only as a subprocess by the takeover interruption qualification"] + async fn bootstrap_child_takeover() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks at DW_TAKEOVER_TEST_BOUNDARY, where the parent kills it. + let report = takeover::take_over_mysql( + MySqlConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_killed_at_each_step_resumes_with_the_same_fingerprint() { + use std::os::unix::process::ExitStatusExt; + for boundary in [ + "before-marker-row", + "after-step-1", + "after-step-3", + "after-step-8", + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::mysql::tests::bootstrap_child_takeover", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_BOUNDARY", boundary) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(90); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before {boundary}: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach {boundary}"); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + sessions_ended(&db).await; + // The marker already fences PHP, and native will not open the + // database until the takeover finishes. + let mut connection = db.connection().await; + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='dw_server_schema'") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(marked, 1, "{boundary}"); + assert!(MySqlStorage::open(db.options.clone()).await.is_err(), "{boundary}"); + if boundary == "after-step-3" { + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + } + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{boundary}: {report}"); + assert_eq!(report["source_layout"], "php-2.9.7", "{boundary}"); + converted(&db).await; + db.remove().await; + } + } } diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index d9029d1ca..ef6baf642 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -511,46 +511,68 @@ async fn inspect_mysql_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let (mut connection, backend) = mysql_source(options, deadline, false, 30).await?; - let result = timeout_at(deadline, async { + let (connection, backend) = mysql_source(options, deadline, false, 30).await?; + bounded(connection, deadline, async |connection| { let mut tx = connection.begin_with("START TRANSACTION READ ONLY").await?; - let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); - let counts = count_tables!(tx, tables, '`'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; - // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible - // catalog rather than joining counts to a different schema. - if mysql::php_source_catalog(&mut tx).await?.0 != catalog { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_changed", - )); - } + let report = mysql_inventory(&mut tx, backend, queues, payloads, false).await; tx.rollback().await?; - report( - backend, - &catalog, - layout, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result +} + +/// The PHP source catalog, or with `fenced` the catalog of a source whose +/// takeover marker is all that has been written, with that marker set aside. +pub(super) async fn mysql_source_catalog( + connection: &mut MySqlConnection, + fenced: bool, +) -> Result<(Vec<(String, String)>, &'static str)> { + if fenced { + mysql::fenced_source_catalog(connection).await + } else { + mysql::php_source_catalog(connection).await + } +} + +/// The MySQL or MariaDB inspection report, read inside the caller's +/// transaction. +pub(super) async fn mysql_inventory( + tx: &mut MySqlConnection, + backend: &str, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, + fenced: bool, +) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); + let counts = count_tables!(tx, tables, '`'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible + // catalog rather than joining counts to a different schema. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + report( + backend, + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) } pub async fn inspect_postgres(options: PgConnectOptions) -> Result { diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index 384995ff2..df82bb3b8 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -8,8 +8,8 @@ //! per-table row counts leave the database. The fingerprint grants nothing: //! every compatibility, backup, quiescence and takeover flag stays false. use super::{ - Result, bounded, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, - refuse, schema, + Result, bounded, expired, mysql_source, mysql_source_catalog, open_sqlite_source, postgres, + readonly_source, refuse, schema, }; use axum::http::StatusCode; use futures_util::TryStreamExt; @@ -17,8 +17,8 @@ use serde::Serialize; use serde_json::{Value, json}; use sha2::{Digest as _, Sha256}; use sqlx::{ - AssertSqlSafe, Connection, PgConnection, Row, SqliteConnection, mysql::MySqlConnectOptions, - postgres::PgConnectOptions, + AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, SqliteConnection, + mysql::MySqlConnectOptions, postgres::PgConnectOptions, }; use std::{collections::BTreeMap, time::Duration}; use tokio::time::{Instant, timeout, timeout_at}; @@ -249,38 +249,46 @@ pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { let mut tx = connection .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") .await?; - let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; - let mut digest = Digest::new(backend, &catalog)?; - for (table, _) in &catalog { - let table = identifier(table)?; - let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") - .bind(table).fetch_all(&mut *tx).await?; - let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") - .bind(table).fetch_all(&mut *tx).await?; - let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); - digest.table(table, &names); - // Binary values render as hex literals; everything else as its - // quoted string form. - let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { - match kinds[index].to_ascii_lowercase().as_str() { - "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( - "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" - ), - _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), - } - })?; - hash_rows!(tx, digest, table, sql, names.len()); - } - // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that - // changed while the rows were read. - if mysql::php_source_catalog(&mut tx).await?.0 != catalog { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_changed", - )); - } + let report = mysql_digest(&mut tx, backend, false).await; tx.rollback().await?; - Ok(digest.report(backend, layout)) + report }) .await } + +/// The MySQL or MariaDB fingerprint report, read inside the caller's +/// consistent snapshot. With `fenced`, a takeover's own marker is set aside, +/// so the digest is the one the backup had. +pub(super) async fn mysql_digest(tx: &mut MySqlConnection, backend: &str, fenced: bool) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let mut digest = Digest::new(backend, &catalog)?; + for (table, _) in &catalog { + let table = identifier(table)?; + let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); + digest.table(table, &names); + // Binary values render as hex literals; everything else as its + // quoted string form. + let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { + match kinds[index].to_ascii_lowercase().as_str() { + "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( + "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" + ), + _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), + } + })?; + hash_rows!(tx, digest, table, sql, names.len()); + } + // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that + // changed while the rows were read. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + Ok(digest.report(backend, layout)) +} diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index b7dbcb1cd..2eb10c25d 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -24,6 +24,9 @@ use sqlx::{ use std::time::Duration; use tokio::time::{Instant, timeout_at}; +mod mysql_journal; +pub use mysql_journal::take_over_mysql; + /// Takeover reads every row twice (inspection and fingerprint). const BUDGET: Duration = Duration::from_secs(3600); /// Native's PostgreSQL bootstrap key, so a native node cannot open the @@ -116,7 +119,7 @@ async fn take_over_sqlite_in( // Exactly the check native startup makes: marker, ledger and catalog. schema::verify(&mut *tx).await?; #[cfg(test)] - interruption_point(); + interruption_point("before-commit"); Ok(report("sqlite", layout, &digest, &inventory, converted)) } @@ -172,16 +175,20 @@ async fn take_over_postgres_in( record_migration::(&mut *tx, postgres::migration()).await?; postgres::verify(&mut *tx).await?; #[cfg(test)] - interruption_point(); + interruption_point("before-commit"); Ok(report("pgsql", layout, &digest, &inventory, converted)) } -/// Lets the interruption qualification kill a real takeover process after -/// every step has run, just before the commit. +/// Lets the interruption qualification kill a real takeover process at a +/// named boundary: `before-commit` unless `DW_TAKEOVER_TEST_BOUNDARY` names +/// another. #[cfg(test)] -fn interruption_point() { - if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") { - std::fs::write(format!("{receipt}.partial"), "before-commit").unwrap(); +fn interruption_point(boundary: &str) { + let wanted = std::env::var("DW_TAKEOVER_TEST_BOUNDARY"); + if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") + && wanted.as_deref().unwrap_or("before-commit") == boundary + { + std::fs::write(format!("{receipt}.partial"), boundary).unwrap(); std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); loop { std::thread::park(); @@ -283,29 +290,87 @@ where Ok(()) } -/// Converts a recorded PHP release layout to native's (the design's recipes). -async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +/// Whether a recorded PHP release layout has PHP's own registration +/// incarnations. Layouts without a recipe are refused. +fn php_incarnations(layout: &str) -> Result { + match layout.trim_end_matches("+mariadb-driver") { + "frozen" => Ok(false), + "php-2.8.0" | "php-2.9.7" => Ok(true), + _ => Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_layout_unconverted", + )), + } +} + +/// Service calls are not part of native yet. +async fn service_calls_absent(tx: &mut DB::Connection) -> Result<()> where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, DB::Arguments: IntoArguments, - for<'q> String: Encode<'q, DB> + Type, { - let php_incarnations = match layout { - "frozen" => false, - "php-2.8.0" | "php-2.9.7" => true, - _ => { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_layout_unconverted", - )); - } - }; if count::(tx, "SELECT COUNT(*) AS n FROM workflow_service_calls").await? != 0 { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_service_calls_unsupported", )); } + Ok(()) +} + +/// Converts a recorded PHP release layout to native's (the design's recipes). +async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let php_incarnations = php_incarnations(layout)?; + service_calls_absent::(tx).await?; + let (copied, created) = adopt_incarnations::(tx, php_incarnations).await?; + if php_incarnations { + sqlx::raw_sql( + "DROP TABLE workflow_worker_registration_incarnations; \ + ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ + DROP INDEX wf_service_calls_deadline_idx; \ + ALTER TABLE workflow_service_calls DROP COLUMN call_deadline_at;", + ) + .execute(&mut *tx) + .await?; + } + let delivery_references = if layout == "php-2.9.7" { + // PHP's own queue delivery lookup metadata; native does not use PHP's + // queue, and the queue inspection bound every job to its task row. + let references = count::(tx, DELIVERY_REFERENCES).await?; + sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .execute(&mut *tx) + .await?; + references + } else { + 0 + }; + Ok(json!({ + "incarnations_copied":copied, "incarnations_created":created, + "queue_delivery_references_removed":delivery_references + })) +} + +const DELIVERY_REFERENCES: &str = + "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL"; + +/// Gives every registration the incarnation native expects. PHP's own +/// incarnations and tokens are copied, so a running SDK worker's fenced +/// deregistration still matches; an active registration without one gets a +/// new incarnation. Returns how many were copied and created. +async fn adopt_incarnations( + tx: &mut DB::Connection, + php_incarnations: bool, +) -> Result<(u64, usize)> +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ let now = super::super::store::now(); let mut copied = 0; if php_incarnations { @@ -331,8 +396,8 @@ where )); } } - // Every active registration gets the authority native expects. A PHP - // 2.8.0+ worker keeps the token it was given; an older one gets a new token. + // A PHP 2.8.0+ worker keeps the token it was given; an older one gets a + // new token. let missing = query::(if php_incarnations { "SELECT r.namespace,r.worker_id,r.registration_token AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" } else { @@ -354,35 +419,7 @@ where .execute(&mut *tx) .await?; } - if php_incarnations { - sqlx::raw_sql( - "DROP TABLE workflow_worker_registration_incarnations; \ - ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ - DROP INDEX wf_service_calls_deadline_idx; \ - ALTER TABLE workflow_service_calls DROP COLUMN call_deadline_at;", - ) - .execute(&mut *tx) - .await?; - } - let delivery_references = if layout == "php-2.9.7" { - // PHP's own queue delivery lookup metadata; native does not use PHP's - // queue, and the queue inspection bound every job to its task row. - let references = count::( - tx, - "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", - ) - .await?; - sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") - .execute(&mut *tx) - .await?; - references - } else { - 0 - }; - Ok(json!({ - "incarnations_copied":copied, "incarnations_created":missing.len(), - "queue_delivery_references_removed":delivery_references - })) + Ok((copied, missing.len())) } async fn count(tx: &mut DB::Connection, sql: &'static str) -> Result diff --git a/rust/src/runtime/upgrade/takeover/mysql_journal.rs b/rust/src/runtime/upgrade/takeover/mysql_journal.rs new file mode 100644 index 000000000..15b3d8ce5 --- /dev/null +++ b/rust/src/runtime/upgrade/takeover/mysql_journal.rs @@ -0,0 +1,397 @@ +//! MySQL and MariaDB takeover. Their DDL commits implicitly, and `LOCK +//! TABLES` refuses to create or drop the tables it does not hold, so the +//! takeover cannot be one transaction. It writes native's marker first, with +//! a `rust-takeover:` row whose version counts the completed steps +//! and whose checksum holds the backup fingerprint's digest. From then on PHP +//! refuses the database and native refuses it until the last step. Every step +//! is idempotent, so a rerun with the same fingerprint resumes after the last +//! recorded one. +use super::{ + BUDGET, DELIVERY_REFERENCES, Result, adopt_incarnations, count, matches_backup, + php_incarnations, preconditions, qualified, report, service_calls_absent, +}; +use crate::runtime::{ + mysql::{self, Flavor, LEDGER_SQL, MARKER_SQL, TAKEOVER_ENGINE}, + upgrade::{ + bounded, fingerprint, mysql_inventory, mysql_source, queue_binding::QueueSourceConfig, + refuse, + }, +}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{Connection, Executor, MySql, MySqlConnection, mysql::MySqlConnectOptions}; +use tokio::time::Instant; + +/// Completed-step counts recorded in the marker row. +const MARKED: i64 = 0; +/// Sessions rechecked and the source fingerprinted again after the marker: +/// nothing PHP wrote can be missing from the backup. +const FENCED: i64 = 1; +const LAST: i64 = 8; + +/// One takeover run: what its steps need and what its report says. +struct Run<'a> { + backend: &'static str, + flavor: Flavor, + digest: Vec, + layout: String, + fingerprint: &'a str, + queues: &'a QueueSourceConfig<'a>, + inventory: Value, + converted: serde_json::Map, +} + +enum Journal { + /// No marker: a PHP database. + Absent, + /// The marker table without its row: a takeover killed between the two. + Empty, + /// A takeover in progress, with its layout, step count and digest. + Takeover(String, i64, Vec), + /// Native's own database, or one native is still initializing. + Owned, +} + +pub async fn take_over_mysql( + options: MySqlConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let digest = fingerprint_digest(backup_fingerprint)?; + let deadline = Instant::now() + BUDGET; + // TIMESTAMP values render in UTC for the fingerprint, and lock waits stay + // short; the client deadline bounds whole-table work. + let (connection, backend) = mysql_source(options, deadline, true, 0).await?; + bounded(connection, deadline, async |connection| { + // The same session lock as native bootstrap, so neither a native node + // nor a second takeover can run on this database meanwhile. + let locked: Option = sqlx::query_scalar( + "SELECT GET_LOCK(CONCAT('dw-native-bootstrap:',SHA1(DATABASE())),5)", + ) + .fetch_one(&mut *connection) + .await?; + if locked != Some(1) { + return Err(refuse( + StatusCode::SERVICE_UNAVAILABLE, + "native_initialization_busy", + )); + } + let mut run = Run { + backend, + flavor: Flavor::detect(connection).await?, + digest, + layout: String::new(), + fingerprint: backup_fingerprint, + queues, + inventory: Value::Null, + converted: serde_json::Map::new(), + }; + let completed = match journal(connection).await? { + Journal::Absent => { + mark(connection, &mut run).await?; + MARKED + } + Journal::Empty => { + // Only a takeover adds the marker to a PHP database. Its layout + // is read again with the marker set aside. + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let layout = mysql_inventory(&mut tx, backend, Some(queues), true, true).await; + tx.rollback().await?; + run.layout = qualified(&layout?)?.to_owned(); + record(connection, &run).await?; + MARKED + } + Journal::Takeover(layout, completed, digest) => { + if digest != run.digest { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + php_incarnations(&layout)?; + run.layout = layout; + run.converted + .insert("resumed_after_step".into(), json!(completed)); + completed + } + Journal::Owned => { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_already_owned")); + } + }; + for step in completed + 1..=LAST { + perform(connection, &mut run, step).await?; + #[cfg(test)] + super::interruption_point(&format!("after-step-{step}")); + } + finish(connection, &run).await?; + mysql::verify(connection, run.flavor).await?; + let digest = json!({"fingerprint": run.fingerprint}); + Ok(report( + run.backend, + &run.layout, + &digest, + &run.inventory, + Value::Object(run.converted), + )) + }) + .await +} + +/// `sha256:` as the 32 bytes the marker row keeps. +fn fingerprint_digest(fingerprint: &str) -> Result> { + fingerprint["sha256:".len()..] + .as_bytes() + .chunks(2) + .map(|pair| { + std::str::from_utf8(pair) + .ok() + .and_then(|pair| u8::from_str_radix(pair, 16).ok()) + }) + .collect::>>() + .ok_or_else(|| { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_backup_fingerprint_required", + ) + }) +} + +async fn journal(connection: &mut MySqlConnection) -> Result { + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(TABLE_NAME)='dw_server_schema'") + .fetch_one(&mut *connection) + .await?; + if marked == 0 { + return Ok(Journal::Absent); + } + let mut rows: Vec<(String, i64, Vec)> = + sqlx::query_as("SELECT engine,version,bootstrap_checksum FROM dw_server_schema") + .fetch_all(&mut *connection) + .await?; + Ok(match rows.pop() { + None => Journal::Empty, + Some((engine, completed, digest)) if rows.is_empty() => { + match engine.strip_prefix(TAKEOVER_ENGINE) { + Some(layout) => Journal::Takeover(layout.to_owned(), completed, digest), + None => Journal::Owned, + } + } + Some(_) => Journal::Owned, + }) +} + +/// No other session may use the database. That needs the PROCESS privilege, +/// since without it MySQL shows only the account's own sessions. +async fn sessions_clear(connection: &mut MySqlConnection) -> Result<()> { + let visible: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM information_schema.USER_PRIVILEGES WHERE PRIVILEGE_TYPE='PROCESS')", + ) + .fetch_one(&mut *connection) + .await?; + if !visible { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sessions_unverifiable", + )); + } + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut *connection) + .await?; + if others != 0 { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_in_use")); + } + Ok(()) +} + +/// The inventory and fingerprint from one consistent snapshot, refused +/// unless the source is qualified and matches the backup. +async fn verified_snapshot( + connection: &mut MySqlConnection, + run: &Run<'_>, + fenced: bool, +) -> Result<(Value, String)> { + sessions_clear(connection).await?; + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let read = async { + let inventory = + mysql_inventory(&mut tx, run.backend, Some(run.queues), true, fenced).await?; + let digest = fingerprint::mysql_digest(&mut tx, run.backend, fenced).await?; + Ok::<_, crate::runtime::RuntimeError>((inventory, digest)) + } + .await; + tx.rollback().await?; + let (inventory, digest) = read?; + let layout = qualified(&inventory)?.to_owned(); + matches_backup(&digest, run.fingerprint)?; + php_incarnations(&layout)?; + service_calls_absent::(connection).await?; + Ok((inventory, layout)) +} + +/// The first write: native's marker with the takeover row, once the source +/// is verified. PHP refuses the database from here on. +async fn mark(connection: &mut MySqlConnection, run: &mut Run<'_>) -> Result<()> { + let (inventory, layout) = verified_snapshot(connection, run, false).await?; + run.inventory = inventory; + run.layout = layout; + connection.execute(MARKER_SQL).await?; + #[cfg(test)] + super::interruption_point("before-marker-row"); + record(connection, run).await +} + +async fn record(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + sqlx::query("INSERT INTO dw_server_schema(engine,version,bootstrap_checksum) VALUES (?,?,?)") + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(MARKED) + .bind(&run.digest[..]) + .execute(&mut *connection) + .await?; + Ok(()) +} + +/// Records one more completed step, in the caller's transaction or alone. +async fn advance<'c, E>(executor: E, run: &Run<'_>, step: i64) -> Result<()> +where + E: Executor<'c, Database = MySql>, +{ + let advanced = sqlx::query("UPDATE dw_server_schema SET version=? WHERE engine=? AND version=?") + .bind(step) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(step - 1) + .execute(executor) + .await? + .rows_affected(); + if advanced != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + Ok(()) +} + +async fn column_exists(connection: &mut MySqlConnection, table: &str, column: &str) -> Result { + let found: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND COLUMN_NAME=?") + .bind(table) + .bind(column) + .fetch_one(&mut *connection) + .await?; + Ok(found != 0) +} + +/// Each step checks what it has to do, so repeating an interrupted one is +/// harmless, and then records itself. +async fn perform(connection: &mut MySqlConnection, run: &mut Run<'_>, step: i64) -> Result<()> { + let php_incarnations = php_incarnations(&run.layout)?; + match step { + FENCED => { + // A session that was still open when the marker appeared could + // have written; none is left, and the source is still the backup. + let (inventory, layout) = verified_snapshot(connection, run, true).await?; + if layout != run.layout { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + run.inventory = inventory; + } + 2 => { + sqlx::raw_sql(sqlx::AssertSqlSafe(run.flavor.native_relations_sql()?)) + .execute(&mut *connection) + .await?; + let indexed: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='workflow_tasks' AND INDEX_NAME='dw_workflow_tasks_poll'") + .fetch_one(&mut *connection) + .await?; + if indexed == 0 { + connection.execute("ALTER TABLE workflow_tasks ADD KEY dw_workflow_tasks_poll (namespace(64),queue(64),task_type(16),status(16),available_at)").await?; + } + } + 3 => { + // Rows, unlike DDL, commit with the step that records them. + let mut tx = connection.begin().await?; + let (copied, created) = adopt_incarnations::(&mut tx, php_incarnations).await?; + advance(&mut *tx, run, step).await?; + tx.commit().await?; + run.converted + .insert("incarnations_copied".into(), json!(copied)); + run.converted + .insert("incarnations_created".into(), json!(created)); + return Ok(()); + } + 4 => { + if php_incarnations { + connection + .execute("DROP TABLE IF EXISTS workflow_worker_registration_incarnations") + .await?; + } + } + 5 => { + if column_exists(connection, "workflow_worker_registrations", "registration_token").await? { + connection + .execute("ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token") + .await?; + } + } + 6 => { + if column_exists(connection, "workflow_service_calls", "call_deadline_at").await? { + // Dropping only the column would leave its index on status alone. + connection.execute("ALTER TABLE workflow_service_calls DROP INDEX wf_service_calls_deadline_idx, DROP COLUMN call_deadline_at").await?; + } + } + 7 => { + if column_exists(connection, "workflow_tasks", "queue_delivery_reference").await? { + let references = count::(connection, DELIVERY_REFERENCES).await?; + run.converted + .insert("queue_delivery_references_removed".into(), json!(references)); + connection + .execute("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .await?; + } + } + 8 => { + if run.layout.ends_with("+mariadb-driver") { + // Laravel's mariadb connection made the backup hold owner a uuid. + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner char(36) DEFAULT NULL") + .await?; + } + } + _ => {} + } + advance(&mut *connection, run, step).await +} + +/// The ledger row and native's final marker commit together. +async fn finish(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + connection.execute(LEDGER_SQL).await?; + let migration = run.flavor.migration(); + let mut tx = connection.begin().await?; + sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES (?,?,TRUE,?,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) + .await?; + let marked = sqlx::query("UPDATE dw_server_schema SET engine='rust-development',version=?,bootstrap_checksum=? WHERE engine=? AND version=?") + .bind(mysql::VERSION) + .bind(run.flavor.checksum()) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(LAST) + .execute(&mut *tx) + .await? + .rows_affected(); + if marked != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + tx.commit().await?; + Ok(()) +} diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index 418448a34..e5a1ac522 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -53,11 +53,20 @@ case "$backend" in *) ;; esac fi -if [[ $backend == pgsql && -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then - docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ - --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ - -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ - >> "$evidence/database-cleanup.txt" 2>&1 +if [[ -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then +case "$backend" in + mysql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ + -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ + -e "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + pgsql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + *) ;; +esac fi if test -f "$evidence/roles-owned.txt" && test "$(cat "$evidence/roles-owned.txt")" = "$RESOURCE_SCOPE"; then if test "$backend" = mysql; then diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index b50f7c585..e41b52c0b 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# A separate published PHP cohort; inventory is not migration or SDK takeover. +# A separate published PHP cohort: inventory and refusals, a verified backup, +# and the takeover of a restored copy that native then resumes. set -euo pipefail test "$#" = 1 || exit 2 backend=$1 @@ -52,10 +53,9 @@ else chmod 0666 "$state/database.sqlite" fi printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" -# SQLite and PostgreSQL also take over a copy and resume the originals, so -# their timer is short. -timer_seconds=600 -if [[ $backend == sqlite || $backend == pgsql ]]; then timer_seconds=60; fi +# Every backend also takes over a copy and resumes the originals, so the +# timer is short. +timer_seconds=60 configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) php_run() { @@ -224,6 +224,16 @@ case "$backend" in target=(-e DB_CONNECTION=pgsql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" -e DB_USERNAME=parity -e DB_PASSWORD="$password") target_network=$RESOURCE_SCOPE ;; + mysql) + # The same backup restored once more, taken over as root, which can see + # every session (the PROCESS privilege). + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + mysql_admin mysql -e "CREATE DATABASE $takeover_database CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 2>> "$evidence/backup.stderr" + mysql_admin mysql "$takeover_database" < "$state/backup.sql" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=mysql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=root -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; *) ;; esac if (( ${#target[@]} )); then From d4a42d40fce19b0dbc129c1f0fb91dcdfda8c1f9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:46:35 +0000 Subject: [PATCH 11/20] Let the cohort's containers write the restored SQLite copy The runner's own user copies the backup, and the cohort's containers run as uid 1000, so the copy is made writable like the source file. Co-Authored-By: Claude Opus 5.5 --- scripts/conformance/server-parity/upgrade-source-profile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 5f06e41d5..c59ac5f06 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -184,6 +184,7 @@ case "$backend" in sqlite) # A stopped SQLite installation is backed up by copying its file. cp "$state/database.sqlite" "$state/restored.sqlite" + chmod 0666 "$state/restored.sqlite" restored=/state/restored.sqlite change_restored() { php_run -r '(new PDO("sqlite:/state/restored.sqlite"))->exec("UPDATE jobs SET attempts=attempts+1") === 1 || exit(1);' From 376ca25c9db786292b1f54de3a8be84e14e30cb6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:46:54 +0000 Subject: [PATCH 12/20] Let the cohort's containers write the SQLite takeover copy Like the restored backup, the runner's user makes the copy that the uid 1000 takeover and native containers write. Co-Authored-By: Claude Opus 5.5 --- scripts/conformance/server-parity/upgrade-source-profile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 198892a23..06a1dd4ec 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -211,6 +211,7 @@ if test "$backend" = sqlite; then # live database: its digest is the one the restored backup matched. Native # then resumes the originals, and PHP must refuse the upgraded file. cp "$state/database.sqlite" "$state/takeover.sqlite" + chmod 0666 "$state/takeover.sqlite" docker run --rm --name "$resource-takeover" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_RUST_EXPERIMENTAL=1 \ -e DW_UPGRADE_BACKUP_FINGERPRINT="$source_fingerprint" \ From 48ced5bb1ff31368bbb69232ca1fedd3bf2f9702 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:56:32 +0000 Subject: [PATCH 13/20] Accept MariaDB's backend name in the cohort's takeover check The cohort's mysql backend runs on MariaDB too, where native reports the takeover backend as mariadb. Co-Authored-By: Claude Opus 5.5 --- scripts/conformance/server-parity/upgrade-source-profile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index df69ac0e4..23c3b2f07 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -248,7 +248,8 @@ if (( ${#target[@]} )); then --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-takeover \ > "$evidence/takeover.json" 2> "$evidence/takeover.stderr" jq -e --arg backend "$backend" --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" - and .backend==$backend and .source_layout==$layout and .fingerprint==$fingerprint + and (.backend==$backend or ($backend=="mysql" and .backend=="mariadb")) + and .source_layout==$layout and .fingerprint==$fingerprint and .backup_fingerprint_matched==true and .converted.incarnations_copied==(if $layout=="frozen" then 0 else 1 end) and .stored_representation_compatibility_verified==false' "$evidence/takeover.json" >/dev/null From 3f01df92a6b194d335ff874d492cd55af3513a0d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:59:12 +0000 Subject: [PATCH 14/20] Resume the two PHP originals in whichever order native offers them The source timer fires 60 s after it was authored. On a slow host the cohort reaches the resume after that, so the timer workflow's task is ready before the resumed activity workflow's and was claimed first. The resume now accepts each original workflow's task in either order and checks the same histories, deadline and stale attempt. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/upgrade-takeover-resume.php | 79 +++++++++++-------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index 55492f72f..bddd3db37 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -54,24 +54,58 @@ $queue = $originals['activity']['workflow_task']['task_queue'] ?? 'upgrade-source'; $result = ['codec' => 'avro', 'blob' => Avro::serialize(['resumed under native'])]; $outcome = []; +$activity = $originals['activity']; +$timer = $originals['timer']; +$complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { + $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], + 'workflow_task_attempt' => $claimed['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, $operation); +}; + +// The timer scheduled under PHP keeps its history and deadline. +$scheduled = array_values(array_filter($timer['history'], static fn (array $event): bool => $event['event_type'] === 'TimerScheduled'))[0]; +$fireAt = $scheduled['payload']['fire_at']; +$check($keep($timer['history'], $history($timer)), 'Original timer history changed'); +$deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); +$check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); // The activity leased under PHP completes with its original attempt. -$activity = $originals['activity']; $task = $activity['activity_task']; $completion = ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], 'result' => $result]; $completed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'original activity completion'); $check(($completed['outcome'] ?? null) === 'completed' && ($completed['recorded'] ?? null) === true, 'Original activity completion was not recorded'); $replayed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'activity completion replay'); $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); -$resume = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, - 'timeout_seconds' => 5]), 200, 'resume claim')['task']; -$check(is_array($resume) && $resume['run_id'] === $activity['run_id'], 'The resumed workflow task was not claimed'); + +// Each original workflow then gets one workflow task: the activity's at +// once and the timer's at its deadline. On a slow host the timer is already +// due, so either may be claimed first. $original = $activity['history']; -$check(array_slice($types($resume['history_events']), 0, count($original)) === $types($original), 'Original activity history changed in the resumed task'); -$check(end($resume['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); -$expect($request('POST', '/api/worker/workflow-tasks/'.$resume['task_id'].'/complete', ['lease_owner' => $resume['lease_owner'], - 'workflow_task_attempt' => $resume['workflow_task_attempt'], - 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, 'resumed workflow completion'); +$claims = []; +$until = max($deadline, microtime(true)) + 30; +while (count($claims) < 2 && microtime(true) < $until) { + $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, + 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; + if (! is_array($claimed)) { + continue; + } + $run = $claimed['run_id']; + $check(in_array($run, [$activity['run_id'], $timer['run_id']], true) && ! isset($claims[$run]), + 'An unexpected workflow task was claimed'); + $claims[$run] = true; + if ($run === $activity['run_id']) { + $check(array_slice($types($claimed['history_events']), 0, count($original)) === $types($original), + 'Original activity history changed in the resumed task'); + $check(end($claimed['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); + $complete($claimed, 'resumed workflow completion'); + } else { + $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); + $complete($claimed, 'timer workflow completion'); + } +} +$check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); +$check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); + // The workflow task PHP already completed is a stale attempt. $stale = $activity['workflow_task']; $refused = $request('POST', '/api/worker/workflow-tasks/'.$stale['task_id'].'/complete', ['lease_owner' => $stale['lease_owner'], @@ -83,31 +117,14 @@ $outcome['activity'] = ['run_id' => $activity['run_id'], 'events' => array_column($closed, 'event_type'), 'stale_attempt_status' => $refused[0], 'stale_attempt_reason' => $refused[1]['reason'] ?? null]; -// The timer scheduled under PHP keeps its deadline and fires under native. -$timer = $originals['timer']; -$scheduled = array_values(array_filter($timer['history'], static fn (array $event): bool => $event['event_type'] === 'TimerScheduled'))[0]; -$fireAt = $scheduled['payload']['fire_at']; -$current = $history($timer); -$check($keep($timer['history'], $current), 'Original timer history changed'); -$deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); -$check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); -$fired = null; -while ($fired === null && microtime(true) < $deadline + 30) { - $poll = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, - 'timeout_seconds' => 5]), 200, 'timer claim'); - $fired = $poll['task'] ?? null; -} -$check(is_array($fired) && $fired['run_id'] === $timer['run_id'], 'The original timer did not fire under native'); -$firedEvent = end($fired['history_events']); -$check($firedEvent['event_type'] === 'TimerFired', 'Timer firing is missing from history'); -$firedPublic = array_values(array_filter($history($timer), static fn (array $event): bool => $event['event_type'] === 'TimerFired')); +$timerHistory = $history($timer); +$firedPublic = array_values(array_filter($timerHistory, static fn (array $event): bool => $event['event_type'] === 'TimerFired')); $check(count($firedPublic) === 1, 'The original timer did not fire exactly once'); $firedAt = (float) (new DateTimeImmutable($firedPublic[0]['timestamp']))->format('U.u'); $check($firedAt >= $deadline, 'The original timer fired before its deadline'); -$expect($request('POST', '/api/worker/workflow-tasks/'.$fired['task_id'].'/complete', ['lease_owner' => $fired['lease_owner'], - 'workflow_task_attempt' => $fired['workflow_task_attempt'], - 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, 'timer workflow completion'); +$check($keep($timer['history'], $timerHistory) && end($timerHistory)['event_type'] === 'WorkflowCompleted', + 'The timer workflow did not complete on its original history'); $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, - 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($history($timer), 'event_type')]; + 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, JSON_THROW_ON_ERROR)."\n"; From 132755a476ee2b257f8fea98a3a3425af452d1ab Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 12:33:35 +0000 Subject: [PATCH 15/20] Prepare a PHP SQLite source in WAL mode for the upgrade commands PHP Server keeps SQLite in WAL mode, and inspection, fingerprint and takeover need the rollback journal, so a default installation could not be upgraded. upgrade-prepare-sqlite, run once PHP is stopped, opens the file as its only connection and switches it to the rollback journal: SQLite checkpoints every committed WAL frame into the file first, so the stored rows are unchanged. It refuses with upgrade_source_in_use while another process has the file open, and confirms the header and that no WAL, shared-memory or journal file remains. A journal left by a crashed writer is rolled back the same way, and a source already in rollback mode is left byte-identical. The upgrade source cohort now runs its SQLite source in PHP's default WAL mode and prepares it after PHP stops, as an operator does. Co-Authored-By: Claude Opus 5.5 --- rust/README.md | 7 + rust/src/main.rs | 22 +++ rust/src/runtime/upgrade.rs | 146 +++++++++++++++--- rust/src/runtime/upgrade_tests.rs | 89 +++++++++++ .../server-parity/upgrade-source-cleanup.sh | 2 +- .../server-parity/upgrade-source-profile.sh | 16 +- 6 files changed, 258 insertions(+), 24 deletions(-) diff --git a/rust/README.md b/rust/README.md index 36b9eee29..679d895d9 100644 --- a/rust/README.md +++ b/rust/README.md @@ -418,6 +418,13 @@ over with the development `upgrade-takeover` command, following the 1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and stop every PHP role. + PHP Server keeps SQLite in WAL mode, which the upgrade commands refuse. With + PHP stopped, run `upgrade-prepare-sqlite` on the source file (writable). + It has SQLite checkpoint every committed WAL frame into the file and switch + it to the rollback journal, refusing with `upgrade_source_in_use` while any + other process has the file open. The rows are unchanged; the backup below + is taken afterwards. A journal left by a crashed writer is rolled back the + same way. 2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, or the database through a read-only account. Back the database up (a file copy, `pg_dump` or `mysqldump --single-transaction`), restore the backup diff --git a/rust/src/main.rs b/rust/src/main.rs index 56a8fe465..ae8efabcb 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -133,12 +133,34 @@ async fn main() -> Result<(), Box> { | "upgrade-queue-inspect" | "upgrade-payload-inspect" | "upgrade-fingerprint" + | "upgrade-prepare-sqlite" | "upgrade-takeover" ) { return Err("unknown development command".into()); } let backend = env::var("DB_CONNECTION").unwrap_or_default(); + if command == "upgrade-prepare-sqlite" { + if backend != "sqlite" { + return Err("upgrade-prepare-sqlite requires DB_CONNECTION=sqlite".into()); + } + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + return match durable_workflow_server::runtime::upgrade::prepare_sqlite_source(&path).await + { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_prepare_unavailable", + }; + eprintln!("{reason}"); + Err("SQLite source preparation refused".into()) + } + }; + } if command == "upgrade-takeover" { use durable_workflow_server::runtime::upgrade::{ queue_binding::QueueSourceConfig, takeover, diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index ef6baf642..b5ac24214 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -407,6 +407,128 @@ pub(super) async fn open_sqlite_source(path: &str, deadline: Instant) -> Result< open_sqlite_file(path, deadline, false).await } +/// The file format version bytes of a SQLite header: rollback journal or WAL. +const ROLLBACK_JOURNAL: [u8; 2] = [1, 1]; +const WRITE_AHEAD_LOG: [u8; 2] = [2, 2]; + +/// The journal format bytes of a regular SQLite file, read without SQLite. +async fn journal_format(path: &str) -> Result<[u8; 2]> { + let regular_file_required = || { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_regular_file_required", + ) + }; + if !tokio::fs::metadata(path).await?.is_file() { + return Err(regular_file_required()); + } + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + // Tokio filesystem opens use blocking workers. A replaced FIFO must + // not strand one beyond the observation budget or prevent CLI exit. + #[cfg(unix)] + options.custom_flags(libc::O_NONBLOCK); + let mut file = options.open(path).await?; + if !file.metadata().await?.is_file() { + return Err(regular_file_required()); + } + let mut header = [0_u8; 20]; + file.read_exact(&mut header).await?; + if &header[..16] != b"SQLite format 3\0" { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok([header[18], header[19]]) +} + +/// Whether a SQLite file has a WAL, shared-memory or rollback journal beside it. +async fn sidecars(path: &str) -> Result { + for suffix in ["-wal", "-shm", "-journal"] { + if tokio::fs::try_exists(format!("{path}{suffix}")).await? { + return Ok(true); + } + } + Ok(false) +} + +/// Prepares a stopped PHP SQLite source for the upgrade commands, which need +/// the rollback journal. PHP Server's default is WAL: switching it makes +/// SQLite checkpoint every committed WAL frame into the file first, so the +/// stored rows are unchanged. A journal left by a crashed writer is rolled +/// back the same way. Refuses while another process has the file open. +pub async fn prepare_sqlite_source(path: &str) -> Result { + if path.is_empty() || path.contains(['?', ':']) { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_file_required", + )); + } + let deadline = Instant::now() + BUDGET; + let previous = match timeout_at(deadline, journal_format(path)) + .await + .map_err(|_| expired())?? + { + WRITE_AHEAD_LOG => "wal", + ROLLBACK_JOURNAL => "delete", + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + }; + let switched = previous == "wal" || sidecars(path).await?; + if switched { + let in_use = || refuse(StatusCode::CONFLICT, "upgrade_source_in_use"); + let opened = timeout_at( + deadline, + SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(path) + .create_if_missing(false) + .foreign_keys(false) + .busy_timeout(Duration::ZERO) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete), + ), + ) + .await + .map_err(|_| expired())?; + // Leaving WAL needs the only connection; SQLite answers busy otherwise. + let mut connection = match opened { + Ok(connection) => connection, + Err(sqlx::Error::Database(error)) + if error + .code() + .and_then(|code| code.parse::().ok()) + .is_some_and(|code| code & 0xff == 5) => + { + return Err(in_use()); + } + Err(error) => return Err(error.into()), + }; + let mode: String = sqlx::query_scalar("PRAGMA journal_mode") + .fetch_one(&mut connection) + .await?; + close(connection.close()).await?; + if mode != "delete" { + return Err(in_use()); + } + } + if journal_format(path).await? != ROLLBACK_JOURNAL || sidecars(path).await? { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok(json!({ + "schema":"durable-workflow.rust-upgrade-sqlite-prepare.v1", + "status":"rollback_journal", "previous_journal_mode":previous, + "source_mutated":switched, "takeover_allowed":false, "backup_verified":false + })) +} + /// Opens a qualified rollback-journal source file, writable only for takeover. /// It keeps the rollback journal: native enables WAL itself after takeover. pub(super) async fn open_sqlite_file( @@ -424,29 +546,9 @@ pub(super) async fn open_sqlite_file( // not open an unqualified WAL source or attempt journal recovery. Unlike // immutable=1 this preserves normal SQLite locking/change detection for // the supported rollback-journal source, rather than ignoring live WAL. + // A WAL source is first switched with upgrade-prepare-sqlite. timeout_at(deadline, async { - let regular_file_required = || { - refuse( - StatusCode::BAD_REQUEST, - "upgrade_source_regular_file_required", - ) - }; - if !tokio::fs::metadata(path).await?.is_file() { - return Err(regular_file_required()); - } - let mut options = tokio::fs::OpenOptions::new(); - options.read(true); - // Tokio filesystem opens use blocking workers. A replaced FIFO must - // not strand one beyond the observation budget or prevent CLI exit. - #[cfg(unix)] - options.custom_flags(libc::O_NONBLOCK); - let mut file = options.open(path).await?; - if !file.metadata().await?.is_file() { - return Err(regular_file_required()); - } - let mut header = [0_u8; 20]; - file.read_exact(&mut header).await?; - if &header[..16] != b"SQLite format 3\0" || header[18..20] != [1, 1] { + if journal_format(path).await? != ROLLBACK_JOURNAL { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_sqlite_journal_unqualified", diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index a140ec2ba..7dee18909 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -1054,3 +1054,92 @@ async fn takeover_refusals_leave_the_php_database_unchanged() { assert_eq!(std::fs::read(&path).unwrap(), before, "{expected}"); } } + +/// The frozen layout in PHP Server's default WAL mode, copied while its +/// writer was still open: one committed row is only in the WAL, as a crashed +/// PHP would leave it. +async fn wal_source() -> (tempfile::TempDir, String) { + let (directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer.execute("PRAGMA wal_autocheckpoint=0").await.unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('only-in-wal',7)") + .await + .unwrap(); + let copy = directory.path().join("wal.sqlite").to_str().unwrap().to_owned(); + for suffix in ["", "-wal", "-shm"] { + std::fs::copy(format!("{path}{suffix}"), format!("{copy}{suffix}")).unwrap(); + } + writer.close().await.unwrap(); + (directory, copy) +} + +#[tokio::test] +async fn prepare_switches_a_wal_source_to_the_rollback_journal_and_keeps_its_rows() { + let (_directory, path) = wal_source().await; + assert!(Path::new(&format!("{path}-wal")).exists()); + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["status"], "rollback_journal", "{report}"); + assert_eq!(report["previous_journal_mode"], "wal"); + assert_eq!(report["source_mutated"], true); + for suffix in ["-wal", "-shm", "-journal"] { + assert!(!Path::new(&format!("{path}{suffix}")).exists(), "{suffix}"); + } + assert_eq!(journal_format(&path).await.unwrap(), ROLLBACK_JOURNAL); + let inventory = inspect_sqlite(&path).await.unwrap(); + assert_eq!(inventory["table_rows"]["migrations"], 1, "{inventory}"); + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}?mode=ro")) + .await + .unwrap(); + let batch: i64 = sqlx::query_scalar("SELECT batch FROM migrations WHERE migration='only-in-wal'") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(batch, 7); + connection.close().await.unwrap(); + // Preparing again changes nothing. + let original = std::fs::read(&path).unwrap(); + assert_eq!(prepare_sqlite_source(&path).await.unwrap()["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} + +#[tokio::test] +async fn prepare_refuses_while_another_connection_holds_the_source() { + let (_directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('open',8)") + .await + .unwrap(); + reason(prepare_sqlite_source(&path).await, "upgrade_source_in_use"); + assert_eq!(journal_format(&path).await.unwrap(), WRITE_AHEAD_LOG); + writer.close().await.unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "wal", "{report}"); +} + +#[tokio::test] +async fn prepare_leaves_a_rollback_journal_source_byte_identical() { + let (_directory, path) = php_layout().await; + let original = std::fs::read(&path).unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "delete", "{report}"); + assert_eq!(report["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index e5a1ac522..bd2b67148 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -15,7 +15,7 @@ source=upgrade_source_ref${label:+_$label} restored=upgrade_restore_ref${label:+_$label} image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 -for role in author control inspect php takeover native resume; do +for role in author control inspect php prepare takeover native resume; do if docker inspect "$resource-$role" >/dev/null 2>&1; then docker stop --signal SIGTERM --time 5 "$resource-$role" >/dev/null docker rm "$resource-$role" >/dev/null diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 23c3b2f07..3613cf5bd 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -57,7 +57,7 @@ printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" # timer is short. timer_seconds=60 configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token - -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) + -e CACHE_STORE=file -e QUEUE_CONNECTION=database) php_run() { docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" "${configuration[@]}" -v "$PWD/$state:/state" -v "$PWD:/source:ro" \ @@ -81,6 +81,20 @@ docker run --rm --name "$resource-author" --user=1000:1000 --cpus=1 --memory=512 docker stop --signal SIGTERM --time 10 "$resource-php" >/dev/null docker inspect --format '{{json .State}}' "$resource-php" > "$evidence/stopped-state.json" jq -e '.Running==false and .ExitCode==0 and .OOMKilled==false' "$evidence/stopped-state.json" >/dev/null +if test "$backend" = sqlite; then + # PHP Server keeps SQLite in WAL mode. Once PHP is stopped, the source is + # switched to the rollback journal the upgrade commands need, as an operator + # does first; its rows are unchanged. + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 2 2" ]] + docker run --rm --name "$resource-prepare" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/database.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-prepare-sqlite \ + > "$evidence/prepare.json" 2> "$evidence/prepare.stderr" + jq -e '.status=="rollback_journal" and .previous_journal_mode=="wal" and .source_mutated==true' \ + "$evidence/prepare.json" >/dev/null + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 1 1" ]] +fi if test "$backend" = sqlite; then docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network=none -v "$PWD/$state:/state:ro" -v "$PWD:/source:ro" --entrypoint php "$image" \ From d3c3c34ebd961c6836786f36cc706b59928ca75b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 13:44:46 +0000 Subject: [PATCH 16/20] Let SQLite source preparation wait briefly for a closing writer With no lock wait, a writer that had just closed could still hold its lock when the preparation opened the file, and the preparation refused it as in use. A two-second wait covers that; a file another process keeps open is still refused. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index b5ac24214..e3f3f7b44 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -489,13 +489,15 @@ pub async fn prepare_sqlite_source(path: &str) -> Result { .filename(path) .create_if_missing(false) .foreign_keys(false) - .busy_timeout(Duration::ZERO) + .busy_timeout(Duration::from_secs(2)) .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete), ), ) .await .map_err(|_| expired())?; - // Leaving WAL needs the only connection; SQLite answers busy otherwise. + // Leaving WAL needs the only connection. SQLite answers busy when another + // still has the file after a short wait, which also covers a writer + // that is just closing. let mut connection = match opened { Ok(connection) => connection, Err(sqlx::Error::Database(error)) From fa7695fd38810f7329fa6b94a907c6deb4ce6635 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 13:45:02 +0000 Subject: [PATCH 17/20] Resume a PHP-accepted signal after takeover and refuse unsupported features The upgrade source author now also writes a third original: a workflow that opens a signal wait, then receives its declared signal under published PHP, whose resume task stays unclaimed. After the takeover, native must deliver that signal from PHP's rows: the resumed task keeps the original history and the signal's PHP-written arguments, and the workflow completes. The resume accepts the three originals in any order. The payload inspection now also checks the codec-tagged payloads of workflow commands, signal records and updates. Takeover refuses a source holding rows of a published PHP feature native does not implement yet (runtime credentials, message streams, service definitions, update validation tasks, worker sessions, build ID rollouts and search attribute definitions), rather than leaving them unread. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade/payload_scan.rs | 11 +++-- rust/src/runtime/upgrade/takeover.rs | 40 ++++++++++++++++++- rust/src/runtime/upgrade_tests.rs | 15 ++++++- .../server-parity/upgrade-source-author.php | 36 ++++++++++++----- .../server-parity/upgrade-source-profile.sh | 14 +++---- .../upgrade-source-sqlite-representations.php | 6 +-- .../server-parity/upgrade-takeover-resume.php | 26 +++++++++--- 7 files changed, 118 insertions(+), 30 deletions(-) diff --git a/rust/src/runtime/upgrade/payload_scan.rs b/rust/src/runtime/upgrade/payload_scan.rs index 555eea3b2..412c2b91e 100644 --- a/rust/src/runtime/upgrade/payload_scan.rs +++ b/rust/src/runtime/upgrade/payload_scan.rs @@ -72,11 +72,15 @@ macro_rules! column { }; } -const COLUMNS: [Column; 4] = [ +const COLUMNS: [Column; 8] = [ column!("workflow_runs", "arguments", "payload_codec"), column!("workflow_runs", "output", "output_payload_codec"), column!("activity_executions", "arguments", "payload_codec"), column!("activity_executions", "result", "payload_codec"), + column!("workflow_commands", "payload", "payload_codec"), + column!("workflow_signal_records", "arguments", "payload_codec"), + column!("workflow_updates", "arguments", "payload_codec"), + column!("workflow_updates", "result", "payload_codec"), ]; #[derive(Default, Serialize)] @@ -218,9 +222,10 @@ where "references_metadata_supported":references_supported, "diagnostics":diagnostics, "diagnostics_omitted":omitted, "inline_values_verified":inline_supported, "nested_payloads_verified":false, - // Published migrations also give these tables codec-tagged payloads. + // These tables also hold codec-tagged payloads; takeover refuses a + // source with any of their rows, since native lacks the features. "other_root_payload_columns_verified":false, - "unscanned_payload_codec_tables":["workflow_commands","workflow_signal_records","workflow_updates"], + "unscanned_payload_codec_tables":["workflow_durable_stream_items","workflow_inbound_stream_items","workflow_update_validation_tasks"], "inline_validation_limits":{"encoded_frame_bytes":65536,"traversal_depth":64, "traversal_visits":16384,"scalar_allocation_bytes":apache_avro::util::max_allocation_bytes(apache_avro::util::DEFAULT_MAX_ALLOCATION_BYTES)}, "provider_bytes_verified":false, "registry_binding_verified":false, diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index 2eb10c25d..0d9188b83 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -218,10 +218,46 @@ fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, conver }) } +/// Published PHP features native does not implement yet. A source holding any +/// of their rows is refused, rather than taken over without them. Derived +/// projections and PHP's own queue and poll bookkeeping are left in place. +const UNSUPPORTED_FEATURES: [(&str, &[&str]); 7] = [ + ("upgrade_source_runtime_credentials_unsupported", &["runtime_credentials"]), + ( + "upgrade_source_message_streams_unsupported", + &[ + "workflow_messages", + "workflow_durable_streams", + "workflow_durable_stream_items", + "workflow_inbound_streams", + "workflow_inbound_stream_items", + ], + ), + ( + "upgrade_source_services_unsupported", + &["workflow_services", "workflow_service_endpoints", "workflow_service_operations"], + ), + ("upgrade_source_update_validation_unsupported", &["workflow_update_validation_tasks"]), + ("upgrade_source_worker_sessions_unsupported", &["workflow_worker_sessions"]), + ("upgrade_source_build_id_rollouts_unsupported", &["workflow_worker_build_id_rollouts"]), + ( + "upgrade_source_search_attribute_definitions_unsupported", + &["search_attribute_definitions"], + ), +]; + /// Takeover needs every inspection result it relies on, and refuses what -/// native cannot represent yet: external payloads and service calls. -/// Returns the qualified source layout. +/// native cannot represent yet: external payloads, service calls and the +/// other unsupported features above. Returns the qualified source layout. fn qualified(inventory: &Value) -> Result<&str> { + for (reason, tables) in UNSUPPORTED_FEATURES { + if tables + .iter() + .any(|table| inventory["table_rows"][*table].as_i64().unwrap_or(0) != 0) + { + return Err(refuse(StatusCode::CONFLICT, reason)); + } + } if inventory["queue_inspection"]["status"] != "known_subset_bound" { return Err(refuse( StatusCode::CONFLICT, diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 7dee18909..35e563a5a 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -406,7 +406,11 @@ async fn payload_scan_validates_inline_pages_and_reports_late_bad_values_without assert_eq!(report["payload_inspection"]["other_root_payload_columns_verified"], false); assert_eq!( report["payload_inspection"]["unscanned_payload_codec_tables"], - json!(["workflow_commands", "workflow_signal_records", "workflow_updates"]) + json!([ + "workflow_durable_stream_items", + "workflow_inbound_stream_items", + "workflow_update_validation_tasks" + ]) ); assert_eq!(report["payload_inspection"]["diagnostics"], json!([])); for (table, column, count) in [ @@ -1042,6 +1046,15 @@ async fn takeover_refusals_leave_the_php_database_unchanged() { "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", "upgrade_source_external_payloads_unsupported", ), + // Features native lacks: their rows would be left behind unread. + ( + "INSERT INTO runtime_credentials(id,subject,roles,tenant,token_prefix,token_hash) VALUES ('c','worker','[]','default','dw_','h')", + "upgrade_source_runtime_credentials_unsupported", + ), + ( + "INSERT INTO workflow_messages(workflow_instance_id,direction,channel,stream_key,sequence) VALUES ('w','inbound','c','k',1)", + "upgrade_source_message_streams_unsupported", + ), ] { let (_directory, path) = php297_layout().await; mutate(&path, statement).await; diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index cae370f95..c16237701 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,12 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal'], + // The signal original declares its one signal, as an SDK worker would. + 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ + 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, + 'variadic' => false, 'allows_null' => false, 'default_available' => false]], + ]]]], 'supported_activity_types' => ['parity.upgrade_activity'], 'capabilities' => [], 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_protocol_author']), @@ -54,7 +59,7 @@ $timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); $check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer'] as $kind) { +foreach (['activity', 'timer', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -63,11 +68,13 @@ 'started_at' => $started, 'original_deadline' => $deadline], JSON_THROW_ON_ERROR)."\n"); $task = $expect($request('POST', '/api/worker/workflow-tasks/poll', $poll), 200, $kind.' claim')['task']; $check($task['run_id'] === $run['run_id'], 'Original workflow claim changed'); - $command = $kind === 'activity' - ? ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, + $command = match ($kind) { + 'activity' => ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], - 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900] - : ['type' => 'start_timer', 'delay_seconds' => $timerSeconds]; + 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], + 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], + 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + }; $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'commands' => [$command], @@ -77,10 +84,21 @@ $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; $check($activity['run_id'] === $run['run_id'] && $activity['attempt_number'] === 1, 'Original activity changed'); } + if ($kind === 'signal') { + // PHP accepts the signal and makes its resume task, which stays + // unclaimed until native resumes it after the takeover. + $signalled = $request('POST', '/api/workflows/'.$id.'/signal/go', ['input' => $envelope]); + $check(in_array($signalled[0], [200, 202], true), 'signal: '.json_encode(['status' => $signalled[0], + 'reason' => $signalled[1]['reason'] ?? null, 'errors' => $signalled[1]['errors'] ?? null])); + } $history = $expect($request('GET', '/api/workflows/'.$id.'/runs/'.$run['run_id'].'/history'), 200, 'history')['events']; - $check(array_column($history, 'event_type') === ($kind === 'activity' - ? ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'] - : ['StartAccepted', 'WorkflowStarted', 'TimerScheduled']), 'Original pending history differs'); + fwrite(STDERR, json_encode(['phase' => 'history', 'kind' => $kind, 'events' => array_column($history, 'event_type')], + JSON_THROW_ON_ERROR)."\n"); + $check(array_column($history, 'event_type') === match ($kind) { + 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], + 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], + 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; } diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 3613cf5bd..e7a504c98 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -101,8 +101,8 @@ if test "$backend" = sqlite; then /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==2 and (.tasks|length)==4 and (.activities|length)==1 - and (.history|length)==7 and (.queue_jobs|length)==1 + and (.runs|length)==3 and (.tasks|length)==6 and (.activities|length)==1 + and (.history|length)==11 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -134,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==2 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -145,10 +145,10 @@ jq -e '.status=="inspected" and .queue_inspection.status=="known_subset_bound" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/queue-inspection.json" >/dev/null inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> "$evidence/payload-inspection.stderr" -jq -e '.status=="inspected" and (.payload_inspection.columns|length)==4 +jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==3 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==9 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -166,7 +166,7 @@ fingerprint() { } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) - and (.table_rows|length)==50 and .table_rows.workflow_runs==2 and .table_rows.jobs==1 + and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") @@ -285,7 +285,7 @@ if (( ${#target[@]} )); then "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 - and .timer.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index da17fe171..b36f69b1d 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 3')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 4')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the two isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the three isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index bddd3db37..2b695dc8d 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -56,6 +56,7 @@ $outcome = []; $activity = $originals['activity']; $timer = $originals['timer']; +$signal = $originals['signal']; $complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], 'workflow_task_attempt' => $claimed['workflow_task_attempt'], @@ -77,20 +78,20 @@ $replayed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'activity completion replay'); $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); -// Each original workflow then gets one workflow task: the activity's at -// once and the timer's at its deadline. On a slow host the timer is already -// due, so either may be claimed first. +// Each original workflow then gets one workflow task: the activity's and the +// signal's at once, the timer's at its deadline. On a slow host the timer is +// already due, so they may be claimed in any order. $original = $activity['history']; $claims = []; $until = max($deadline, microtime(true)) + 30; -while (count($claims) < 2 && microtime(true) < $until) { +while (count($claims) < 3 && microtime(true) < $until) { $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; if (! is_array($claimed)) { continue; } $run = $claimed['run_id']; - $check(in_array($run, [$activity['run_id'], $timer['run_id']], true) && ! isset($claims[$run]), + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id']], true) && ! isset($claims[$run]), 'An unexpected workflow task was claimed'); $claims[$run] = true; if ($run === $activity['run_id']) { @@ -98,12 +99,23 @@ 'Original activity history changed in the resumed task'); $check(end($claimed['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); $complete($claimed, 'resumed workflow completion'); + } elseif ($run === $signal['run_id']) { + // The signal PHP accepted is delivered from PHP's rows, with its arguments. + $check(array_slice($types($claimed['history_events']), 0, count($signal['history'])) === $types($signal['history']), + 'Original signal history changed in the resumed task'); + $received = array_values(array_filter($claimed['history_events'], + static fn (array $event): bool => $event['event_type'] === 'SignalReceived')); + $check(count($received) === 1 && json_encode($sorted($received[0]['payload']['arguments']), JSON_THROW_ON_ERROR) + === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), + 'The PHP-accepted signal arguments differ under native'); + $complete($claimed, 'signal workflow completion'); } else { $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); $complete($claimed, 'timer workflow completion'); } } $check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); +$check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); $check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); // The workflow task PHP already completed is a stale attempt. @@ -124,6 +136,10 @@ $check($firedAt >= $deadline, 'The original timer fired before its deadline'); $check($keep($timer['history'], $timerHistory) && end($timerHistory)['event_type'] === 'WorkflowCompleted', 'The timer workflow did not complete on its original history'); +$signalHistory = $history($signal); +$check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', + 'The signal workflow did not complete on its original history'); +$outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, From 6978d60b137a26c70e946a1ad33a4dafa28c3c32 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 14:43:53 +0000 Subject: [PATCH 18/20] Redeliver a PHP lease that expires after the takeover The upgrade source author adds a fourth original whose worker stops after claiming its workflow task. PHP's lease on that task expires after the takeover. Native must then redeliver the same task as its next attempt, refuse the expired PHP attempt's completion, and complete the workflow on its original history. Part of #325. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/upgrade-source-author.php | 20 ++++++--- .../server-parity/upgrade-source-profile.sh | 14 +++--- .../upgrade-source-sqlite-representations.php | 6 +-- .../server-parity/upgrade-takeover-resume.php | 44 ++++++++++++++++--- 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index c16237701..fa85db118 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,8 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal', + 'parity.upgrade_lease'], // The signal original declares its one signal, as an SDK worker would. 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, @@ -59,7 +60,8 @@ $timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); $check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer', 'signal'] as $kind) { +// The lease original claims before the signal original makes its resume task. +foreach (['activity', 'timer', 'lease', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -74,11 +76,16 @@ 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + // The lease original's worker stops after its claim. PHP's workflow + // task lease expires after the takeover, and native redelivers it. + 'lease' => null, }; - $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ - 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], - 'commands' => [$command], - ]), 200, $kind.' command'); + if ($command !== null) { + $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command], + ]), 200, $kind.' command'); + } $activity = null; if ($kind === 'activity') { $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; @@ -98,6 +105,7 @@ 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + 'lease' => ['StartAccepted', 'WorkflowStarted'], }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index e7a504c98..b5f340ef0 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -101,8 +101,8 @@ if test "$backend" = sqlite; then /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==3 and (.tasks|length)==6 and (.activities|length)==1 - and (.history|length)==11 and (.queue_jobs|length)==1 + and (.runs|length)==4 and (.tasks|length)==7 and (.activities|length)==1 + and (.history|length)==13 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -134,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==3 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -148,7 +148,7 @@ inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==9 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==11 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -166,7 +166,7 @@ fingerprint() { } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) - and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.jobs==1 + and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") @@ -285,7 +285,9 @@ if (( ${#target[@]} )); then "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 - and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted" + and .lease.attempt==2 and .lease.redelivered_after_seconds>=0 and .lease.stale_attempt_status==409 + and .lease.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index b36f69b1d..09da717d5 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 4')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 5')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-signal', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the three isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-lease', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the four isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index 2b695dc8d..41f0281d5 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -57,7 +57,8 @@ $activity = $originals['activity']; $timer = $originals['timer']; $signal = $originals['signal']; -$complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { +$lease = $originals['lease']; +$complete =static function (array $claimed, string $operation) use ($expect, $request, $result): void { $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], 'workflow_task_attempt' => $claimed['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, $operation); @@ -70,6 +71,12 @@ $deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); $check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); +// The workflow task PHP leased to a stopped worker keeps its PHP expiry. +$check(is_string($lease['workflow_task']['lease_expires_at'] ?? null), 'The original lease expiry is missing'); +$leaseExpiry = (float) (new DateTimeImmutable($lease['workflow_task']['lease_expires_at']))->format('U.u'); +$check($leaseExpiry - microtime(true) <= (float) $timerBudget, 'The original lease expiry is outside the wait budget'); +$check($keep($lease['history'], $history($lease)), 'Original lease history changed'); + // The activity leased under PHP completes with its original attempt. $task = $activity['activity_task']; $completion = ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], 'result' => $result]; @@ -79,19 +86,21 @@ $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); // Each original workflow then gets one workflow task: the activity's and the -// signal's at once, the timer's at its deadline. On a slow host the timer is -// already due, so they may be claimed in any order. +// signal's at once, the timer's at its deadline and the lease's when PHP's +// lease expires. On a slow host some are already due, so they may be claimed +// in any order. $original = $activity['history']; $claims = []; -$until = max($deadline, microtime(true)) + 30; -while (count($claims) < 3 && microtime(true) < $until) { +$leaseOutcome = null; +$until = max($deadline, $leaseExpiry, microtime(true)) + 30; +while (count($claims) < 4 && microtime(true) < $until) { $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; if (! is_array($claimed)) { continue; } $run = $claimed['run_id']; - $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id']], true) && ! isset($claims[$run]), + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id'], $lease['run_id']], true) && ! isset($claims[$run]), 'An unexpected workflow task was claimed'); $claims[$run] = true; if ($run === $activity['run_id']) { @@ -109,6 +118,23 @@ === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), 'The PHP-accepted signal arguments differ under native'); $complete($claimed, 'signal workflow completion'); + } elseif ($run === $lease['run_id']) { + // After PHP's lease expires, native redelivers the same task as its + // next attempt. The expired PHP attempt cannot complete over it. + $expired = $lease['workflow_task']; + $check($claimed['task_id'] === $expired['task_id'] + && $claimed['workflow_task_attempt'] === $expired['workflow_task_attempt'] + 1, + 'The expired PHP lease was not redelivered as its next attempt'); + $check(microtime(true) >= $leaseExpiry, 'Native redelivered a PHP lease before it expired'); + $check($types($claimed['history_events']) === $types($lease['history']), + 'Original lease history changed in the redelivered task'); + $stale = $request('POST', '/api/worker/workflow-tasks/'.$expired['task_id'].'/complete', ['lease_owner' => $expired['lease_owner'], + 'workflow_task_attempt' => $expired['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]); + $check($stale[0] === 409, 'The expired PHP attempt completed over its redelivery'); + $leaseOutcome = ['attempt' => $claimed['workflow_task_attempt'], + 'redelivered_after_seconds' => round(microtime(true) - $leaseExpiry, 3), + 'stale_attempt_status' => $stale[0], 'stale_attempt_reason' => $stale[1]['reason'] ?? null]; + $complete($claimed, 'redelivered lease completion'); } else { $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); $complete($claimed, 'timer workflow completion'); @@ -117,6 +143,7 @@ $check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); $check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); $check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); +$check(isset($claims[$lease['run_id']]), 'The expired PHP lease was not redelivered'); // The workflow task PHP already completed is a stale attempt. $stale = $activity['workflow_task']; @@ -140,6 +167,11 @@ $check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', 'The signal workflow did not complete on its original history'); $outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; +$leaseHistory = $history($lease); +$check($keep($lease['history'], $leaseHistory) && end($leaseHistory)['event_type'] === 'WorkflowCompleted', + 'The redelivered lease workflow did not complete on its original history'); +$outcome['lease'] = ['run_id' => $lease['run_id'], 'lease_expires_at' => $lease['workflow_task']['lease_expires_at']] + + $leaseOutcome + ['events' => array_column($leaseHistory, 'event_type')]; $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, From 70f610bef99f1bf1171cba9838f3d0d5a7089198 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 17:27:18 +0000 Subject: [PATCH 19/20] Compare restart history without the public principal; scope the SQLite 2.9.7 cohort Public history now lifts each command's principal onto its event, and worker task history has none, so the recovered terminal resumption compares the fields both views share. The SQLite PHP 2.9.7 upgrade cohort steps set the job's resource scope like their neighbours; the cleanup step failed without it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 2 ++ scripts/conformance/server-parity/restart.php | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 681aa9e1b..699b59641 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -1671,6 +1671,7 @@ jobs: - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 @@ -1679,6 +1680,7 @@ jobs: if: always() timeout-minutes: 3 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 diff --git a/scripts/conformance/server-parity/restart.php b/scripts/conformance/server-parity/restart.php index a72f94979..5ded7dc5a 100644 --- a/scripts/conformance/server-parity/restart.php +++ b/scripts/conformance/server-parity/restart.php @@ -53,6 +53,13 @@ function restartControlHistory(array $events): array 'event_type' => $event['event_type'], 'timestamp' => $event['recorded_at'], 'payload' => $event['payload']], $events); } +// Public history lifts each command's principal onto its event; worker task +// history has no principal. Compare the fields both views share. +function restartWithoutPrincipal(array $events): array +{ + return array_map(static fn (array $event): array => array_diff_key($event, ['principal' => true]), $events); +} + function restartSignalWorkflow(WorkflowContext $context, array $input): array { $context->waitCondition(fn (): bool => count($context->signals('payload')) > 0, 'restart.payload'); @@ -336,7 +343,7 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array $terminalClaim = $client->pollWorkflowTask('restart-terminal-new', 'restart-terminal-v1', 0); assertRestart(is_array($terminalClaim) && $terminalClaim['task_id'] === $terminal['failure_receipt']['next_task_id'] && $terminalClaim['run_id'] === $terminal['run_id'] - && sameRestartValue(restartControlHistory($terminalClaim['history_events']), $terminal['history']) + && sameRestartValue(restartControlHistory($terminalClaim['history_events']), restartWithoutPrincipal($terminal['history'])) && $terminalClaim['history_events'][4]['workflow_task_id'] === $terminalActivity['task_id'], 'Recovered terminal resumption lost original task/run/history.'); $terminalReplay = (new Replayer($client->payloadCodec()))->replay(restartTerminalWorkflow(...), $terminalClaim['history_events'], $client->payloadCodec()->decodeEnvelope($terminalClaim['arguments']), 'restart-terminal-v1', $terminalClaim); From a7bd4d07c62eb1f22d7ef93bb483f8485e2bc0de Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 20:41:00 +0000 Subject: [PATCH 20/20] Supervise the takeover kill test's child off the async runtime Sonar flagged a blocking process API and blocking file reads inside async tests. The SQLite takeover kill test now supervises its child on a blocking thread, the WAL copy and original read use Tokio's file API, and the SQLite upgrade cohort uses [[ for its backend test. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade_tests.rs | 63 +++++++++++-------- .../server-parity/upgrade-source-profile.sh | 4 +- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 35e563a5a..045caea82 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -955,34 +955,41 @@ async fn takeover_killed_before_commit_leaves_php_data_and_can_be_rerun() { use std::os::unix::process::ExitStatusExt; let (directory, path) = php297_layout().await; let fingerprint = backup_fingerprint(&path).await; - let original = std::fs::read(&path).unwrap(); + let original = tokio::fs::read(&path).await.unwrap(); let receipt = directory.path().join("boundary"); - let mut child = std::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "runtime::upgrade::tests::sqlite_takeover_child", - "--ignored", - "--nocapture", - ]) - .env("DW_TAKEOVER_TEST_DATABASE", &path) - .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) - .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) - .spawn() - .unwrap(); - let deadline = tokio::time::Instant::now() + Duration::from_secs(60); - while !receipt.exists() && tokio::time::Instant::now() < deadline { - if let Some(status) = child.try_wait().unwrap() { - panic!("child exited before the commit point: {status}"); + // The child process is supervised on a blocking thread. + let (child_path, child_fingerprint) = (path.clone(), fingerprint.clone()); + let status = tokio::task::spawn_blocking(move || { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::upgrade::tests::sqlite_takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &child_path) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &child_fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && std::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + std::thread::sleep(Duration::from_millis(10)); + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); } - tokio::time::sleep(Duration::from_millis(10)).await; - } - if !receipt.exists() { child.kill().unwrap(); - child.wait().unwrap(); - panic!("child did not reach the commit point"); - } - child.kill().unwrap(); - assert_eq!(child.wait().unwrap().signal(), Some(9)); + child.wait().unwrap() + }) + .await + .unwrap(); + assert_eq!(status.signal(), Some(9)); // Either SQLite had not written the file yet, or its rollback journal // still holds the original pages. Read-only inspection never recovers it. if Path::new(&format!("{path}-journal")).exists() { @@ -991,7 +998,7 @@ async fn takeover_killed_before_commit_leaves_php_data_and_can_be_rerun() { "upgrade_source_sqlite_journal_unqualified", ); } else { - assert_eq!(std::fs::read(&path).unwrap(), original); + assert_eq!(tokio::fs::read(&path).await.unwrap(), original); } // Rerunning with the same verified fingerprint rolls the journal back, // finds the backed-up PHP database and takes it over. @@ -1087,7 +1094,9 @@ async fn wal_source() -> (tempfile::TempDir, String) { .unwrap(); let copy = directory.path().join("wal.sqlite").to_str().unwrap().to_owned(); for suffix in ["", "-wal", "-shm"] { - std::fs::copy(format!("{path}{suffix}"), format!("{copy}{suffix}")).unwrap(); + tokio::fs::copy(format!("{path}{suffix}"), format!("{copy}{suffix}")) + .await + .unwrap(); } writer.close().await.unwrap(); (directory, copy) diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index b5f340ef0..6d869e1f3 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -81,7 +81,7 @@ docker run --rm --name "$resource-author" --user=1000:1000 --cpus=1 --memory=512 docker stop --signal SIGTERM --time 10 "$resource-php" >/dev/null docker inspect --format '{{json .State}}' "$resource-php" > "$evidence/stopped-state.json" jq -e '.Running==false and .ExitCode==0 and .OOMKilled==false' "$evidence/stopped-state.json" >/dev/null -if test "$backend" = sqlite; then +if [[ $backend == sqlite ]]; then # PHP Server keeps SQLite in WAL mode. Once PHP is stopped, the source is # switched to the rollback journal the upgrade commands need, as an operator # does first; its rows are unchanged. @@ -95,7 +95,7 @@ if test "$backend" = sqlite; then "$evidence/prepare.json" >/dev/null [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 1 1" ]] fi -if test "$backend" = sqlite; then +if [[ $backend == sqlite ]]; then docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network=none -v "$PWD/$state:/state:ro" -v "$PWD:/source:ro" --entrypoint php "$image" \ /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \