diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 9e14ac03e..2fce23a8b 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -676,13 +676,28 @@ jobs: cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json - name: Compare separate configured role-token cohort run: bash scripts/conformance/server-parity/role-profile.sh mysql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh mysql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source + timeout-minutes: 8 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-profile.sh mysql + - name: Retire PHP 2.9.7 upgrade source cohort + if: always() + timeout-minutes: 3 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql - name: Compare corrected markers, ranges, clocks and activity expiry if: ${{ !cancelled() && steps.prepare_runtime_comparison.outcome == 'success' }} # The preceding process-loss case deliberately kills native-a. @@ -1160,13 +1175,28 @@ jobs: env: POSTGRES_IMAGE: ${{ matrix.image }} run: bash scripts/conformance/server-parity/role-profile.sh pgsql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh pgsql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source + timeout-minutes: 8 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-profile.sh pgsql + - name: Retire PHP 2.9.7 upgrade source cohort + if: always() + timeout-minutes: 3 + env: + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql - name: Compare corrected markers, ranges, clocks and activity expiry if: ${{ !cancelled() && steps.prepare_runtime_comparison.outcome == 'success' }} env: @@ -1635,7 +1665,7 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/role-profile.sh sqlite - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} @@ -1646,6 +1676,23 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source + timeout-minutes: 8 + env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-profile.sh sqlite + - name: Retire PHP 2.9.7 upgrade source cohort + if: always() + timeout-minutes: 3 + env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + DW_PARITY_UPGRADE_LABEL: php297 + DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 + DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 + run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite - name: Compare corrected markers, ranges, clocks and activity expiry if: ${{ !cancelled() && steps.prepare_runtime_comparison.outcome == 'success' }} env: diff --git a/rust/README.md b/rust/README.md index d94d2400e..679d895d9 100644 --- a/rust/README.md +++ b/rust/README.md @@ -412,10 +412,58 @@ version-1/version-2/version-3 development schemas are refused without conversion needed old development data separately; this unpublished slice supplies no older-development converter. Use a new isolated file for version 4. -This refusal is not the upgrade mechanism. Backup-first conversion of PHP's -stored representations and interrupted takeover remain required on all three -database families. The PHP ownership fence is merged in source, while the frozen -published PHP image still lacks it. **Never connect PHP to a Rust development database.** +This refusal is not the upgrade mechanism. A stopped PHP database can be taken +over with the development `upgrade-takeover` command, following the +[takeover design](../docs/rust-server-upgrade-takeover.md): + +1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and + stop every PHP role. + PHP Server keeps SQLite in WAL mode, which the upgrade commands refuse. With + PHP stopped, run `upgrade-prepare-sqlite` on the source file (writable). + It has SQLite checkpoint every committed WAL frame into the file and switch + it to the rollback journal, refusing with `upgrade_source_in_use` while any + other process has the file open. The rows are unchanged; the backup below + is taken afterwards. A journal left by a crashed writer is rolled back the + same way. +2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, + or the database through a read-only account. Back the database up (a file + copy, `pg_dump` or `mysqldump --single-transaction`), restore the backup + separately, and fingerprint that too. The digests must be equal. +3. Run `upgrade-takeover` with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that + digest and the original queue settings (`DW_RUST_UPGRADE_QUEUE_*`, as for + `upgrade-queue-inspect`). PostgreSQL uses the owner's `DB_*` settings. + MySQL and MariaDB use an account that owns the database and has the + `PROCESS` privilege, so the takeover can see every other session. + +On SQLite and PostgreSQL, inspection, the fingerprint check, the conversion +of the recorded PHP release layout, the native relations, the migration ledger +and native's own startup verification all happen in one transaction. SQLite +holds the file +exclusively; PostgreSQL takes native's bootstrap lock and locks every table +before reading, and refuses with `upgrade_source_in_use` if a session still +holds one after five seconds. A refusal leaves the PHP database unchanged. An +interrupted takeover commits nothing: rerun it with the same digest. On +SQLite, the rerun lets SQLite roll back the journal the interrupted run left, +and the fingerprint check then proves the file is the backed-up one. + +MySQL and MariaDB commit DDL implicitly, so their takeover runs in steps under +native's bootstrap lock. It refuses with `upgrade_source_in_use` while any +other session uses the database. Once the source is verified, it first writes +native's marker with a `rust-takeover:` row: PHP refuses the database +from then on, and native refuses it with `native_takeover_incomplete` until +the takeover finishes. It then checks the sessions and the fingerprint again, +and runs each conversion step, recording each in that row. An interrupted +takeover is rerun with the same digest and continues after the last recorded +step; a different digest is refused. + +The command refuses: + +- a digest that differs, so any write after the backup stops it; +- unrecognized queued jobs, external payloads and service calls; +- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7 (with either + Laravel MariaDB driver), or a database native already owns. + +Nested payload columns and multi-node qualification remain open. **Never connect PHP to a Rust development database.** Named namespace isolation and configured worker/operator/admin roles have representative shared qualification. Principal/custom-provider credentials and the full authorization contract remain open. Broader timer/signal semantics, failure/retry policy, cancellation and the rest of the API also diff --git a/rust/src/main.rs b/rust/src/main.rs index 22abcb22c..ae8efabcb 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -132,11 +132,113 @@ async fn main() -> Result<(), Box> { | "upgrade-inspect" | "upgrade-queue-inspect" | "upgrade-payload-inspect" + | "upgrade-fingerprint" + | "upgrade-prepare-sqlite" + | "upgrade-takeover" ) { return Err("unknown development command".into()); } let backend = env::var("DB_CONNECTION").unwrap_or_default(); + if command == "upgrade-prepare-sqlite" { + if backend != "sqlite" { + return Err("upgrade-prepare-sqlite requires DB_CONNECTION=sqlite".into()); + } + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + return match durable_workflow_server::runtime::upgrade::prepare_sqlite_source(&path).await + { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_prepare_unavailable", + }; + eprintln!("{reason}"); + Err("SQLite source preparation refused".into()) + } + }; + } + if command == "upgrade-takeover" { + use durable_workflow_server::runtime::upgrade::{ + queue_binding::QueueSourceConfig, takeover, + }; + if !matches!(backend.as_str(), "sqlite" | "pgsql" | "mysql" | "mariadb") { + eprintln!("upgrade_takeover_backend_unqualified"); + return Err("upgrade takeover refused".into()); + } + let fingerprint = env::var("DW_UPGRADE_BACKUP_FINGERPRINT") + .map_err(|_| "DW_UPGRADE_BACKUP_FINGERPRINT is required")?; + // The original Laravel queue transport, as for upgrade-queue-inspect. + let (connection, default_connection, default_queue) = ( + env::var("DW_RUST_UPGRADE_QUEUE_CONNECTION") + .map_err(|_| "original queue connection is required")?, + env::var("DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION") + .map_err(|_| "original default queue connection is required")?, + env::var("DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE") + .map_err(|_| "original default transport queue is required")?, + ); + let queues = QueueSourceConfig { + connection: &connection, + default_connection: &default_connection, + default_queue: &default_queue, + }; + let result = if backend == "pgsql" { + takeover::take_over_postgres(postgres_options()?, &fingerprint, &queues).await + } else if backend != "sqlite" { + takeover::take_over_mysql(mysql_options()?, &fingerprint, &queues).await + } else { + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + takeover::take_over_sqlite(&path, &fingerprint, &queues).await + }; + return match result { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_takeover_unavailable", + }; + eprintln!("{reason}"); + Err("upgrade takeover refused".into()) + } + }; + } + if command == "upgrade-fingerprint" { + use durable_workflow_server::runtime::upgrade::fingerprint; + let result = match backend.as_str() { + "sqlite" => { + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + fingerprint::fingerprint_sqlite_cli(&path).await + } + "mysql" | "mariadb" => fingerprint::fingerprint_mysql(mysql_options()?).await, + "pgsql" => fingerprint::fingerprint_postgres(postgres_options()?).await, + _ => { + return Err( + "upgrade fingerprint requires DB_CONNECTION=sqlite, pgsql, mysql or mariadb" + .into(), + ); + } + }; + return match result { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_fingerprint_unavailable", + }; + eprintln!("{reason}"); + Err("upgrade source fingerprint refused".into()) + } + }; + } if matches!( command.as_str(), "upgrade-inspect" | "upgrade-queue-inspect" | "upgrade-payload-inspect" diff --git a/rust/src/runtime/history_presentation.rs b/rust/src/runtime/history_presentation.rs new file mode 100644 index 000000000..e46d0abac --- /dev/null +++ b/rust/src/runtime/history_presentation.rs @@ -0,0 +1,182 @@ +//! Presents stored history payloads the way published PHP Server does +//! (`ExternalPayloadEnvelopeService::historyPayload` and +//! `HistoryController::eventPrincipal`). PHP stores encoded values as bare +//! strings beside their codec and wraps them while reading; native stores +//! the envelope itself. With the same rules, history written by PHP before a +//! takeover reads as PHP returned it, and native's own envelopes pass through +//! unchanged. +use serde_json::{Map, Value, json}; + +/// Kept from a PHP activity failure; PHP redacts its runtime internals. +const NEUTRAL_EXCEPTION_KEYS: [&str; 12] = [ + "type", + "message", + "code", + "kind", + "classification", + "retryable", + "non_retryable", + "timeout_type", + "cancelled", + "malformed_output", + "details", + "details_payload_codec", +]; +const DIAGNOSTIC_KEYS: [&str; 2] = ["diagnostics", "runtime_diagnostics"]; +/// PHP's stored provider reference, which only PHP's external payload store +/// can resolve. Takeover refuses sources that hold any, so it is left as is. +const PHP_EXTERNAL_REFERENCE: &str = "dw-external-payload:v1:"; + +pub(super) fn payload(mut payload: Value, event_type: &str) -> Value { + let Value::Object(fields) = &mut payload else { + return payload; + }; + let codec = codec(fields, "payload_codec", Some("avro")); + for field in ["arguments", "result", "output"] { + wrap(fields, field, codec.as_deref()); + } + if let Some(Value::Object(command)) = fields.get_mut("command") { + let codec = self::codec(command, "payload_codec", codec.as_deref()); + wrap(command, "payload", codec.as_deref()); + } + if let Some(Value::Object(activity)) = fields.get_mut("activity") { + let codec = self::codec(activity, "payload_codec", codec.as_deref()); + for field in ["arguments", "result"] { + wrap(activity, field, codec.as_deref()); + } + } + let activity_failure = event_type == "ActivityFailed"; + let opaque_legacy = activity_failure + && fields + .get("migration_projection") + .and_then(|projection| projection.get("source_record")) + == Some(&json!("workflow_exception")); + if let Some(Value::Object(exception)) = fields.get_mut("exception") { + let codec = self::codec(exception, "details_payload_codec", codec.as_deref()); + wrap(exception, "details", codec.as_deref()); + if activity_failure { + let mut neutral = Map::new(); + for key in NEUTRAL_EXCEPTION_KEYS { + if let Some(value) = exception.get(key) { + neutral.insert(key.to_owned(), value.clone()); + } + } + for key in DIAGNOSTIC_KEYS { + if let Some(value @ (Value::Object(_) | Value::Array(_))) = exception.get(key) { + neutral.insert(key.to_owned(), value.clone()); + } + } + if opaque_legacy + && let Some(value) = exception.get("value").filter(|value| opaque_descriptor(value)) + { + neutral.insert("value".to_owned(), value.clone()); + } + *exception = neutral; + } + } + payload +} + +/// The actor PHP surfaces at the top of a public history event: the +/// principal recorded on the event's command, or null. +pub(super) fn principal(payload: &Value) -> Value { + let Some(command) = payload.get("command").filter(|command| command.is_object()) else { + return Value::Null; + }; + let mut principal = Map::new(); + for (key, field) in [ + ("type", "principal_type"), + ("id", "principal_id"), + ("label", "principal_label"), + ] { + if let Some(value) = command[field].as_str().filter(|value| !value.is_empty()) { + principal.insert(key.to_owned(), json!(value)); + } + } + if principal.is_empty() { + Value::Null + } else { + Value::Object(principal) + } +} + +/// The section's own codec, or the enclosing one. `None` means a codec PHP +/// rejects, so nothing in that section is wrapped. +fn codec(section: &Map, field: &str, fallback: Option<&str>) -> Option { + match section.get(field).and_then(Value::as_str).filter(|codec| !codec.is_empty()) { + Some("avro") => Some("avro".to_owned()), + Some(_) => None, + None => fallback.map(str::to_owned), + } +} + +fn wrap(section: &mut Map, field: &str, codec: Option<&str>) { + let Some(codec) = codec else { + return; + }; + if let Some(Value::String(blob)) = section.get(field) + && !blob.starts_with(PHP_EXTERNAL_REFERENCE) + { + let envelope = json!({"codec": codec, "blob": blob}); + section.insert(field.to_owned(), envelope); + } +} + +fn opaque_descriptor(value: &Value) -> bool { + value["available"].is_boolean() + && value["encoding"] == "php-serialize" + && value.get("value").is_some_and(|inner| inner.is_string() || inner.is_null()) + && value["portable"] == false + && value["unsupported_reason"] == "legacy_php_serialization_not_portable" + && value["remediation"] + == "Treat the preserved value as opaque; export decoded JSON from the source application if portable payload access is required." +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn php_stored_strings_become_envelopes_and_native_envelopes_stay() { + let stored = json!({"payload_codec":"avro","arguments":"AAA=","output":null, + "command":{"payload_codec":"avro","payload":"BBB=","principal_type":"auth:token", + "principal_id":"legacy-token","principal_label":"Admin"}, + "activity":{"arguments":"CCC=","result":{"codec":"avro","blob":"DDD="}}}); + let presented = payload(stored.clone(), "ActivityScheduled"); + assert_eq!(presented["arguments"], json!({"codec":"avro","blob":"AAA="})); + assert_eq!(presented["output"], Value::Null); + assert_eq!(presented["command"]["payload"], json!({"codec":"avro","blob":"BBB="})); + assert_eq!(presented["activity"]["arguments"], json!({"codec":"avro","blob":"CCC="})); + assert_eq!(presented["activity"]["result"], json!({"codec":"avro","blob":"DDD="})); + assert_eq!(payload(presented.clone(), "ActivityScheduled"), presented); + assert_eq!( + principal(&stored), + json!({"type":"auth:token","id":"legacy-token","label":"Admin"}) + ); + assert_eq!(principal(&json!({"command":{"type":"start"}})), Value::Null); + assert_eq!(principal(&json!({})), Value::Null); + } + + #[test] + fn unsupported_codecs_and_php_references_are_left_unchanged() { + let stored = json!({"payload_codec":"json","result":"plain", + "command":{"payload":"dw-external-payload:v1:eyJ9"}}); + let presented = payload(stored.clone(), "ActivityCompleted"); + assert_eq!(presented["result"], "plain"); + assert_eq!(presented["command"]["payload"], "dw-external-payload:v1:eyJ9"); + } + + #[test] + fn activity_failures_keep_only_neutral_fields() { + let stored = json!({"exception":{"type":"RuntimeException","message":"boom","file":"/app/x.php", + "line":3,"trace":["frame"],"details":"EEE=","diagnostics":{"hint":"retry"},"runtime_diagnostics":"text"}}); + let presented = payload(stored.clone(), "ActivityFailed"); + assert_eq!( + presented["exception"], + json!({"type":"RuntimeException","message":"boom","details":{"codec":"avro","blob":"EEE="}, + "diagnostics":{"hint":"retry"}}) + ); + // Other failures keep every field, as PHP presents them. + assert_eq!(payload(stored, "WorkflowFailed")["exception"]["file"], "/app/x.php"); + } +} diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 678bc41ac..4c0da6139 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -28,6 +28,7 @@ mod external_transport; mod external_wait_summary; mod failure_commands; mod history_budget; +mod history_presentation; mod http; mod memo_json; mod memo_limits; diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 5ae70bc04..a4a76dbf5 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -13,15 +13,19 @@ pub const VERSION: i64 = 4; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; -pub const LEDGER_SQL: &str = "CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; +/// The marker row of a takeover in progress: `rust-takeover:`, with +/// the completed step count as its version and the backup fingerprint's +/// digest as its checksum. Native startup refuses it and PHP sees the marker. +pub(super) const TAKEOVER_ENGINE: &str = "rust-takeover:"; +pub const LEDGER_SQL: &str ="CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; #[derive(Clone, Copy)] -enum Flavor { +pub(super) enum Flavor { MySql, MariaDb, } impl Flavor { - async fn detect(connection: &mut MySqlConnection) -> Result { + pub(super) async fn detect(connection: &mut MySqlConnection) -> Result { let version: String = sqlx::query_scalar("SELECT VERSION()") .fetch_one(connection) .await?; @@ -67,12 +71,40 @@ impl Flavor { }) } fn expected(self) -> Result { - Ok(serde_json::from_str(match self { + Ok(serde_json::from_str(&canonical(match self { Self::MySql => include_str!("../../migrations/mysql/0004_catalog.json"), Self::MariaDb => include_str!("../../migrations/mariadb/0004_catalog.json"), - })?) + }))?) } - fn checksum(self) -> &'static [u8] { + /// The single native migration, which a takeover records as applied. + pub(super) fn migration(self) -> &'static Migration { + self.migrator().iter().next().expect("one native MySQL migration") + } + /// The native relations a fresh bootstrap adds after the frozen PHP + /// layout, each created only if missing so a resumed takeover can repeat + /// them. The poll index on `workflow_tasks` is added separately. + pub(super) fn native_relations_sql(self) -> Result { + let (frozen, incarnations) = match self { + Self::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + ), + Self::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + ), + }; + Ok(super::upgrade::native_relations( + frozen, + "-- Native receipts are separate", + incarnations, + )? + .replace( + "CREATE TABLE dw_worker_registration_incarnations", + "CREATE TABLE IF NOT EXISTS dw_worker_registration_incarnations", + )) + } + pub(super) fn checksum(self) -> &'static [u8] { self.migrator().iter().next().unwrap().checksum.as_ref() } } @@ -83,7 +115,7 @@ fn options(options: MySqlConnectOptions) -> MySqlConnectOptions { .collation("utf8mb4_unicode_ci") .timezone(Some("+00:00".into())) } -async fn configure(connection: &mut MySqlConnection) -> Result<()> { +pub(super) async fn configure(connection: &mut MySqlConnection) -> Result<()> { sqlx::query("SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5") .execute(connection) .await?; @@ -106,6 +138,38 @@ fn table_sql(prefix: &str, table: &str, suffix: &str) -> Result String { + const CHARSET: &str = " CHARACTER SET "; + let mut result = String::with_capacity(sql.len()); + let mut rest = sql; + while let Some(at) = rest.find(CHARSET) { + let after = &rest[at + CHARSET.len()..]; + let length = after + .bytes() + .take_while(|b| b.is_ascii_alphanumeric() || *b == b'_') + .count(); + let (charset, tail) = after.split_at(length); + if length > 0 + && tail + .strip_prefix(" COLLATE ") + .is_some_and(|collation| { + collation.starts_with(charset) && collation[length..].starts_with('_') + }) + { + result.push_str(&rest[..at]); + } else { + result.push_str(&rest[..at + CHARSET.len() + length]); + } + rest = tail; + } + result.push_str(rest); + result +} + pub fn definition(sql: String) -> String { let Some(footer) = sql.rfind(") ENGINE=") else { return sql; @@ -144,12 +208,12 @@ async fn catalog(connection: &mut MySqlConnection) -> Result { let row = sqlx::query(table_sql("SHOW CREATE TABLE ", &name, "")?) .fetch_one(&mut *connection) .await?; - result.push((name, definition(row.try_get(1)?))); + result.push((name, canonical(&definition(row.try_get(1)?)))); } Ok(result) } -pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Result { +pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Result<(Catalog, &'static str)> { let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(TABLE_NAME)='dw_server_schema'") .fetch_one(&mut *connection).await?; if marked != 0 { @@ -173,32 +237,91 @@ pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Resu )); } } + let actual = catalog(connection).await?; + source_layout(connection, actual).await +} + +/// The PHP source catalog of a database whose takeover has written its +/// marker and nothing else yet: native's exact marker table, set aside. +pub(super) async fn fenced_source_catalog(connection: &mut MySqlConnection) -> Result<(Catalog, &'static str)> { + let flavor = Flavor::detect(connection).await?; + let marker = flavor + .expected()? + .into_iter() + .find(|(table, _)| table == "dw_server_schema"); + let mut actual = catalog(connection).await?; + let position = actual.iter().position(|(table, _)| table == "dw_server_schema"); + match position { + Some(position) if Some(&actual[position]) == marker.as_ref() => { + actual.remove(position); + } + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_marker_unqualified", + )); + } + } + source_layout(connection, actual).await +} + +async fn source_layout( + connection: &mut MySqlConnection, + mut actual: Catalog, +) -> Result<(Catalog, &'static str)> { let flavor = Flavor::detect(connection).await?; - let mut expected: Catalog = flavor.expected()?.into_iter() + let expected: Catalog = flavor.expected()?.into_iter() .filter(|(table, _)| !super::upgrade::native_relation(table)) .map(|(table, ddl)| (table, ddl.replace( ",\n KEY `dw_workflow_tasks_poll` (`namespace`(64),`queue`(64),`task_type`(16),`status`(16),`available_at`)", ""))) .collect(); - let mut actual = catalog(connection).await?; // MariaDB's information_schema collation ignores some underscores. Only // canonicalize catalog presentation; preserve every full definition. - expected.sort_by(|left, right| left.0.cmp(&right.0)); actual.sort_by(|left, right| left.0.cmp(&right.0)); - if actual != expected { - let delta = match flavor { - Flavor::MySql => include_str!("../../upgrade/php-2.8.0/mysql.json"), - Flavor::MariaDb => include_str!("../../upgrade/php-2.8.0/mariadb.json"), - }; - let mut current = super::upgrade::source_catalog_variant(&expected, delta)?; - current.sort_by(|left, right| left.0.cmp(&right.0)); - if actual != current { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_unqualified", - )); + let deltas = match flavor { + Flavor::MySql => [ + canonical(include_str!("../../upgrade/php-2.8.0/mysql.json")), + canonical(include_str!("../../upgrade/php-2.9.7/mysql.json")), + ], + Flavor::MariaDb => [ + canonical(include_str!("../../upgrade/php-2.8.0/mariadb.json")), + canonical(include_str!("../../upgrade/php-2.9.7/mariadb.json")), + ], + }; + let layout = super::upgrade::source_layout( + &actual, + expected.clone(), + [&deltas[0], &deltas[1]], + |row| row.0.clone(), + ); + // Laravel's separate mariadb connection stores the backup hold owner as + // uuid rather than char(36); every release layout is otherwise the same. + let layout = match (layout, flavor) { + (Err(_), Flavor::MariaDb) => { + let driver = super::upgrade::source_catalog_variant( + &expected, + &canonical(include_str!("../../upgrade/mariadb-driver.json")), + )?; + match super::upgrade::source_layout( + &actual, + driver, + [&deltas[0], &deltas[1]], + |row| row.0.clone(), + )? { + "frozen" => "frozen+mariadb-driver", + "php-2.8.0" => "php-2.8.0+mariadb-driver", + "php-2.9.7" => "php-2.9.7+mariadb-driver", + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + } } - } - Ok(actual) + (layout, _) => layout?, + }; + Ok((actual, layout)) } pub(super) fn source_options(options: MySqlConnectOptions) -> MySqlConnectOptions { @@ -262,7 +385,7 @@ pub(super) async fn verify_history(connection: &mut MySqlConnection) -> Result<( Ok(()) } -async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { +pub(super) async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { history(connection, flavor, false).await?; if catalog(connection).await? != flavor.expected()? { return Err(refuse( @@ -328,6 +451,9 @@ async fn preflight(connection: &mut MySqlConnection, flavor: Flavor) -> Result<( if engines == ["rust-initializing"] { verify_initializing(connection, flavor).await?; Ok(()) + } else if engines.iter().any(|engine| engine.starts_with(TAKEOVER_ENGINE)) { + // An interrupted takeover resumes only through upgrade-takeover. + Err(refuse(StatusCode::CONFLICT, "native_takeover_incomplete")) } else { verify(connection, flavor).await } @@ -514,6 +640,23 @@ mod tests { sqlx::query("INSERT INTO _sqlx_migrations VALUES (?, 'interrupted qualification', CURRENT_TIMESTAMP, FALSE, ?, -1)").bind(VERSION).bind(flavor.checksum()).execute(connection).await.unwrap(); } + #[test] + fn restored_column_character_sets_compare_as_their_collation() { + let created = "CREATE TABLE `t` (\n `a` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `b` char(8) COLLATE ascii_bin\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; + let restored = "CREATE TABLE `t` (\n `a` varchar(26) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,\n `b` char(8) CHARACTER SET ascii COLLATE ascii_bin\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; + assert_eq!(canonical(restored), created); + assert_eq!(canonical(created), created); + // Only a character set its collation names is redundant. + for kept in [ + "`a` text CHARACTER SET utf8 COLLATE utf8mb4_bin", + "`a` text CHARACTER SET utf8mb4", + "`a` text CHARACTER SET latin1 COLLATE utf8mb4_bin", + "`a` text COMMENT ' CHARACTER SET '", + ] { + assert_eq!(canonical(kept), kept); + } + } + #[tokio::test] #[ignore = "requires disposable MySQL/MariaDB; run qualification_ explicitly"] async fn qualification_schedule_creation_keeps_original_audit_and_unsigned_counters() { @@ -876,7 +1019,11 @@ mod tests { connection.execute(LEDGER_SQL).await.unwrap(); flavor.migrator().run(&mut connection).await.unwrap(); } - std::fs::write(std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(), phase).unwrap(); + // The parent waits for the receipt to exist, so it must appear with + // its contents: write beside it and rename into place. + let receipt = std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(); + std::fs::write(format!("{receipt}.partial"), &phase).unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); loop { std::thread::park(); } @@ -957,4 +1104,356 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout, or a release layout rebuilt from the recorded + /// deltas (each changed table exactly as that release's PHP created it). + async fn php_layout(db: &Database, release: Option<&str>) -> Flavor { + let mut connection = db.connection().await; + let flavor = Flavor::detect(&mut connection).await.unwrap(); + let (frozen, deltas) = match flavor { + Flavor::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mysql.json"), + include_str!("../../upgrade/php-2.9.7/mysql.json"), + ], + ), + Flavor::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mariadb.json"), + include_str!("../../upgrade/php-2.9.7/mariadb.json"), + ], + ), + }; + let php = &frozen[..frozen.find("-- Native receipts are separate").unwrap()]; + connection.execute(php).await.unwrap(); + connection + .execute("SET FOREIGN_KEY_CHECKS=1; ALTER TABLE workflow_tasks DROP KEY dw_workflow_tasks_poll") + .await + .unwrap(); + let releases = match release { + None => 0, + Some("php-2.8.0") => 1, + Some(_) => 2, + }; + for delta in &deltas[..releases] { + let delta: Value = serde_json::from_str(delta).unwrap(); + for added in delta["add"].as_array().unwrap() { + let (table, ddl) = (added[0].as_str().unwrap(), added[1].as_str().unwrap()); + connection + .execute(table_sql("DROP TABLE IF EXISTS ", table, "").unwrap()) + .await + .unwrap(); + connection.execute(AssertSqlSafe(ddl.to_owned())).await.unwrap(); + } + } + connection.close().await.unwrap(); + flavor + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_mysql(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// The physical schema and the fingerprint of every PHP row. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let schema = catalog(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + (schema, backup_fingerprint(db).await) + } + + async fn sessions_ended(db: &Database) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if others == 0 { + return; + } + assert!(tokio::time::Instant::now() < deadline, "a takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + async fn registrations(db: &Database) { + let mut connection = db.connection().await; + connection.execute( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .await + .unwrap(); + connection.close().await.unwrap(); + } + + /// Native opens the database and holds the converted incarnations. + async fn converted(db: &Database) { + let storage = MySqlStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + MySqlStorage::check(db.options.clone()).await.unwrap(); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + converted(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_the_mariadb_driver_layout() { + let db = Database::new().await; + if let Flavor::MySql = php_layout(&db, Some("php-2.9.7")).await { + db.remove().await; + return; + } + // Laravel's mariadb connection stores the backup hold owner as uuid. + let mut connection = db.connection().await; + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner uuid DEFAULT NULL") + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "php-2.9.7+mariadb-driver", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + php_layout(&db, None).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_refusals_before_the_marker_leave_php_unchanged() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "z".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + // Any other session on the database refuses the takeover. + let php = db.connection().await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + php.close().await.unwrap(); + assert_eq!(source_state(&db).await, original); + // A write after the backup changes the digest, so the old one is refused. + let mut php = db.connection().await; + php.execute("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .await + .unwrap(); + php.close().await.unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let mut connection = db.connection().await; + connection.execute(statement).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + // Named under bootstrap_child so the qualification run's existing skip of + // that subprocess helper also skips this one. + #[tokio::test] + #[ignore = "invoked only as a subprocess by the takeover interruption qualification"] + async fn bootstrap_child_takeover() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks at DW_TAKEOVER_TEST_BOUNDARY, where the parent kills it. + let report = takeover::take_over_mysql( + MySqlConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_killed_at_each_step_resumes_with_the_same_fingerprint() { + use std::os::unix::process::ExitStatusExt; + for boundary in [ + "before-marker-row", + "after-step-1", + "after-step-3", + "after-step-8", + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::mysql::tests::bootstrap_child_takeover", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_BOUNDARY", boundary) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(90); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before {boundary}: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach {boundary}"); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + sessions_ended(&db).await; + // The marker already fences PHP, and native will not open the + // database until the takeover finishes. + let mut connection = db.connection().await; + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='dw_server_schema'") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(marked, 1, "{boundary}"); + assert!(MySqlStorage::open(db.options.clone()).await.is_err(), "{boundary}"); + if boundary == "after-step-3" { + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + } + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{boundary}: {report}"); + assert_eq!(report["source_layout"], "php-2.9.7", "{boundary}"); + converted(&db).await; + db.remove().await; + } + } } diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index f7200b79b..06af1a9ce 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -39,6 +39,23 @@ fn migrator() -> &'static Migrator { }) } +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native PostgreSQL migration") +} + +/// The native relations, indexes and ownership marker that a fresh +/// bootstrap adds after the frozen PHP layout. Takeover applies them to a +/// converted PHP database instead of running the whole migration. +pub(super) fn native_relations_sql() -> Result { + super::upgrade::native_relations( + include_str!("../../migrations/postgres/0003_full_schema.sql"), + "CREATE TABLE public.dw_server_schema ", + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + ) +} + fn options(options: PgConnectOptions) -> PgConnectOptions { options .application_name("durable-workflow-rust-development") @@ -54,7 +71,7 @@ async fn catalog(connection: &mut PgConnection) -> Result { Ok(sqlx::query_as(CATALOG_QUERY).fetch_all(connection).await?) } -pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result { +pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result<(Catalog, &'static str)> { verify_scope(connection).await?; if marked(connection).await? { return Err(refuse(StatusCode::CONFLICT, "upgrade_source_already_owned")); @@ -80,20 +97,16 @@ pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result< }) .collect(); let actual = catalog(connection).await?; - if actual != expected { - let mut current = super::upgrade::source_catalog_variant( - &expected, + let layout = super::upgrade::source_layout( + &actual, + expected, + [ include_str!("../../upgrade/php-2.8.0/postgres.json"), - )?; - current.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); - if actual != current { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_unqualified", - )); - } - } - Ok(actual) + include_str!("../../upgrade/php-2.9.7/postgres.json"), + ], + |row| (row.0.clone(), row.1.clone()), + )?; + Ok((actual, layout)) } pub(super) fn source_options(options: PgConnectOptions) -> PgConnectOptions { @@ -160,7 +173,7 @@ pub(super) async fn verify_history(connection: &mut PgConnection) -> Result<()> Ok(()) } -async fn verify(connection: &mut PgConnection) -> Result<()> { +pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { verify_scope(connection).await?; verify_history(connection).await?; // A canonical catalog is captured with PostgreSQL itself, not inferred @@ -658,7 +671,11 @@ mod tests { verify(&mut tx).await.unwrap(); } // An actual server acknowledgement at a still-uncommitted boundary. - std::fs::write(std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(), phase).unwrap(); + // The parent waits for the receipt to exist, so it must appear with + // its contents: write beside it and rename into place. + let receipt = std::env::var("DW_SCHEMA_TEST_RECEIPT").unwrap(); + std::fs::write(format!("{receipt}.partial"), &phase).unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); loop { std::thread::park(); } @@ -712,4 +729,289 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout moved to PHP 2.9.7's by the changes PHP's own + /// migrations make (rust/upgrade/php-2.8.0 and php-2.9.7). Inspection + /// must call it php-2.9.7. + async fn php_297(db: &Database) -> PgConnection { + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let php = &frozen[..frozen.find("-- Native bookkeeping").unwrap()]; + let mut connection = db.connection().await; + sqlx::raw_sql(php).execute(&mut connection).await.unwrap(); + sqlx::raw_sql( + "ALTER TABLE workflow_worker_registrations ADD COLUMN registration_token varchar(32); + CREATE TABLE workflow_worker_registration_incarnations (token varchar(32) NOT NULL, namespace varchar(255) NOT NULL, worker_id varchar(255) NOT NULL, status varchar(32) NOT NULL, recovered_workflow_task_count integer NOT NULL DEFAULT 0, finished_at timestamp(6) without time zone, created_at timestamp(6) without time zone, updated_at timestamp(6) without time zone, PRIMARY KEY (token)); + CREATE INDEX workflow_worker_registration_incarnations_finished_at_index ON workflow_worker_registration_incarnations (finished_at); + CREATE INDEX dw_worker_incarnation_scope_idx ON workflow_worker_registration_incarnations (namespace, worker_id); + ALTER TABLE workflow_service_calls ADD COLUMN call_deadline_at timestamp(6) without time zone; + CREATE INDEX wf_service_calls_deadline_idx ON workflow_service_calls (status, call_deadline_at); + ALTER TABLE workflow_tasks ADD COLUMN queue_delivery_reference json;", + ) + .execute(&mut connection) + .await + .unwrap(); + connection + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_postgres(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// Every row and the physical schema, for byte-for-byte comparisons. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let state = ( + catalog(&mut connection).await.unwrap(), + backup_fingerprint(db).await, + ); + connection.close().await.unwrap(); + state + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["backend"], "pgsql"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + // The current worker keeps its PHP token; the retired ones keep their state. + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "current" && row.2 == "superseded" && row.3.is_none() && row.4)); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + connection.close().await.unwrap(); + let storage = PostgresStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + PostgresStorage::check(db.options.clone()).await.unwrap(); + // A second takeover finds a native database. + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let mut connection = db.connection().await; + sqlx::raw_sql(&frozen[..frozen.find("-- Native bookkeeping").unwrap()]) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_refusals_leave_the_php_database_unchanged() { + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), "not-a-fingerprint", &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + assert_eq!(source_state(&db).await, original); + // A PHP connection still holding a table refuses the takeover within + // the lock wait instead of stalling it. + let mut php = db.connection().await; + sqlx::raw_sql("BEGIN; LOCK TABLE jobs IN ROW EXCLUSIVE MODE") + .execute(&mut php) + .await + .unwrap(); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + sqlx::raw_sql("ROLLBACK").execute(&mut php).await.unwrap(); + // A write after the backup changes the digest, so the old one is refused. + sqlx::raw_sql("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .execute(&mut php) + .await + .unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + php.close().await.unwrap(); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql(statement).execute(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + #[tokio::test] + #[ignore = "only invoked as a real subprocess by the takeover interruption qualification"] + async fn takeover_child() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_postgres( + PgConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_killed_before_commit_leaves_php_and_retries() { + use std::os::unix::process::ExitStatusExt; + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::postgres::tests::takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + assert_eq!(std::fs::read_to_string(&receipt).unwrap(), "before-commit"); + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + // PostgreSQL ends the killed session's transaction and its locks. + let released = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let sessions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if sessions == 0 { + break; + } + assert!(tokio::time::Instant::now() < released, "killed takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert_eq!(source_state(&db).await, original, "partial takeover survived"); + // The same verified backup fingerprint still takes the database over. + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } } diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 279e78d67..9b602f319 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -31,6 +31,23 @@ fn migrator() -> &'static Migrator { }) } +/// The native relations, indexes and ownership marker that a fresh +/// bootstrap adds after the frozen PHP layout. Takeover applies them to a +/// converted PHP database instead of running the whole migration. +pub(super) fn native_relations_sql() -> Result { + super::upgrade::native_relations( + include_str!("../../migrations/sqlite/0003_full_schema.sql"), + "CREATE TABLE dw_server_schema ", + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + ) +} + +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native SQLite migration") +} + type Catalog = Vec<(String, String, String, String)>; static EXPECTED: OnceCell = OnceCell::const_new(); @@ -67,7 +84,7 @@ async fn verify_catalog(connection: &mut SqliteConnection) -> Result<()> { Ok(()) } -pub(super) async fn php_source_catalog(connection: &mut SqliteConnection) -> Result { +pub(super) async fn php_source_catalog(connection: &mut SqliteConnection) -> Result<(Catalog, &'static str)> { let owned: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name='dw_server_schema' COLLATE NOCASE)", ) @@ -85,20 +102,16 @@ pub(super) async fn php_source_catalog(connection: &mut SqliteConnection) -> Res .cloned() .collect(); let actual = catalog(connection).await?; - if actual != expected { - let mut current = super::upgrade::source_catalog_variant( - &expected, + let layout = super::upgrade::source_layout( + &actual, + expected, + [ include_str!("../../upgrade/php-2.8.0/sqlite.json"), - )?; - current.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); - if actual != current { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_unqualified", - )); - } - } - Ok(actual) + include_str!("../../upgrade/php-2.9.7/sqlite.json"), + ], + |row| (row.0.clone(), row.1.clone()), + )?; + Ok((actual, layout)) } pub(super) async fn verify(connection: &mut SqliteConnection) -> Result<()> { @@ -180,7 +193,10 @@ mod tests { } // This receipt acknowledges an actual SQLite boundary inside the // still-open outer transaction. The parent sends a real SIGKILL. - std::fs::write(format!("{path}.boundary"), phase).unwrap(); + // The parent waits for the receipt to exist, so it must appear with + // its contents: write beside it and rename into place. + std::fs::write(format!("{path}.boundary.partial"), &phase).unwrap(); + std::fs::rename(format!("{path}.boundary.partial"), format!("{path}.boundary")).unwrap(); loop { std::thread::park(); } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6208b3123..3c1499129 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -410,7 +410,8 @@ where .into_iter() .map(|e| { json!({"sequence": e["sequence"], - "event_type": e["event_type"], "timestamp": e["recorded_at"], "payload": e["payload"]}) + "event_type": e["event_type"], "timestamp": e["recorded_at"], + "principal": super::history_presentation::principal(&e["payload"]), "payload": e["payload"]}) }) .collect(); Ok( @@ -2742,9 +2743,12 @@ where } fn event(row: DB::Row) -> Result { + let event_type = DB::string(&row, "event_type")?; + let payload = + super::history_presentation::payload(DB::document_row(&row, "payload")?, &event_type); Ok( json!({"id": DB::string(&row,"id")?, "sequence": DB::number(&row,"sequence")?, "event_type": DB::string(&row,"event_type")?, - "payload": DB::document_row(&row,"payload")?, + "payload": payload, "recorded_at": DB::instant(&row,"recorded_at")?.to_rfc3339_opts(chrono::SecondsFormat::Micros, true), "workflow_task_id": DB::optional_string(&row,"workflow_task_id")?, "workflow_command_id": DB::optional_string(&row,"workflow_command_id")?}), ) diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 6172da77c..e3f3f7b44 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -1,5 +1,7 @@ //! Read-only PHP source inventory. This command never grants takeover authority. pub mod external_reference; +pub mod fingerprint; +pub mod takeover; mod payload_scan; pub mod queue; pub mod queue_binding; @@ -20,6 +22,44 @@ use tokio::time::{Instant, timeout, timeout_at}; const BUDGET: Duration = Duration::from_secs(30); +/// Published PHP layouts after the frozen one, oldest first. Each backend +/// records one delta per layout, applied to the layout before it. +pub(super) const SOURCE_LAYOUTS: [&str; 2] = ["php-2.8.0", "php-2.9.7"]; + +/// Names the qualified layout `actual` matches exactly: the frozen layout, or +/// one reached by applying the recorded deltas in order. Rows are compared in +/// `order`, so only presentation order, never a definition, is normalized. +pub(super) fn source_layout( + actual: &[T], + frozen: Vec, + deltas: [&str; 2], + order: impl Fn(&T) -> K, +) -> Result<&'static str> +where + T: Clone + PartialEq + DeserializeOwned, +{ + let sorted = |rows: &[T]| { + let mut rows = rows.to_vec(); + rows.sort_by_key(&order); + rows + }; + let actual = sorted(actual); + let mut layout = sorted(&frozen); + if actual == layout { + return Ok("frozen"); + } + for (name, delta) in SOURCE_LAYOUTS.into_iter().zip(deltas) { + layout = sorted(&source_catalog_variant(&layout, delta)?); + if actual == layout { + return Ok(name); + } + } + Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )) +} + /// Exact published-catalog additions/replacements, applied only in memory. /// Neither unknown objects nor unmatched old definitions are discarded. pub(super) fn source_catalog_variant(baseline: &[T], delta: &str) -> Result> @@ -97,9 +137,62 @@ async fn close(future: impl Future Ok(()) } +/// A dedicated MySQL or MariaDB source connection, with its backend name. +/// Lock waits stay short and transactions are repeatable reads. The client +/// deadline drops the socket, but the server would keep a running statement, +/// its metadata locks and its snapshot, so `statement_seconds` bounds each +/// statement server-side; zero leaves that to the client deadline. With `utc`, +/// TIMESTAMP values render in UTC. +async fn mysql_source( + options: MySqlConnectOptions, + deadline: Instant, + utc: bool, + statement_seconds: u32, +) -> Result<(MySqlConnection, &'static str)> { + let mut connection = timeout_at( + deadline, + MySqlConnection::connect_with(&mysql::source_options(options)), + ) + .await + .map_err(|_| expired())??; + let setup = timeout_at(deadline, async { + let version: String = sqlx::query_scalar("SELECT VERSION()") + .fetch_one(&mut connection) + .await?; + let (backend, limit) = if version.contains("MariaDB") { + ("mariadb", format!("max_statement_time={statement_seconds}")) + } else { + let milliseconds = u64::from(statement_seconds) * 1000; + ("mysql", format!("max_execution_time={milliseconds}")) + }; + let zone = if utc { "time_zone='+00:00'," } else { "" }; + sqlx::query(AssertSqlSafe(format!( + "SET {zone}sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,{limit}" + ))) + .execute(&mut connection) + .await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + .execute(&mut connection) + .await?; + Ok(backend) + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + match setup { + Ok(backend) => Ok((connection, backend)), + Err(error) => { + // The setup failure explains more than a failed close would. + let _ = close(connection.close()).await; + Err(error) + } + } +} + fn report( backend: &str, catalog: &impl Serialize, + layout: &str, counts: BTreeMap, queue_inspection: Option, payload_inspection: Option, @@ -111,6 +204,7 @@ fn report( let mut report = json!({ "schema":"durable-workflow.rust-upgrade-source-inspection.v1", "status":"inspected", "backend":backend, "catalog_visibility":"configured_role", + "source_layout":layout, "catalog_sha256":digest, "table_rows":counts, "source_mutated":false, "takeover_allowed":false, "backup_verified":false, "php_quiescence_verified":false, "stored_representation_compatibility_verified":false @@ -189,65 +283,68 @@ pub async fn inspect_sqlite_cli( ) -> Result { timeout(BUDGET, async { queue_config(queues)?; - if path.is_empty() || path.contains(['?', ':']) { + let resolved = readonly_source(path).await?; + inspect_sqlite_inner(&resolved, queues, payloads).await + }) + .await + .map_err(|_| expired())? +} + +/// Resolves a CLI source file and requires that it and its directory are on a +/// kernel read-only mount. +pub(super) async fn readonly_source(path: &str) -> Result { + if path.is_empty() || path.contains(['?', ':']) { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_file_required", + )); + } + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + // SQLite resolves symlinks before locating its sidecars. Check the + // resolved parent as well: a file-only read-only bind mount can + // still have a writable directory beside it. + let resolved = tokio::fs::canonicalize(path).await?; + let mut options = tokio::fs::OpenOptions::new(); + options.read(true).custom_flags(libc::O_NONBLOCK); + let file = options.open(&resolved).await?; + if !file.metadata().await?.is_file() { return Err(refuse( StatusCode::BAD_REQUEST, - "upgrade_source_file_required", + "upgrade_source_regular_file_required", )); } - #[cfg(unix)] - { - use std::os::fd::AsRawFd; - // SQLite resolves symlinks before locating its sidecars. Check the - // resolved parent as well: a file-only read-only bind mount can - // still have a writable directory beside it. - let resolved = tokio::fs::canonicalize(path).await?; - let mut options = tokio::fs::OpenOptions::new(); - options.read(true).custom_flags(libc::O_NONBLOCK); - let file = options.open(&resolved).await?; - if !file.metadata().await?.is_file() { + let parent = options + .open(resolved.parent().ok_or_else(|| { + refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required") + })?) + .await?; + for descriptor in [&file, &parent] { + let mut filesystem = std::mem::MaybeUninit::::uninit(); + // fstatvfs uses this live descriptor's mount. Only read the + // returned structure after success. + if unsafe { libc::fstatvfs(descriptor.as_raw_fd(), filesystem.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error().into()); + } + let filesystem = unsafe { filesystem.assume_init() }; + if filesystem.f_flag & libc::ST_RDONLY == 0 { return Err(refuse( - StatusCode::BAD_REQUEST, - "upgrade_source_regular_file_required", + StatusCode::CONFLICT, + "upgrade_source_readonly_filesystem_required", )); } - let parent = options - .open(resolved.parent().ok_or_else(|| { - refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required") - })?) - .await?; - for descriptor in [&file, &parent] { - let mut filesystem = std::mem::MaybeUninit::::uninit(); - // fstatvfs uses this live descriptor's mount. Only read the - // returned structure after success. - if unsafe { libc::fstatvfs(descriptor.as_raw_fd(), filesystem.as_mut_ptr()) } != 0 { - return Err(std::io::Error::last_os_error().into()); - } - let filesystem = unsafe { filesystem.assume_init() }; - if filesystem.f_flag & libc::ST_RDONLY == 0 { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_readonly_filesystem_required", - )); - } - } - return inspect_sqlite_inner( - resolved.to_str().ok_or_else(|| { - refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required") - })?, - queues, - payloads, - ) - .await; } - #[cfg(not(unix))] - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_readonly_filesystem_unavailable", - )); - }) - .await - .map_err(|_| expired())? + return Ok(resolved + .to_str() + .ok_or_else(|| refuse(StatusCode::BAD_REQUEST, "upgrade_source_file_required"))? + .to_owned()); + } + #[cfg(not(unix))] + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_readonly_filesystem_unavailable", + )); } async fn inspect_sqlite_inner( @@ -257,6 +354,190 @@ async fn inspect_sqlite_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; + let mut connection = open_sqlite_source(path, deadline).await?; + let result = timeout_at(deadline, async { + let mut tx = connection.begin().await?; + let report = sqlite_inventory(&mut tx, queues, payloads).await; + tx.rollback().await?; + report + }) + .await + .map_err(|_| expired()) + .and_then(|result| result); + close(connection.close()).await?; + result +} + +/// The SQLite inspection report, read inside the caller's transaction. +pub(super) async fn sqlite_inventory( + tx: &mut SqliteConnection, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, +) -> Result { + let (catalog, layout) = schema::php_source_catalog(&mut *tx).await?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, _, _)| kind == "table") + .map(|(_, name, _, _)| name.clone()) + .collect(); + let counts = count_tables!(tx, tables, '"'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + report( + "sqlite", + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) +} + +/// Opens a qualified rollback-journal source file read-only. +pub(super) async fn open_sqlite_source(path: &str, deadline: Instant) -> Result { + open_sqlite_file(path, deadline, false).await +} + +/// The file format version bytes of a SQLite header: rollback journal or WAL. +const ROLLBACK_JOURNAL: [u8; 2] = [1, 1]; +const WRITE_AHEAD_LOG: [u8; 2] = [2, 2]; + +/// The journal format bytes of a regular SQLite file, read without SQLite. +async fn journal_format(path: &str) -> Result<[u8; 2]> { + let regular_file_required = || { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_regular_file_required", + ) + }; + if !tokio::fs::metadata(path).await?.is_file() { + return Err(regular_file_required()); + } + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + // Tokio filesystem opens use blocking workers. A replaced FIFO must + // not strand one beyond the observation budget or prevent CLI exit. + #[cfg(unix)] + options.custom_flags(libc::O_NONBLOCK); + let mut file = options.open(path).await?; + if !file.metadata().await?.is_file() { + return Err(regular_file_required()); + } + let mut header = [0_u8; 20]; + file.read_exact(&mut header).await?; + if &header[..16] != b"SQLite format 3\0" { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok([header[18], header[19]]) +} + +/// Whether a SQLite file has a WAL, shared-memory or rollback journal beside it. +async fn sidecars(path: &str) -> Result { + for suffix in ["-wal", "-shm", "-journal"] { + if tokio::fs::try_exists(format!("{path}{suffix}")).await? { + return Ok(true); + } + } + Ok(false) +} + +/// Prepares a stopped PHP SQLite source for the upgrade commands, which need +/// the rollback journal. PHP Server's default is WAL: switching it makes +/// SQLite checkpoint every committed WAL frame into the file first, so the +/// stored rows are unchanged. A journal left by a crashed writer is rolled +/// back the same way. Refuses while another process has the file open. +pub async fn prepare_sqlite_source(path: &str) -> Result { + if path.is_empty() || path.contains(['?', ':']) { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_file_required", + )); + } + let deadline = Instant::now() + BUDGET; + let previous = match timeout_at(deadline, journal_format(path)) + .await + .map_err(|_| expired())?? + { + WRITE_AHEAD_LOG => "wal", + ROLLBACK_JOURNAL => "delete", + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + }; + let switched = previous == "wal" || sidecars(path).await?; + if switched { + let in_use = || refuse(StatusCode::CONFLICT, "upgrade_source_in_use"); + let opened = timeout_at( + deadline, + SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(path) + .create_if_missing(false) + .foreign_keys(false) + .busy_timeout(Duration::from_secs(2)) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete), + ), + ) + .await + .map_err(|_| expired())?; + // Leaving WAL needs the only connection. SQLite answers busy when another + // still has the file after a short wait, which also covers a writer + // that is just closing. + let mut connection = match opened { + Ok(connection) => connection, + Err(sqlx::Error::Database(error)) + if error + .code() + .and_then(|code| code.parse::().ok()) + .is_some_and(|code| code & 0xff == 5) => + { + return Err(in_use()); + } + Err(error) => return Err(error.into()), + }; + let mode: String = sqlx::query_scalar("PRAGMA journal_mode") + .fetch_one(&mut connection) + .await?; + close(connection.close()).await?; + if mode != "delete" { + return Err(in_use()); + } + } + if journal_format(path).await? != ROLLBACK_JOURNAL || sidecars(path).await? { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok(json!({ + "schema":"durable-workflow.rust-upgrade-sqlite-prepare.v1", + "status":"rollback_journal", "previous_journal_mode":previous, + "source_mutated":switched, "takeover_allowed":false, "backup_verified":false + })) +} + +/// Opens a qualified rollback-journal source file, writable only for takeover. +/// It keeps the rollback journal: native enables WAL itself after takeover. +pub(super) async fn open_sqlite_file( + path: &str, + deadline: Instant, + writable: bool, +) -> Result { if path.is_empty() || path.contains(['?', ':']) { return Err(refuse( StatusCode::BAD_REQUEST, @@ -267,35 +548,24 @@ async fn inspect_sqlite_inner( // not open an unqualified WAL source or attempt journal recovery. Unlike // immutable=1 this preserves normal SQLite locking/change detection for // the supported rollback-journal source, rather than ignoring live WAL. + // A WAL source is first switched with upgrade-prepare-sqlite. timeout_at(deadline, async { - let regular_file_required = || { - refuse( - StatusCode::BAD_REQUEST, - "upgrade_source_regular_file_required", - ) - }; - if !tokio::fs::metadata(path).await?.is_file() { - return Err(regular_file_required()); - } - let mut options = tokio::fs::OpenOptions::new(); - options.read(true); - // Tokio filesystem opens use blocking workers. A replaced FIFO must - // not strand one beyond the observation budget or prevent CLI exit. - #[cfg(unix)] - options.custom_flags(libc::O_NONBLOCK); - let mut file = options.open(path).await?; - if !file.metadata().await?.is_file() { - return Err(regular_file_required()); - } - let mut header = [0_u8; 20]; - file.read_exact(&mut header).await?; - if &header[..16] != b"SQLite format 3\0" || header[18..20] != [1, 1] { + if journal_format(path).await? != ROLLBACK_JOURNAL { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_sqlite_journal_unqualified", )); } - for suffix in ["-wal", "-shm", "-journal"] { + // A killed takeover leaves its rollback journal. Takeover opens the + // file writable, so SQLite rolls that journal back before anything is + // read, and the fingerprint then proves the result is the backed-up + // PHP database. Read-only opens never attempt that recovery. + let sidecars: &[&str] = if writable { + &["-wal", "-shm"] + } else { + &["-wal", "-shm", "-journal"] + }; + for suffix in sidecars { if tokio::fs::try_exists(format!("{path}{suffix}")).await? { return Err(refuse( StatusCode::CONFLICT, @@ -307,52 +577,20 @@ async fn inspect_sqlite_inner( }) .await .map_err(|_| expired())??; - let mut connection = timeout_at( + let connection = timeout_at( deadline, SqliteConnection::connect_with( &SqliteConnectOptions::new() .filename(path) - .read_only(true) + .read_only(!writable) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete) .create_if_missing(false) .foreign_keys(false), ), ) .await .map_err(|_| expired())??; - let result = timeout_at(deadline, async { - let mut tx = connection.begin().await?; - let catalog = schema::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, _, _)| kind == "table") - .map(|(_, name, _, _)| name.clone()) - .collect(); - let counts = count_tables!(tx, tables, '"'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; - tx.rollback().await?; - report( - "sqlite", - &catalog, - counts, - queue_inspection, - payload_inspection, - ) - }) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result + Ok(connection) } pub async fn inspect_mysql(options: MySqlConnectOptions) -> Result { @@ -377,70 +615,68 @@ async fn inspect_mysql_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( - deadline, - MySqlConnection::connect_with(&mysql::source_options(options)), - ) - .await - .map_err(|_| expired())??; - let result = timeout_at(deadline, async { - let version: String = sqlx::query_scalar("SELECT VERSION()") - .fetch_one(&mut connection) - .await?; - let backend = if version.contains("MariaDB") { - "mariadb" - } else { - "mysql" - }; - // The client deadline drops the socket, but the server would keep a running - // statement, its metadata locks and its snapshot. Bound them server-side. - sqlx::query(if backend == "mariadb" { - "SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_statement_time=30" - } else { - "SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5,max_execution_time=30000" - }) - .execute(&mut connection) - .await?; - sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - .execute(&mut connection) - .await?; + let (connection, backend) = mysql_source(options, deadline, false, 30).await?; + bounded(connection, deadline, async |connection| { let mut tx = connection.begin_with("START TRANSACTION READ ONLY").await?; - let catalog = mysql::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); - let counts = count_tables!(tx, tables, '`'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; - // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible - // catalog rather than joining counts to a different schema. - if mysql::php_source_catalog(&mut tx).await? != catalog { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_changed", - )); - } + let report = mysql_inventory(&mut tx, backend, queues, payloads, false).await; tx.rollback().await?; - report( - backend, - &catalog, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result +} + +/// The PHP source catalog, or with `fenced` the catalog of a source whose +/// takeover marker is all that has been written, with that marker set aside. +pub(super) async fn mysql_source_catalog( + connection: &mut MySqlConnection, + fenced: bool, +) -> Result<(Vec<(String, String)>, &'static str)> { + if fenced { + mysql::fenced_source_catalog(connection).await + } else { + mysql::php_source_catalog(connection).await + } +} + +/// The MySQL or MariaDB inspection report, read inside the caller's +/// transaction. +pub(super) async fn mysql_inventory( + tx: &mut MySqlConnection, + backend: &str, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, + fenced: bool, +) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); + let counts = count_tables!(tx, tables, '`'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible + // catalog rather than joining counts to a different schema. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + report( + backend, + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) } pub async fn inspect_postgres(options: PgConnectOptions) -> Result { @@ -465,46 +701,77 @@ async fn inspect_postgres_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( + let connection = timeout_at( deadline, PgConnection::connect_with(&postgres::source_options(options)), ) .await .map_err(|_| expired())??; - let result = timeout_at(deadline, async { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - let catalog = postgres::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") - .map(|(_, name, _)| name.clone()) - .collect(); - let counts = count_tables!(tx, tables, '"'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; + let report = postgres_inventory(&mut tx, queues, payloads).await; tx.rollback().await?; - report( - "pgsql", - &catalog, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); +} + +/// The PostgreSQL inspection report, read inside the caller's transaction. +pub(super) async fn postgres_inventory( + tx: &mut PgConnection, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, +) -> Result { + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + let counts = count_tables!(tx, tables, '"'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + report( + "pgsql", + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) +} + +/// The frozen migration from its ownership marker on, then the incarnation +/// migration: what a fresh native bootstrap adds after the frozen PHP layout. +pub(super) fn native_relations(frozen: &str, marker: &str, incarnations: &str) -> Result { + let start = frozen + .find(marker) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; + Ok(format!("{}\n{incarnations}", &frozen[start..])) +} + +/// Runs work on a dedicated source connection within the deadline, then +/// closes the connection whatever the work returned. +pub(super) async fn bounded( + mut connection: C, + deadline: Instant, + work: impl AsyncFnOnce(&mut C) -> Result, +) -> Result { + let result = timeout_at(deadline, work(&mut connection)) + .await + .map_err(|_| expired()) + .and_then(|result| result); close(connection.close()).await?; result } diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs new file mode 100644 index 000000000..df82bb3b8 --- /dev/null +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -0,0 +1,294 @@ +//! Logical fingerprint of a stopped PHP source database. +//! +//! The digest covers the qualified source catalog and every row of every table +//! in primary-key order, with each value rendered by the database itself in one +//! fixed text form. A stopped source and its backup restored into a separate +//! database give equal digests only when the restore reproduces every row, and +//! takeover compares it again before writing anything. Only the digest and the +//! per-table row counts leave the database. The fingerprint grants nothing: +//! every compatibility, backup, quiescence and takeover flag stays false. +use super::{ + Result, bounded, expired, mysql_source, mysql_source_catalog, open_sqlite_source, postgres, + readonly_source, refuse, schema, +}; +use axum::http::StatusCode; +use futures_util::TryStreamExt; +use serde::Serialize; +use serde_json::{Value, json}; +use sha2::{Digest as _, Sha256}; +use sqlx::{ + AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, SqliteConnection, + mysql::MySqlConnectOptions, postgres::PgConnectOptions, +}; +use std::{collections::BTreeMap, time::Duration}; +use tokio::time::{Instant, timeout, timeout_at}; + +/// A fingerprint reads every row, so it gets far longer than an inspection. +const BUDGET: Duration = Duration::from_secs(3600); + +struct Digest { + hasher: Sha256, + rows: BTreeMap, + started: Instant, +} + +impl Digest { + fn new(backend: &str, catalog: &impl Serialize) -> Result { + let mut digest = Self { + hasher: Sha256::new(), + rows: BTreeMap::new(), + started: Instant::now(), + }; + digest.frame(b"durable-workflow.rust-upgrade-source-fingerprint.v1"); + digest.frame(backend.as_bytes()); + digest.frame(&serde_json::to_vec(catalog)?); + Ok(digest) + } + + /// Length-prefixed, so no value can be mistaken for a boundary. + fn frame(&mut self, bytes: &[u8]) { + self.hasher.update((bytes.len() as u64).to_be_bytes()); + self.hasher.update(bytes); + } + + fn table(&mut self, name: &str, columns: &[String]) { + self.frame(b"table"); + self.frame(name.as_bytes()); + self.frame(&(columns.len() as u64).to_be_bytes()); + for column in columns { + self.frame(column.as_bytes()); + } + } + + fn report(self, backend: &str, layout: &str) -> Value { + let hex: String = self + .hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + json!({ + "schema":"durable-workflow.rust-upgrade-source-fingerprint.v1", + "status":"fingerprinted", "backend":backend, "source_layout":layout, + "fingerprint":format!("sha256:{hex}"), + "table_rows":self.rows, "elapsed_ms":self.started.elapsed().as_millis() as u64, + "source_mutated":false, "takeover_allowed":false, "backup_verified":false, + "php_quiescence_verified":false, "stored_representation_compatibility_verified":false + }) + } +} + +/// Names come from a catalog equal to a qualified PHP layout. Anything else is +/// refused rather than quoted. +pub(super) fn identifier(name: &str) -> Result<&str> { + if name.is_empty() || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + Ok(name) +} + +/// Primary-key order, or every rendered column when a table has no key. +fn order(key: &[String], width: usize, quote: char) -> Result { + if key.is_empty() { + return Ok((1..=width).map(|n| n.to_string()).collect::>().join(",")); + } + Ok(key + .iter() + .map(|column| identifier(column).map(|column| format!("{quote}{column}{quote}"))) + .collect::>>()? + .join(",")) +} + +/// Every row of one table in primary-key order, with each column rendered by +/// `render(column, position)`. +fn row_select( + from: &str, + columns: &[String], + key: &[String], + quote: char, + render: impl Fn(&str, usize) -> String, +) -> Result { + let select = columns + .iter() + .enumerate() + .map(|(position, column)| identifier(column).map(|column| render(column, position))) + .collect::>>()? + .join(","); + Ok(format!( + "SELECT {select} FROM {from} ORDER BY {}", + order(key, columns.len(), quote)? + )) +} + +// Every rendered value is text, framed in column order. +macro_rules! hash_rows { + ($tx:ident, $digest:ident, $table:expr, $sql:expr, $width:expr) => {{ + let mut count = 0_i64; + let mut rows = sqlx::query(AssertSqlSafe($sql)).fetch(&mut *$tx); + while let Some(row) = rows.try_next().await? { + $digest.frame(b"row"); + for index in 0..$width { + let value: String = row.try_get(index)?; + $digest.frame(value.as_bytes()); + } + count += 1; + } + drop(rows); + $digest.rows.insert($table.to_owned(), count); + }}; +} + +/// Fingerprints a SQLite source the operator CLI found on a read-only mount. +pub async fn fingerprint_sqlite_cli(path: &str) -> Result { + timeout(BUDGET, async { + let resolved = readonly_source(path).await?; + fingerprint_sqlite(&resolved).await + }) + .await + .map_err(|_| expired())? +} + +pub async fn fingerprint_sqlite(path: &str) -> Result { + let deadline = Instant::now() + BUDGET; + let connection = open_sqlite_source(path, deadline).await?; + bounded(connection, deadline, async |connection| { + // One deferred read transaction keeps one snapshot of the file. + let mut tx = connection.begin().await?; + let report = sqlite_digest(&mut tx).await; + tx.rollback().await?; + report + }) + .await +} + +/// The SQLite fingerprint report, read inside the caller's transaction. +pub(super) async fn sqlite_digest(tx: &mut SqliteConnection) -> Result { + let (catalog, layout) = schema::php_source_catalog(&mut *tx).await?; + let mut digest = Digest::new("sqlite", &catalog)?; + for (_, table, _, _) in catalog.iter().filter(|(kind, _, _, _)| kind == "table") { + let table = identifier(table)?; + let columns: Vec = + sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) ORDER BY cid") + .bind(table) + .fetch_all(&mut *tx) + .await?; + let key: Vec = + sqlx::query_scalar("SELECT name FROM pragma_table_info(?1) WHERE pk>0 ORDER BY pk") + .bind(table) + .fetch_all(&mut *tx) + .await?; + digest.table(table, &columns); + // quote() keeps the storage class: integers, text and blobs render + // exactly. Reals use 17 significant digits, which round-trip. + let sql = row_select(&format!("\"{table}\""), &columns, &key, '"', |c, _| { + format!("CASE typeof(\"{c}\") WHEN 'real' THEN printf('%!.17g',\"{c}\") ELSE quote(\"{c}\") END") + })?; + hash_rows!(tx, digest, table, sql, columns.len()); + } + Ok(digest.report("sqlite", layout)) +} + +pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { + let deadline = Instant::now() + BUDGET; + let connection = timeout_at( + deadline, + PgConnection::connect_with(&postgres::source_options(options)), + ) + .await + .map_err(|_| expired())??; + bounded(connection, deadline, async |connection| { + let mut tx = connection + .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + .await?; + let report = postgres_digest(&mut tx).await; + tx.rollback().await?; + report + }) + .await +} + +/// The PostgreSQL fingerprint report, read inside the caller's transaction. +/// Its settings last until that transaction ends. +pub(super) async fn postgres_digest(tx: &mut PgConnection) -> Result { + // Fixed text forms for timestamps, intervals, floats and bytes. Long table + // reads are expected, so only the client deadline bounds them. + sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'; SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let mut digest = Digest::new("pgsql", &catalog)?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + for table in &tables { + let table = identifier(table)?; + let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") + .bind(table).fetch_all(&mut *tx).await?; + digest.table(table, &columns); + let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { + format!("quote_nullable(\"{c}\")") + })?; + hash_rows!(tx, digest, table, sql, columns.len()); + } + Ok(digest.report("pgsql", layout)) +} + +pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { + let deadline = Instant::now() + BUDGET; + // TIMESTAMP values render in UTC. Long table reads are expected, but lock + // waits stay short; the client deadline bounds the whole read. + let (connection, backend) = mysql_source(options, deadline, true, 0).await?; + bounded(connection, deadline, async |connection| { + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let report = mysql_digest(&mut tx, backend, false).await; + tx.rollback().await?; + report + }) + .await +} + +/// The MySQL or MariaDB fingerprint report, read inside the caller's +/// consistent snapshot. With `fenced`, a takeover's own marker is set aside, +/// so the digest is the one the backup had. +pub(super) async fn mysql_digest(tx: &mut MySqlConnection, backend: &str, fenced: bool) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let mut digest = Digest::new(backend, &catalog)?; + for (table, _) in &catalog { + let table = identifier(table)?; + let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); + digest.table(table, &names); + // Binary values render as hex literals; everything else as its + // quoted string form. + let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { + match kinds[index].to_ascii_lowercase().as_str() { + "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( + "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" + ), + _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), + } + })?; + hash_rows!(tx, digest, table, sql, names.len()); + } + // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that + // changed while the rows were read. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + Ok(digest.report(backend, layout)) +} diff --git a/rust/src/runtime/upgrade/payload_scan.rs b/rust/src/runtime/upgrade/payload_scan.rs index 555eea3b2..412c2b91e 100644 --- a/rust/src/runtime/upgrade/payload_scan.rs +++ b/rust/src/runtime/upgrade/payload_scan.rs @@ -72,11 +72,15 @@ macro_rules! column { }; } -const COLUMNS: [Column; 4] = [ +const COLUMNS: [Column; 8] = [ column!("workflow_runs", "arguments", "payload_codec"), column!("workflow_runs", "output", "output_payload_codec"), column!("activity_executions", "arguments", "payload_codec"), column!("activity_executions", "result", "payload_codec"), + column!("workflow_commands", "payload", "payload_codec"), + column!("workflow_signal_records", "arguments", "payload_codec"), + column!("workflow_updates", "arguments", "payload_codec"), + column!("workflow_updates", "result", "payload_codec"), ]; #[derive(Default, Serialize)] @@ -218,9 +222,10 @@ where "references_metadata_supported":references_supported, "diagnostics":diagnostics, "diagnostics_omitted":omitted, "inline_values_verified":inline_supported, "nested_payloads_verified":false, - // Published migrations also give these tables codec-tagged payloads. + // These tables also hold codec-tagged payloads; takeover refuses a + // source with any of their rows, since native lacks the features. "other_root_payload_columns_verified":false, - "unscanned_payload_codec_tables":["workflow_commands","workflow_signal_records","workflow_updates"], + "unscanned_payload_codec_tables":["workflow_durable_stream_items","workflow_inbound_stream_items","workflow_update_validation_tasks"], "inline_validation_limits":{"encoded_frame_bytes":65536,"traversal_depth":64, "traversal_visits":16384,"scalar_allocation_bytes":apache_avro::util::max_allocation_bytes(apache_avro::util::DEFAULT_MAX_ALLOCATION_BYTES)}, "provider_bytes_verified":false, "registry_binding_verified":false, diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs new file mode 100644 index 000000000..0d9188b83 --- /dev/null +++ b/rust/src/runtime/upgrade/takeover.rs @@ -0,0 +1,467 @@ +//! Backup-first, one-way takeover of a stopped PHP database, following +//! docs/rust-server-upgrade-takeover.md. +//! +//! On SQLite and PostgreSQL everything happens in one transaction that no +//! other writer can interleave with: the inspections, the fingerprint check +//! against the operator's verified backup, the layout conversion, the native +//! relations, the migration ledger and native's own startup verification. An +//! interrupted or refused takeover therefore leaves the PHP database exactly +//! as it was, and a committed one is a native database that PHP 2.5.14 and +//! later refuse to open. +use super::{ + Result, bounded, fingerprint, open_sqlite_file, postgres, postgres_inventory, + queue_binding::QueueSourceConfig, queue_config, refuse, schema, sqlite_inventory, +}; +use crate::runtime::backend::Backend; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{ + AssertSqlSafe, Connection, Encode, Executor, IntoArguments, PgConnection, Postgres, Sqlite, + SqliteConnection, Transaction, Type, + migrate::{Migrate, Migration}, + postgres::PgConnectOptions, +}; +use std::time::Duration; +use tokio::time::{Instant, timeout_at}; + +mod mysql_journal; +pub use mysql_journal::take_over_mysql; + +/// Takeover reads every row twice (inspection and fingerprint). +const BUDGET: Duration = Duration::from_secs(3600); +/// Native's PostgreSQL bootstrap key, so a native node cannot open the +/// database while it is taken over. +const POSTGRES_BOOTSTRAP_KEY: i64 = 4924501473315676162; + +pub async fn take_over_sqlite( + path: &str, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = open_sqlite_file(path, deadline, true).await?; + bounded(connection, deadline, async |connection| { + let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; + let outcome = take_over_sqlite_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +/// Takes over with the owner's credentials. The connection keeps native's +/// short lock wait, so a PHP process still holding a table refuses the +/// takeover instead of stalling it. +pub async fn take_over_postgres( + options: PgConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = timeout_at( + deadline, + PgConnection::connect_with(&postgres::source_options(options)), + ) + .await + .map_err(|_| super::expired())??; + bounded(connection, deadline, async |connection| { + // Read committed: once every table is locked, each statement reads + // the latest committed state and nothing else can change it. + let mut tx = connection.begin().await?; + let outcome = take_over_postgres_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +fn preconditions(backup_fingerprint: &str, queues: &QueueSourceConfig<'_>) -> Result<()> { + queue_config(Some(queues))?; + if !backup_fingerprint.starts_with("sha256:") || backup_fingerprint.len() != 71 { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_backup_fingerprint_required", + )); + } + Ok(()) +} + +/// Commits a completed takeover, and rolls back anything else. +async fn settle(tx: Transaction<'_, DB>, outcome: Result) -> Result { + match outcome { + Ok(report) => { + tx.commit().await?; + Ok(report) + } + Err(error) => { + tx.rollback().await?; + Err(error) + } + } +} + +async fn take_over_sqlite_in( + tx: &mut SqliteConnection, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + let inventory = sqlite_inventory(&mut *tx, Some(queues), true).await?; + let layout = qualified(&inventory)?; + // The same digest the operator compared with the restored backup. Any + // write after that backup changes it, so nothing below can run. + let digest = fingerprint::sqlite_digest(&mut *tx).await?; + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + .execute(&mut *tx) + .await?; + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, schema::migration()).await?; + // Exactly the check native startup makes: marker, ledger and catalog. + schema::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point("before-commit"); + Ok(report("sqlite", layout, &digest, &inventory, converted)) +} + +async fn take_over_postgres_in( + tx: &mut PgConnection, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + // Whole-table reads and index builds are expected; the client deadline + // bounds them, while lock waits stay short. + sqlx::raw_sql("SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(POSTGRES_BOOTSTRAP_KEY) + .execute(&mut *tx) + .await?; + // The later DDL needs ACCESS EXCLUSIVE anyway; taking it first means no + // lock is ever upgraded while another session waits. + let tables: Vec = sqlx::query_scalar("SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname") + .fetch_all(&mut *tx) + .await?; + if tables.is_empty() { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + let tables = tables + .iter() + .map(|table| fingerprint::identifier(table).map(|table| format!("public.\"{table}\""))) + .collect::>>()? + .join(","); + sqlx::raw_sql(AssertSqlSafe(format!( + "LOCK TABLE {tables} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(&mut *tx) + .await + .map_err(|error| match &error { + sqlx::Error::Database(database) if database.code().as_deref() == Some("55P03") => { + refuse(StatusCode::CONFLICT, "upgrade_source_in_use") + } + _ => error.into(), + })?; + let inventory = postgres_inventory(&mut *tx, Some(queues), true).await?; + let layout = qualified(&inventory)?; + let digest = fingerprint::postgres_digest(&mut *tx).await?; + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(postgres::native_relations_sql()?)) + .execute(&mut *tx) + .await?; + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, postgres::migration()).await?; + postgres::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point("before-commit"); + Ok(report("pgsql", layout, &digest, &inventory, converted)) +} + +/// Lets the interruption qualification kill a real takeover process at a +/// named boundary: `before-commit` unless `DW_TAKEOVER_TEST_BOUNDARY` names +/// another. +#[cfg(test)] +fn interruption_point(boundary: &str) { + let wanted = std::env::var("DW_TAKEOVER_TEST_BOUNDARY"); + if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") + && wanted.as_deref().unwrap_or("before-commit") == boundary + { + std::fs::write(format!("{receipt}.partial"), boundary).unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); + loop { + std::thread::park(); + } + } +} + +fn matches_backup(digest: &Value, backup_fingerprint: &str) -> Result<()> { + if digest["fingerprint"] != backup_fingerprint { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + Ok(()) +} + +fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, converted: Value) -> Value { + json!({ + "schema":"durable-workflow.rust-upgrade-takeover.v1", + "status":"taken_over", "backend":backend, "source_layout":layout, + "fingerprint":digest["fingerprint"], "table_rows":inventory["table_rows"], + "converted":converted, "backup_fingerprint_matched":true, + // What the inspections cannot yet establish stays explicit. + "nested_payloads_verified":false, "other_root_payload_columns_verified":false, + "stored_representation_compatibility_verified":false + }) +} + +/// Published PHP features native does not implement yet. A source holding any +/// of their rows is refused, rather than taken over without them. Derived +/// projections and PHP's own queue and poll bookkeeping are left in place. +const UNSUPPORTED_FEATURES: [(&str, &[&str]); 7] = [ + ("upgrade_source_runtime_credentials_unsupported", &["runtime_credentials"]), + ( + "upgrade_source_message_streams_unsupported", + &[ + "workflow_messages", + "workflow_durable_streams", + "workflow_durable_stream_items", + "workflow_inbound_streams", + "workflow_inbound_stream_items", + ], + ), + ( + "upgrade_source_services_unsupported", + &["workflow_services", "workflow_service_endpoints", "workflow_service_operations"], + ), + ("upgrade_source_update_validation_unsupported", &["workflow_update_validation_tasks"]), + ("upgrade_source_worker_sessions_unsupported", &["workflow_worker_sessions"]), + ("upgrade_source_build_id_rollouts_unsupported", &["workflow_worker_build_id_rollouts"]), + ( + "upgrade_source_search_attribute_definitions_unsupported", + &["search_attribute_definitions"], + ), +]; + +/// Takeover needs every inspection result it relies on, and refuses what +/// native cannot represent yet: external payloads, service calls and the +/// other unsupported features above. Returns the qualified source layout. +fn qualified(inventory: &Value) -> Result<&str> { + for (reason, tables) in UNSUPPORTED_FEATURES { + if tables + .iter() + .any(|table| inventory["table_rows"][*table].as_i64().unwrap_or(0) != 0) + { + return Err(refuse(StatusCode::CONFLICT, reason)); + } + } + if inventory["queue_inspection"]["status"] != "known_subset_bound" { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_queue_binding_unqualified", + )); + } + let payloads = &inventory["payload_inspection"]; + if payloads["references_metadata_supported"] != true || payloads["inline_values_verified"] != true + { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_payload_metadata_unqualified", + )); + } + let total = |field: &str| { + payloads["columns"].as_array().map_or(0, |columns| { + columns + .iter() + .map(|column| column["counts"][field].as_i64().unwrap_or(0)) + .sum::() + }) + }; + if total("php_v1_references") != 0 + || total("unqualified_references") != 0 + || total("other_unvalidated") != 0 + || inventory["table_rows"]["runtime_external_payloads"] != 0 + { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_external_payloads_unsupported", + )); + } + inventory["source_layout"] + .as_str() + .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified")) +} + +fn query(sql: &'static str) -> sqlx::query::Query<'static, DB, DB::Arguments> +where + DB::Arguments: IntoArguments, +{ + sqlx::query(AssertSqlSafe(DB::statement(sql))) +} + +/// Records the native migration as applied, for a schema that came from +/// conversion. Native verification then checks this ledger like any other. +async fn record_migration( + tx: &mut DB::Connection, + migration: &'static Migration, +) -> Result<()> +where + DB::Connection: Migrate, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, +{ + tx.ensure_migrations_table("_sqlx_migrations").await?; + query::("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) + .await?; + Ok(()) +} + +/// Whether a recorded PHP release layout has PHP's own registration +/// incarnations. Layouts without a recipe are refused. +fn php_incarnations(layout: &str) -> Result { + match layout.trim_end_matches("+mariadb-driver") { + "frozen" => Ok(false), + "php-2.8.0" | "php-2.9.7" => Ok(true), + _ => Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_layout_unconverted", + )), + } +} + +/// Service calls are not part of native yet. +async fn service_calls_absent(tx: &mut DB::Connection) -> Result<()> +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, +{ + if count::(tx, "SELECT COUNT(*) AS n FROM workflow_service_calls").await? != 0 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_service_calls_unsupported", + )); + } + Ok(()) +} + +/// Converts a recorded PHP release layout to native's (the design's recipes). +async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let php_incarnations = php_incarnations(layout)?; + service_calls_absent::(tx).await?; + let (copied, created) = adopt_incarnations::(tx, php_incarnations).await?; + if php_incarnations { + sqlx::raw_sql( + "DROP TABLE workflow_worker_registration_incarnations; \ + ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ + DROP INDEX wf_service_calls_deadline_idx; \ + ALTER TABLE workflow_service_calls DROP COLUMN call_deadline_at;", + ) + .execute(&mut *tx) + .await?; + } + let delivery_references = if layout == "php-2.9.7" { + // PHP's own queue delivery lookup metadata; native does not use PHP's + // queue, and the queue inspection bound every job to its task row. + let references = count::(tx, DELIVERY_REFERENCES).await?; + sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .execute(&mut *tx) + .await?; + references + } else { + 0 + }; + Ok(json!({ + "incarnations_copied":copied, "incarnations_created":created, + "queue_delivery_references_removed":delivery_references + })) +} + +const DELIVERY_REFERENCES: &str = + "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL"; + +/// Gives every registration the incarnation native expects. PHP's own +/// incarnations and tokens are copied, so a running SDK worker's fenced +/// deregistration still matches; an active registration without one gets a +/// new incarnation. Returns how many were copied and created. +async fn adopt_incarnations( + tx: &mut DB::Connection, + php_incarnations: bool, +) -> Result<(u64, usize)> +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let now = super::super::store::now(); + let mut copied = 0; + if php_incarnations { + // Same statuses as native. Native keeps the recovered count only on + // deregistration and a finish time on every retired incarnation. + copied = DB::affected( + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$2),COALESCE(updated_at,created_at,$3) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .execute(&mut *tx) + .await?, + ); + let total = count::( + tx, + "SELECT COUNT(*) AS n FROM workflow_worker_registration_incarnations", + ) + .await?; + if copied != total as u64 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_incarnation_unqualified", + )); + } + } + // A PHP 2.8.0+ worker keeps the token it was given; an older one gets a + // new token. + let missing = query::(if php_incarnations { + "SELECT r.namespace,r.worker_id,r.registration_token AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + } else { + "SELECT r.namespace,r.worker_id,NULL AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + }) + .fetch_all(&mut *tx) + .await?; + for row in &missing { + let token = match DB::optional_string(row, "token")? { + Some(token) => token, + None => super::super::worker_incarnations::token().await?, + }; + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") + .bind(token) + .bind(DB::string(row, "namespace")?) + .bind(DB::string(row, "worker_id")?) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .execute(&mut *tx) + .await?; + } + Ok((copied, missing.len())) +} + +async fn count(tx: &mut DB::Connection, sql: &'static str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, +{ + DB::number(&query::(sql).fetch_one(&mut *tx).await?, "n") +} diff --git a/rust/src/runtime/upgrade/takeover/mysql_journal.rs b/rust/src/runtime/upgrade/takeover/mysql_journal.rs new file mode 100644 index 000000000..15b3d8ce5 --- /dev/null +++ b/rust/src/runtime/upgrade/takeover/mysql_journal.rs @@ -0,0 +1,397 @@ +//! MySQL and MariaDB takeover. Their DDL commits implicitly, and `LOCK +//! TABLES` refuses to create or drop the tables it does not hold, so the +//! takeover cannot be one transaction. It writes native's marker first, with +//! a `rust-takeover:` row whose version counts the completed steps +//! and whose checksum holds the backup fingerprint's digest. From then on PHP +//! refuses the database and native refuses it until the last step. Every step +//! is idempotent, so a rerun with the same fingerprint resumes after the last +//! recorded one. +use super::{ + BUDGET, DELIVERY_REFERENCES, Result, adopt_incarnations, count, matches_backup, + php_incarnations, preconditions, qualified, report, service_calls_absent, +}; +use crate::runtime::{ + mysql::{self, Flavor, LEDGER_SQL, MARKER_SQL, TAKEOVER_ENGINE}, + upgrade::{ + bounded, fingerprint, mysql_inventory, mysql_source, queue_binding::QueueSourceConfig, + refuse, + }, +}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{Connection, Executor, MySql, MySqlConnection, mysql::MySqlConnectOptions}; +use tokio::time::Instant; + +/// Completed-step counts recorded in the marker row. +const MARKED: i64 = 0; +/// Sessions rechecked and the source fingerprinted again after the marker: +/// nothing PHP wrote can be missing from the backup. +const FENCED: i64 = 1; +const LAST: i64 = 8; + +/// One takeover run: what its steps need and what its report says. +struct Run<'a> { + backend: &'static str, + flavor: Flavor, + digest: Vec, + layout: String, + fingerprint: &'a str, + queues: &'a QueueSourceConfig<'a>, + inventory: Value, + converted: serde_json::Map, +} + +enum Journal { + /// No marker: a PHP database. + Absent, + /// The marker table without its row: a takeover killed between the two. + Empty, + /// A takeover in progress, with its layout, step count and digest. + Takeover(String, i64, Vec), + /// Native's own database, or one native is still initializing. + Owned, +} + +pub async fn take_over_mysql( + options: MySqlConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let digest = fingerprint_digest(backup_fingerprint)?; + let deadline = Instant::now() + BUDGET; + // TIMESTAMP values render in UTC for the fingerprint, and lock waits stay + // short; the client deadline bounds whole-table work. + let (connection, backend) = mysql_source(options, deadline, true, 0).await?; + bounded(connection, deadline, async |connection| { + // The same session lock as native bootstrap, so neither a native node + // nor a second takeover can run on this database meanwhile. + let locked: Option = sqlx::query_scalar( + "SELECT GET_LOCK(CONCAT('dw-native-bootstrap:',SHA1(DATABASE())),5)", + ) + .fetch_one(&mut *connection) + .await?; + if locked != Some(1) { + return Err(refuse( + StatusCode::SERVICE_UNAVAILABLE, + "native_initialization_busy", + )); + } + let mut run = Run { + backend, + flavor: Flavor::detect(connection).await?, + digest, + layout: String::new(), + fingerprint: backup_fingerprint, + queues, + inventory: Value::Null, + converted: serde_json::Map::new(), + }; + let completed = match journal(connection).await? { + Journal::Absent => { + mark(connection, &mut run).await?; + MARKED + } + Journal::Empty => { + // Only a takeover adds the marker to a PHP database. Its layout + // is read again with the marker set aside. + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let layout = mysql_inventory(&mut tx, backend, Some(queues), true, true).await; + tx.rollback().await?; + run.layout = qualified(&layout?)?.to_owned(); + record(connection, &run).await?; + MARKED + } + Journal::Takeover(layout, completed, digest) => { + if digest != run.digest { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + php_incarnations(&layout)?; + run.layout = layout; + run.converted + .insert("resumed_after_step".into(), json!(completed)); + completed + } + Journal::Owned => { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_already_owned")); + } + }; + for step in completed + 1..=LAST { + perform(connection, &mut run, step).await?; + #[cfg(test)] + super::interruption_point(&format!("after-step-{step}")); + } + finish(connection, &run).await?; + mysql::verify(connection, run.flavor).await?; + let digest = json!({"fingerprint": run.fingerprint}); + Ok(report( + run.backend, + &run.layout, + &digest, + &run.inventory, + Value::Object(run.converted), + )) + }) + .await +} + +/// `sha256:` as the 32 bytes the marker row keeps. +fn fingerprint_digest(fingerprint: &str) -> Result> { + fingerprint["sha256:".len()..] + .as_bytes() + .chunks(2) + .map(|pair| { + std::str::from_utf8(pair) + .ok() + .and_then(|pair| u8::from_str_radix(pair, 16).ok()) + }) + .collect::>>() + .ok_or_else(|| { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_backup_fingerprint_required", + ) + }) +} + +async fn journal(connection: &mut MySqlConnection) -> Result { + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(TABLE_NAME)='dw_server_schema'") + .fetch_one(&mut *connection) + .await?; + if marked == 0 { + return Ok(Journal::Absent); + } + let mut rows: Vec<(String, i64, Vec)> = + sqlx::query_as("SELECT engine,version,bootstrap_checksum FROM dw_server_schema") + .fetch_all(&mut *connection) + .await?; + Ok(match rows.pop() { + None => Journal::Empty, + Some((engine, completed, digest)) if rows.is_empty() => { + match engine.strip_prefix(TAKEOVER_ENGINE) { + Some(layout) => Journal::Takeover(layout.to_owned(), completed, digest), + None => Journal::Owned, + } + } + Some(_) => Journal::Owned, + }) +} + +/// No other session may use the database. That needs the PROCESS privilege, +/// since without it MySQL shows only the account's own sessions. +async fn sessions_clear(connection: &mut MySqlConnection) -> Result<()> { + let visible: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM information_schema.USER_PRIVILEGES WHERE PRIVILEGE_TYPE='PROCESS')", + ) + .fetch_one(&mut *connection) + .await?; + if !visible { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sessions_unverifiable", + )); + } + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut *connection) + .await?; + if others != 0 { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_in_use")); + } + Ok(()) +} + +/// The inventory and fingerprint from one consistent snapshot, refused +/// unless the source is qualified and matches the backup. +async fn verified_snapshot( + connection: &mut MySqlConnection, + run: &Run<'_>, + fenced: bool, +) -> Result<(Value, String)> { + sessions_clear(connection).await?; + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let read = async { + let inventory = + mysql_inventory(&mut tx, run.backend, Some(run.queues), true, fenced).await?; + let digest = fingerprint::mysql_digest(&mut tx, run.backend, fenced).await?; + Ok::<_, crate::runtime::RuntimeError>((inventory, digest)) + } + .await; + tx.rollback().await?; + let (inventory, digest) = read?; + let layout = qualified(&inventory)?.to_owned(); + matches_backup(&digest, run.fingerprint)?; + php_incarnations(&layout)?; + service_calls_absent::(connection).await?; + Ok((inventory, layout)) +} + +/// The first write: native's marker with the takeover row, once the source +/// is verified. PHP refuses the database from here on. +async fn mark(connection: &mut MySqlConnection, run: &mut Run<'_>) -> Result<()> { + let (inventory, layout) = verified_snapshot(connection, run, false).await?; + run.inventory = inventory; + run.layout = layout; + connection.execute(MARKER_SQL).await?; + #[cfg(test)] + super::interruption_point("before-marker-row"); + record(connection, run).await +} + +async fn record(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + sqlx::query("INSERT INTO dw_server_schema(engine,version,bootstrap_checksum) VALUES (?,?,?)") + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(MARKED) + .bind(&run.digest[..]) + .execute(&mut *connection) + .await?; + Ok(()) +} + +/// Records one more completed step, in the caller's transaction or alone. +async fn advance<'c, E>(executor: E, run: &Run<'_>, step: i64) -> Result<()> +where + E: Executor<'c, Database = MySql>, +{ + let advanced = sqlx::query("UPDATE dw_server_schema SET version=? WHERE engine=? AND version=?") + .bind(step) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(step - 1) + .execute(executor) + .await? + .rows_affected(); + if advanced != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + Ok(()) +} + +async fn column_exists(connection: &mut MySqlConnection, table: &str, column: &str) -> Result { + let found: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND COLUMN_NAME=?") + .bind(table) + .bind(column) + .fetch_one(&mut *connection) + .await?; + Ok(found != 0) +} + +/// Each step checks what it has to do, so repeating an interrupted one is +/// harmless, and then records itself. +async fn perform(connection: &mut MySqlConnection, run: &mut Run<'_>, step: i64) -> Result<()> { + let php_incarnations = php_incarnations(&run.layout)?; + match step { + FENCED => { + // A session that was still open when the marker appeared could + // have written; none is left, and the source is still the backup. + let (inventory, layout) = verified_snapshot(connection, run, true).await?; + if layout != run.layout { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + run.inventory = inventory; + } + 2 => { + sqlx::raw_sql(sqlx::AssertSqlSafe(run.flavor.native_relations_sql()?)) + .execute(&mut *connection) + .await?; + let indexed: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='workflow_tasks' AND INDEX_NAME='dw_workflow_tasks_poll'") + .fetch_one(&mut *connection) + .await?; + if indexed == 0 { + connection.execute("ALTER TABLE workflow_tasks ADD KEY dw_workflow_tasks_poll (namespace(64),queue(64),task_type(16),status(16),available_at)").await?; + } + } + 3 => { + // Rows, unlike DDL, commit with the step that records them. + let mut tx = connection.begin().await?; + let (copied, created) = adopt_incarnations::(&mut tx, php_incarnations).await?; + advance(&mut *tx, run, step).await?; + tx.commit().await?; + run.converted + .insert("incarnations_copied".into(), json!(copied)); + run.converted + .insert("incarnations_created".into(), json!(created)); + return Ok(()); + } + 4 => { + if php_incarnations { + connection + .execute("DROP TABLE IF EXISTS workflow_worker_registration_incarnations") + .await?; + } + } + 5 => { + if column_exists(connection, "workflow_worker_registrations", "registration_token").await? { + connection + .execute("ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token") + .await?; + } + } + 6 => { + if column_exists(connection, "workflow_service_calls", "call_deadline_at").await? { + // Dropping only the column would leave its index on status alone. + connection.execute("ALTER TABLE workflow_service_calls DROP INDEX wf_service_calls_deadline_idx, DROP COLUMN call_deadline_at").await?; + } + } + 7 => { + if column_exists(connection, "workflow_tasks", "queue_delivery_reference").await? { + let references = count::(connection, DELIVERY_REFERENCES).await?; + run.converted + .insert("queue_delivery_references_removed".into(), json!(references)); + connection + .execute("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .await?; + } + } + 8 => { + if run.layout.ends_with("+mariadb-driver") { + // Laravel's mariadb connection made the backup hold owner a uuid. + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner char(36) DEFAULT NULL") + .await?; + } + } + _ => {} + } + advance(&mut *connection, run, step).await +} + +/// The ledger row and native's final marker commit together. +async fn finish(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + connection.execute(LEDGER_SQL).await?; + let migration = run.flavor.migration(); + let mut tx = connection.begin().await?; + sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES (?,?,TRUE,?,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) + .await?; + let marked = sqlx::query("UPDATE dw_server_schema SET engine='rust-development',version=?,bootstrap_checksum=? WHERE engine=? AND version=?") + .bind(mysql::VERSION) + .bind(run.flavor.checksum()) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(LAST) + .execute(&mut *tx) + .await? + .rows_affected(); + if marked != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + tx.commit().await?; + Ok(()) +} diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 08ea8309d..045caea82 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -406,7 +406,11 @@ async fn payload_scan_validates_inline_pages_and_reports_late_bad_values_without assert_eq!(report["payload_inspection"]["other_root_payload_columns_verified"], false); assert_eq!( report["payload_inspection"]["unscanned_payload_codec_tables"], - json!(["workflow_commands", "workflow_signal_records", "workflow_updates"]) + json!([ + "workflow_durable_stream_items", + "workflow_inbound_stream_items", + "workflow_update_validation_tasks" + ]) ); assert_eq!(report["payload_inspection"]["diagnostics"], json!([])); for (table, column, count) in [ @@ -725,3 +729,439 @@ async fn read_only_source_file_can_be_inspected_without_write_permissions() { ); assert_eq!(std::fs::read(&path).unwrap(), before); } + +#[tokio::test] +async fn fingerprint_matches_a_byte_copy_and_changes_with_any_row() { + let (directory, path) = php_layout().await; + let before = tokio::fs::read(&path).await.unwrap(); + let first = fingerprint::fingerprint_sqlite(&path).await.unwrap(); + assert_eq!(first["status"], "fingerprinted"); + assert_eq!(first["table_rows"]["jobs"], 1); + for flag in [ + "source_mutated", + "takeover_allowed", + "backup_verified", + "php_quiescence_verified", + "stored_representation_compatibility_verified", + ] { + assert_eq!(first[flag], false, "{flag}"); + } + assert!(!first.to_string().contains("private-payload"), "{first}"); + assert_eq!(tokio::fs::read(&path).await.unwrap(), before); + assert_eq!( + fingerprint::fingerprint_sqlite(&path).await.unwrap()["fingerprint"], + first["fingerprint"] + ); + let copy = directory.path().join("copy.sqlite"); + tokio::fs::copy(&path, ©).await.unwrap(); + let copy = copy.to_str().unwrap(); + assert_eq!( + fingerprint::fingerprint_sqlite(copy).await.unwrap()["fingerprint"], + first["fingerprint"] + ); + let mut seen = vec![first["fingerprint"].clone()]; + for statement in [ + "UPDATE jobs SET attempts=1", + "UPDATE jobs SET reserved_at=7", + "UPDATE jobs SET reserved_at=0", + "UPDATE jobs SET payload='private-payload-λ '", + "INSERT INTO migrations(migration,batch) VALUES ('extra',1)", + ] { + mutate(copy, statement).await; + let changed = fingerprint::fingerprint_sqlite(copy).await.unwrap()["fingerprint"].clone(); + assert!(!seen.contains(&changed), "{statement}"); + seen.push(changed); + } + // The digest describes contents, so restoring earlier values restores it. + mutate( + copy, + "UPDATE jobs SET attempts=0,reserved_at=NULL,payload='private-payload-λ'; DELETE FROM migrations", + ) + .await; + assert_eq!( + fingerprint::fingerprint_sqlite(copy).await.unwrap()["fingerprint"], + first["fingerprint"] + ); +} + +#[tokio::test] +async fn fingerprint_ignores_physical_row_order_but_not_storage_class() { + let (_first_directory, first) = php_layout().await; + let (_second_directory, second) = php_layout().await; + mutate(&first, "INSERT INTO migrations(id,migration,batch) VALUES (5,'a',1),(3,'b',1)").await; + mutate(&second, "INSERT INTO migrations(id,migration,batch) VALUES (3,'b',1)").await; + mutate(&second, "INSERT INTO migrations(id,migration,batch) VALUES (5,'a',1)").await; + let one = fingerprint::fingerprint_sqlite(&first).await.unwrap(); + assert_eq!( + fingerprint::fingerprint_sqlite(&second).await.unwrap()["fingerprint"], + one["fingerprint"] + ); + // A text value is not the same row as an equal-looking blob. + mutate(&second, "UPDATE migrations SET migration=CAST('a' AS BLOB) WHERE id=5").await; + assert_ne!( + fingerprint::fingerprint_sqlite(&second).await.unwrap()["fingerprint"], + one["fingerprint"] + ); +} + +#[tokio::test] +async fn fingerprint_refuses_an_owned_or_unknown_source() { + let (_directory, path) = php_layout().await; + mutate(&path, "CREATE TABLE dw_server_schema(engine TEXT)").await; + reason( + fingerprint::fingerprint_sqlite(&path).await, + "upgrade_source_already_owned", + ); + let (_directory, path) = php_layout().await; + mutate(&path, "ALTER TABLE jobs ADD COLUMN unexpected TEXT").await; + reason( + fingerprint::fingerprint_sqlite(&path).await, + "upgrade_source_catalog_unqualified", + ); +} + +#[test] +fn source_layout_names_the_release_each_recorded_delta_reaches() { + let rows = |rows: &[(&str, &str)]| -> Vec<(String, String)> { + rows.iter() + .map(|(name, ddl)| ((*name).to_owned(), (*ddl).to_owned())) + .collect() + }; + let frozen = rows(&[("a", "1"), ("b", "2")]); + let deltas = [ + r#"{"remove":[["a","1"]],"add":[["a","1 changed"]]}"#, + r#"{"remove":[],"add":[["c","3"]]}"#, + ]; + let layout = |actual: &[(&str, &str)]| { + source_layout(&rows(actual), frozen.clone(), deltas, |row| row.0.clone()) + .map(|layout| Value::from(layout)) + }; + assert_eq!(layout(&[("b", "2"), ("a", "1")]).unwrap(), "frozen"); + assert_eq!(layout(&[("a", "1 changed"), ("b", "2")]).unwrap(), "php-2.8.0"); + assert_eq!( + layout(&[("c", "3"), ("a", "1 changed"), ("b", "2")]).unwrap(), + "php-2.9.7" + ); + // A later release's addition without the earlier change is no recorded layout. + reason( + layout(&[("a", "1"), ("b", "2"), ("c", "3")]), + "upgrade_source_catalog_unqualified", + ); + reason(layout(&[("a", "1")]), "upgrade_source_catalog_unqualified"); +} + +#[tokio::test] +async fn frozen_layout_is_reported_by_inspection_and_fingerprint() { + let (_directory, path) = php_layout().await; + assert_eq!(inspect_sqlite(&path).await.unwrap()["source_layout"], "frozen"); + assert_eq!( + fingerprint::fingerprint_sqlite(&path).await.unwrap()["source_layout"], + "frozen" + ); +} + +/// The frozen test layout moved to PHP 2.9.7's, by the same column, table and +/// index changes PHP's migrations make. Inspection must call it php-2.9.7. +async fn php297_layout() -> (tempfile::TempDir, String) { + let (directory, path) = php_layout().await; + for statement in [ + // The frozen test layout's private job is not a recognized delivery. + "DELETE FROM jobs", + r#"ALTER TABLE "workflow_worker_registrations" ADD COLUMN "registration_token" varchar"#, + r#"CREATE TABLE "workflow_worker_registration_incarnations" ("token" varchar not null, "namespace" varchar not null, "worker_id" varchar not null, "status" varchar not null, "recovered_workflow_task_count" integer not null default '0', "finished_at" datetime, "created_at" datetime, "updated_at" datetime, primary key ("token"))"#, + r#"CREATE INDEX "workflow_worker_registration_incarnations_finished_at_index" on "workflow_worker_registration_incarnations" ("finished_at")"#, + r#"CREATE INDEX "dw_worker_incarnation_scope_idx" on "workflow_worker_registration_incarnations" ("namespace", "worker_id")"#, + r#"ALTER TABLE "workflow_service_calls" ADD COLUMN "call_deadline_at" datetime"#, + r#"CREATE INDEX "wf_service_calls_deadline_idx" on "workflow_service_calls" ("status", "call_deadline_at")"#, + r#"ALTER TABLE "workflow_tasks" ADD COLUMN "queue_delivery_reference" text"#, + ] { + mutate(&path, statement).await; + } + (directory, path) +} + +async fn backup_fingerprint(path: &str) -> String { + fingerprint::fingerprint_sqlite(path).await.unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() +} + +#[tokio::test] +async fn takeover_converts_a_php_297_database_that_native_then_opens() { + let (_directory, path) = php297_layout().await; + assert_eq!(inspect_sqlite(&path).await.unwrap()["source_layout"], "php-2.9.7"); + for statement in [ + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00')", + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00')", + "INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00.000000','2026-10-20 09:00:00.000000')", + "INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00.000000','2026-10-20 07:00:00.000000','2026-10-20 08:00:00.000000')", + "INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00.000000','2026-10-20 07:00:00.000000','2026-10-20 08:30:00.000000')", + ] { + mutate(&path, statement).await; + } + let fingerprint = backup_fingerprint(&path).await; + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}")).await.unwrap(); + let incarnations: Vec<(String, String, String, Option, Option)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + // The current worker keeps its PHP token; the retired ones keep their state. + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + None + ))); + assert!(incarnations.iter().any(|row| row.1 == "gone" && row.2 == "deregistered" && row.3 == Some(3))); + assert!(incarnations.iter().any(|row| row.1 == "current" && row.2 == "superseded" && row.3.is_none() && row.4.is_some())); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + connection.close().await.unwrap(); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}")).await.unwrap(); + let marker: (String, i64) = sqlx::query_as("SELECT engine,version FROM dw_server_schema") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(marker, ("rust-development".to_owned(), 4)); + connection.close().await.unwrap(); +} + +#[tokio::test] +#[ignore = "only invoked as a real subprocess by the takeover interruption test"] +async fn sqlite_takeover_child() { + let path = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await; + panic!("takeover returned instead of parking: {report:?}"); +} + +#[cfg(unix)] +#[tokio::test] +async fn takeover_killed_before_commit_leaves_php_data_and_can_be_rerun() { + use std::os::unix::process::ExitStatusExt; + let (directory, path) = php297_layout().await; + let fingerprint = backup_fingerprint(&path).await; + let original = tokio::fs::read(&path).await.unwrap(); + let receipt = directory.path().join("boundary"); + // The child process is supervised on a blocking thread. + let (child_path, child_fingerprint) = (path.clone(), fingerprint.clone()); + let status = tokio::task::spawn_blocking(move || { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::upgrade::tests::sqlite_takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &child_path) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &child_fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && std::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + std::thread::sleep(Duration::from_millis(10)); + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + child.kill().unwrap(); + child.wait().unwrap() + }) + .await + .unwrap(); + assert_eq!(status.signal(), Some(9)); + // Either SQLite had not written the file yet, or its rollback journal + // still holds the original pages. Read-only inspection never recovers it. + if Path::new(&format!("{path}-journal")).exists() { + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + } else { + assert_eq!(tokio::fs::read(&path).await.unwrap(), original); + } + // Rerunning with the same verified fingerprint rolls the journal back, + // finds the backed-up PHP database and takes it over. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["fingerprint"], fingerprint.as_str()); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; +} + +#[tokio::test] +async fn takeover_of_the_frozen_layout_adds_only_native_relations() { + let (_directory, path) = php_layout().await; + mutate(&path, "DELETE FROM jobs").await; + let fingerprint = backup_fingerprint(&path).await; + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; +} + +#[tokio::test] +async fn takeover_refusals_leave_the_php_database_unchanged() { + let (_directory, path) = php297_layout().await; + let fingerprint = backup_fingerprint(&path).await; + let original = std::fs::read(&path).unwrap(); + reason( + takeover::take_over_sqlite(&path, &format!("sha256:{}", "0".repeat(64)), &queue_config()).await, + "upgrade_backup_fingerprint_mismatch", + ); + reason( + takeover::take_over_sqlite(&path, "not-a-fingerprint", &queue_config()).await, + "upgrade_backup_fingerprint_required", + ); + assert_eq!(std::fs::read(&path).unwrap(), original); + // A write after the backup changes the digest, so the old one is refused. + mutate(&path, "INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)").await; + reason( + takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await, + "upgrade_backup_fingerprint_mismatch", + ); + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + // Features native lacks: their rows would be left behind unread. + ( + "INSERT INTO runtime_credentials(id,subject,roles,tenant,token_prefix,token_hash) VALUES ('c','worker','[]','default','dw_','h')", + "upgrade_source_runtime_credentials_unsupported", + ), + ( + "INSERT INTO workflow_messages(workflow_instance_id,direction,channel,stream_key,sequence) VALUES ('w','inbound','c','k',1)", + "upgrade_source_message_streams_unsupported", + ), + ] { + let (_directory, path) = php297_layout().await; + mutate(&path, statement).await; + let fingerprint = backup_fingerprint(&path).await; + let before = std::fs::read(&path).unwrap(); + reason( + takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await, + expected, + ); + assert_eq!(std::fs::read(&path).unwrap(), before, "{expected}"); + } +} + +/// The frozen layout in PHP Server's default WAL mode, copied while its +/// writer was still open: one committed row is only in the WAL, as a crashed +/// PHP would leave it. +async fn wal_source() -> (tempfile::TempDir, String) { + let (directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer.execute("PRAGMA wal_autocheckpoint=0").await.unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('only-in-wal',7)") + .await + .unwrap(); + let copy = directory.path().join("wal.sqlite").to_str().unwrap().to_owned(); + for suffix in ["", "-wal", "-shm"] { + tokio::fs::copy(format!("{path}{suffix}"), format!("{copy}{suffix}")) + .await + .unwrap(); + } + writer.close().await.unwrap(); + (directory, copy) +} + +#[tokio::test] +async fn prepare_switches_a_wal_source_to_the_rollback_journal_and_keeps_its_rows() { + let (_directory, path) = wal_source().await; + assert!(Path::new(&format!("{path}-wal")).exists()); + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["status"], "rollback_journal", "{report}"); + assert_eq!(report["previous_journal_mode"], "wal"); + assert_eq!(report["source_mutated"], true); + for suffix in ["-wal", "-shm", "-journal"] { + assert!(!Path::new(&format!("{path}{suffix}")).exists(), "{suffix}"); + } + assert_eq!(journal_format(&path).await.unwrap(), ROLLBACK_JOURNAL); + let inventory = inspect_sqlite(&path).await.unwrap(); + assert_eq!(inventory["table_rows"]["migrations"], 1, "{inventory}"); + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}?mode=ro")) + .await + .unwrap(); + let batch: i64 = sqlx::query_scalar("SELECT batch FROM migrations WHERE migration='only-in-wal'") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(batch, 7); + connection.close().await.unwrap(); + // Preparing again changes nothing. + let original = std::fs::read(&path).unwrap(); + assert_eq!(prepare_sqlite_source(&path).await.unwrap()["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} + +#[tokio::test] +async fn prepare_refuses_while_another_connection_holds_the_source() { + let (_directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('open',8)") + .await + .unwrap(); + reason(prepare_sqlite_source(&path).await, "upgrade_source_in_use"); + assert_eq!(journal_format(&path).await.unwrap(), WRITE_AHEAD_LOG); + writer.close().await.unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "wal", "{report}"); +} + +#[tokio::test] +async fn prepare_leaves_a_rollback_journal_source_byte_identical() { + let (_directory, path) = php_layout().await; + let original = std::fs::read(&path).unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "delete", "{report}"); + assert_eq!(report["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 0c82502d0..166d9572f 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -2378,7 +2378,8 @@ async fn queries_fence_recovered_leases_across_pools_without_mutating_the_run() let claim = await_query(&b, "query-a").await; assert_eq!(claim["run_status"], "completed"); // Worker snapshots retain durable row IDs/links; the control-plane - // history endpoint exposes its documented timestamp projection. + // history endpoint exposes its documented timestamp projection and the + // command principal, which these unauthenticated commands do not record. let projected = Value::Array( claim["history_events"] .as_array() @@ -2386,7 +2387,7 @@ async fn queries_fence_recovered_leases_across_pools_without_mutating_the_run() .iter() .map(|event| { json!({"sequence":event["sequence"],"event_type":event["event_type"], - "payload":event["payload"],"timestamp":event["recorded_at"]}) + "payload":event["payload"],"timestamp":event["recorded_at"],"principal":null}) }) .collect(), ); @@ -4681,13 +4682,14 @@ async fn terminal_activity_failures_resume_once_and_preserve_original_failure() .collect(); assert_eq!(event_ids.len(), worker_events.len()); // Worker history retains event IDs/attribution and recorded_at; - // the control API explicitly returns these four common fields. + // the control API returns these common fields plus the command + // principal, which these unauthenticated commands do not record. let control_events: Vec<_> = worker_events .iter() .map(|event| { json!({ "sequence":event["sequence"],"event_type":event["event_type"], - "timestamp":event["recorded_at"],"payload":event["payload"] + "timestamp":event["recorded_at"],"payload":event["payload"],"principal":null }) }) .collect(); diff --git a/rust/upgrade/mariadb-driver.json b/rust/upgrade/mariadb-driver.json new file mode 100644 index 000000000..4f4a74900 --- /dev/null +++ b/rust/upgrade/mariadb-driver.json @@ -0,0 +1,14 @@ +{ + "remove": [ + [ + "runtime_external_payload_backup_hold", + "CREATE TABLE `runtime_external_payload_backup_hold` (\n `id` tinyint(3) unsigned NOT NULL,\n `owner` char(36) DEFAULT NULL,\n `acquired_at` timestamp NULL DEFAULT NULL,\n `expires_at` timestamp NULL DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ], + "add": [ + [ + "runtime_external_payload_backup_hold", + "CREATE TABLE `runtime_external_payload_backup_hold` (\n `id` tinyint(3) unsigned NOT NULL,\n `owner` uuid DEFAULT NULL,\n `acquired_at` timestamp NULL DEFAULT NULL,\n `expires_at` timestamp NULL DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/README.md b/rust/upgrade/php-2.9.7/README.md new file mode 100644 index 000000000..8874b00d2 --- /dev/null +++ b/rust/upgrade/php-2.9.7/README.md @@ -0,0 +1,29 @@ +# Exact PHP source-catalog variant: Server 2.9.7 + +These deltas apply to the [PHP 2.8.0 layout](../php-2.8.0/README.md), not to +the frozen layout. They come from normal bootstrap of published Server 2.9.7, +tag `2.9.7`, image index +`sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50` +(amd64 `sha256:40e969d0b6b240b835dcdb2fc531507266216ec34a83f59f561674f41e35f461`), +which pins Core (Workflow package) 2.7.2. + +The only physical change from 2.8.0 is Core's nullable JSON column +`workflow_tasks.queue_delivery_reference`, added by its +`2026_10_16_000100_add_queue_delivery_reference_to_workflow_tasks` migration. +PHP records there the database queue job that delivers a task. Server's own +migrations are unchanged from 2.8.0. + +Definitions were read from actual published-image databases: SQLite, MySQL +8.0, MariaDB 10.11 through the `mysql` connection, and PostgreSQL 17. MySQL +8.4 was checked against the same definitions. No application row values or +credentials are recorded. + +[`../mariadb-driver.json`](../mariadb-driver.json) records the one difference +when PHP uses Laravel's separate `mariadb` connection instead of `mysql`: +`runtime_external_payload_backup_hold.owner` is `uuid` rather than `char(36)`. +Inspection applies it to every MariaDB release layout and reports, for +example, `php-2.9.7+mariadb-driver`. + +Like the 2.8.0 variant, this recognizes physical layout for read-only +inventory only. It neither identifies the installed package nor qualifies +stored representations, backup, writer quiescence or takeover of that layout. diff --git a/rust/upgrade/php-2.9.7/mariadb.json b/rust/upgrade/php-2.9.7/mariadb.json new file mode 100644 index 000000000..de626b086 --- /dev/null +++ b/rust/upgrade/php-2.9.7/mariadb.json @@ -0,0 +1,14 @@ +{ + "remove": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) NOT NULL,\n `workflow_run_id` varchar(26) NOT NULL,\n `namespace` varchar(255) DEFAULT NULL,\n `task_type` varchar(255) NOT NULL,\n `status` varchar(255) NOT NULL,\n `compatibility` varchar(255) DEFAULT NULL,\n `payload` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`payload`)),\n `connection` varchar(255) DEFAULT NULL,\n `queue` varchar(255) DEFAULT NULL,\n `priority` tinyint(3) unsigned NOT NULL DEFAULT 5,\n `fairness_key` varchar(64) DEFAULT NULL,\n `fairness_weight` smallint(5) unsigned NOT NULL DEFAULT 1,\n `sticky_worker_id` varchar(255) DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int(10) unsigned NOT NULL DEFAULT 0,\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text DEFAULT NULL,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text DEFAULT NULL,\n `repair_count` int(10) unsigned NOT NULL DEFAULT 0,\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text DEFAULT NULL,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ], + "add": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) NOT NULL,\n `workflow_run_id` varchar(26) NOT NULL,\n `namespace` varchar(255) DEFAULT NULL,\n `task_type` varchar(255) NOT NULL,\n `status` varchar(255) NOT NULL,\n `compatibility` varchar(255) DEFAULT NULL,\n `payload` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`payload`)),\n `connection` varchar(255) DEFAULT NULL,\n `queue` varchar(255) DEFAULT NULL,\n `priority` tinyint(3) unsigned NOT NULL DEFAULT 5,\n `fairness_key` varchar(64) DEFAULT NULL,\n `fairness_weight` smallint(5) unsigned NOT NULL DEFAULT 1,\n `sticky_worker_id` varchar(255) DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int(10) unsigned NOT NULL DEFAULT 0,\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text DEFAULT NULL,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text DEFAULT NULL,\n `repair_count` int(10) unsigned NOT NULL DEFAULT 0,\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text DEFAULT NULL,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n `queue_delivery_reference` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`queue_delivery_reference`)),\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/mysql.json b/rust/upgrade/php-2.9.7/mysql.json new file mode 100644 index 000000000..dfd764add --- /dev/null +++ b/rust/upgrade/php-2.9.7/mysql.json @@ -0,0 +1,14 @@ +{ + "remove": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `workflow_run_id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `namespace` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `task_type` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `status` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `compatibility` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `payload` json DEFAULT NULL,\n `connection` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `queue` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `priority` tinyint unsigned NOT NULL DEFAULT '5',\n `fairness_key` varchar(64) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `fairness_weight` smallint unsigned NOT NULL DEFAULT '1',\n `sticky_worker_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int unsigned NOT NULL DEFAULT '0',\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text COLLATE utf8mb4_unicode_ci,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text COLLATE utf8mb4_unicode_ci,\n `repair_count` int unsigned NOT NULL DEFAULT '0',\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text COLLATE utf8mb4_unicode_ci,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ], + "add": [ + [ + "workflow_tasks", + "CREATE TABLE `workflow_tasks` (\n `id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `workflow_run_id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `namespace` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `task_type` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `status` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `compatibility` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `payload` json DEFAULT NULL,\n `connection` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `queue` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `priority` tinyint unsigned NOT NULL DEFAULT '5',\n `fairness_key` varchar(64) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `fairness_weight` smallint unsigned NOT NULL DEFAULT '1',\n `sticky_worker_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_until` timestamp(6) NULL DEFAULT NULL,\n `sticky_replay_mode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `sticky_claimed_at` timestamp(6) NULL DEFAULT NULL,\n `available_at` timestamp(6) NULL DEFAULT NULL,\n `leased_at` timestamp(6) NULL DEFAULT NULL,\n `lease_owner` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,\n `lease_expires_at` timestamp(6) NULL DEFAULT NULL,\n `attempt_count` int unsigned NOT NULL DEFAULT '0',\n `last_dispatch_attempt_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatched_at` timestamp(6) NULL DEFAULT NULL,\n `last_dispatch_error` text COLLATE utf8mb4_unicode_ci,\n `last_claim_failed_at` timestamp(6) NULL DEFAULT NULL,\n `last_claim_error` text COLLATE utf8mb4_unicode_ci,\n `repair_count` int unsigned NOT NULL DEFAULT '0',\n `repair_available_at` timestamp(6) NULL DEFAULT NULL,\n `last_error` text COLLATE utf8mb4_unicode_ci,\n `created_at` timestamp(6) NULL DEFAULT NULL,\n `updated_at` timestamp(6) NULL DEFAULT NULL,\n `queue_delivery_reference` json DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `workflow_tasks_status_available_at_index` (`status`,`available_at`),\n KEY `workflow_tasks_workflow_run_id_index` (`workflow_run_id`),\n KEY `workflow_tasks_namespace_index` (`namespace`),\n KEY `workflow_tasks_sticky_worker_id_index` (`sticky_worker_id`),\n KEY `workflow_tasks_sticky_until_index` (`sticky_until`),\n KEY `workflow_tasks_sticky_replay_mode_index` (`sticky_replay_mode`),\n KEY `workflow_tasks_sticky_claimed_at_index` (`sticky_claimed_at`),\n KEY `workflow_tasks_dispatch_order_index` (`queue`,`status`,`priority`,`available_at`),\n KEY `workflow_tasks_namespace_queue_status_idx` (`namespace`,`queue`,`status`),\n KEY `workflow_tasks_fairness_class_index` (`queue`,`status`,`fairness_key`),\n KEY `workflow_tasks_prom_metrics_queue_idx` (`namespace`,`queue`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/postgres.json b/rust/upgrade/php-2.9.7/postgres.json new file mode 100644 index 000000000..ddfd2bbb6 --- /dev/null +++ b/rust/upgrade/php-2.9.7/postgres.json @@ -0,0 +1,18 @@ +{ + "remove": [], + "add": [ + [ + "column", + "workflow_tasks.32", + { + "name": "queue_delivery_reference", + "type": "json", + "default": null, + "identity": "", + "not_null": false, + "collation": null, + "generated": "" + } + ] + ] +} diff --git a/rust/upgrade/php-2.9.7/sqlite.json b/rust/upgrade/php-2.9.7/sqlite.json new file mode 100644 index 000000000..525aa789b --- /dev/null +++ b/rust/upgrade/php-2.9.7/sqlite.json @@ -0,0 +1,18 @@ +{ + "remove": [ + [ + "table", + "workflow_tasks", + "workflow_tasks", + "CREATE TABLE \"workflow_tasks\" (\"id\" varchar not null, \"workflow_run_id\" varchar not null, \"namespace\" varchar, \"task_type\" varchar not null, \"status\" varchar not null, \"compatibility\" varchar, \"payload\" text, \"connection\" varchar, \"queue\" varchar, \"sticky_worker_id\" varchar, \"sticky_until\" datetime, \"sticky_replay_mode\" varchar, \"sticky_claimed_at\" datetime, \"available_at\" datetime, \"leased_at\" datetime, \"lease_owner\" varchar, \"lease_expires_at\" datetime, \"attempt_count\" integer not null default '0', \"last_dispatch_attempt_at\" datetime, \"last_dispatched_at\" datetime, \"last_dispatch_error\" text, \"last_claim_failed_at\" datetime, \"last_claim_error\" text, \"repair_count\" integer not null default '0', \"repair_available_at\" datetime, \"last_error\" text, \"created_at\" datetime, \"updated_at\" datetime, \"priority\" integer not null default '5', \"fairness_key\" varchar, \"fairness_weight\" integer not null default '1', primary key (\"id\"))" + ] + ], + "add": [ + [ + "table", + "workflow_tasks", + "workflow_tasks", + "CREATE TABLE \"workflow_tasks\" (\"id\" varchar not null, \"workflow_run_id\" varchar not null, \"namespace\" varchar, \"task_type\" varchar not null, \"status\" varchar not null, \"compatibility\" varchar, \"payload\" text, \"connection\" varchar, \"queue\" varchar, \"sticky_worker_id\" varchar, \"sticky_until\" datetime, \"sticky_replay_mode\" varchar, \"sticky_claimed_at\" datetime, \"available_at\" datetime, \"leased_at\" datetime, \"lease_owner\" varchar, \"lease_expires_at\" datetime, \"attempt_count\" integer not null default '0', \"last_dispatch_attempt_at\" datetime, \"last_dispatched_at\" datetime, \"last_dispatch_error\" text, \"last_claim_failed_at\" datetime, \"last_claim_error\" text, \"repair_count\" integer not null default '0', \"repair_available_at\" datetime, \"last_error\" text, \"created_at\" datetime, \"updated_at\" datetime, \"priority\" integer not null default '5', \"fairness_key\" varchar, \"fairness_weight\" integer not null default '1', \"queue_delivery_reference\" text, primary key (\"id\"))" + ] + ] +} diff --git a/scripts/conformance/server-parity/restart.php b/scripts/conformance/server-parity/restart.php index a72f94979..5ded7dc5a 100644 --- a/scripts/conformance/server-parity/restart.php +++ b/scripts/conformance/server-parity/restart.php @@ -53,6 +53,13 @@ function restartControlHistory(array $events): array 'event_type' => $event['event_type'], 'timestamp' => $event['recorded_at'], 'payload' => $event['payload']], $events); } +// Public history lifts each command's principal onto its event; worker task +// history has no principal. Compare the fields both views share. +function restartWithoutPrincipal(array $events): array +{ + return array_map(static fn (array $event): array => array_diff_key($event, ['principal' => true]), $events); +} + function restartSignalWorkflow(WorkflowContext $context, array $input): array { $context->waitCondition(fn (): bool => count($context->signals('payload')) > 0, 'restart.payload'); @@ -336,7 +343,7 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array $terminalClaim = $client->pollWorkflowTask('restart-terminal-new', 'restart-terminal-v1', 0); assertRestart(is_array($terminalClaim) && $terminalClaim['task_id'] === $terminal['failure_receipt']['next_task_id'] && $terminalClaim['run_id'] === $terminal['run_id'] - && sameRestartValue(restartControlHistory($terminalClaim['history_events']), $terminal['history']) + && sameRestartValue(restartControlHistory($terminalClaim['history_events']), restartWithoutPrincipal($terminal['history'])) && $terminalClaim['history_events'][4]['workflow_task_id'] === $terminalActivity['task_id'], 'Recovered terminal resumption lost original task/run/history.'); $terminalReplay = (new Replayer($client->payloadCodec()))->replay(restartTerminalWorkflow(...), $terminalClaim['history_events'], $client->payloadCodec()->decodeEnvelope($terminalClaim['arguments']), 'restart-terminal-v1', $terminalClaim); diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index a2524546d..fa85db118 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,13 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal', + 'parity.upgrade_lease'], + // The signal original declares its one signal, as an SDK worker would. + 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ + 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, + 'variadic' => false, 'allows_null' => false, 'default_available' => false]], + ]]]], 'supported_activity_types' => ['parity.upgrade_activity'], 'capabilities' => [], 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_protocol_author']), @@ -50,8 +56,12 @@ $value = [9007199254740993, null, 'upgradeλ']; $envelope = ['codec' => 'avro', 'blob' => Avro::serialize([$value])]; $poll = ['worker_id' => $worker, 'task_queue' => $prefix, 'timeout_seconds' => 0]; +// A takeover run resumes this timer under native, so it may choose a shorter delay. +$timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); +$check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer'] as $kind) { +// The lease original claims before the signal original makes its resume task. +foreach (['activity', 'timer', 'lease', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -60,24 +70,43 @@ 'started_at' => $started, 'original_deadline' => $deadline], JSON_THROW_ON_ERROR)."\n"); $task = $expect($request('POST', '/api/worker/workflow-tasks/poll', $poll), 200, $kind.' claim')['task']; $check($task['run_id'] === $run['run_id'], 'Original workflow claim changed'); - $command = $kind === 'activity' - ? ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, + $command = match ($kind) { + 'activity' => ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], - 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900] - : ['type' => 'start_timer', 'delay_seconds' => 600]; - $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ - 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], - 'commands' => [$command], - ]), 200, $kind.' command'); + 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], + 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], + 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + // The lease original's worker stops after its claim. PHP's workflow + // task lease expires after the takeover, and native redelivers it. + 'lease' => null, + }; + if ($command !== null) { + $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command], + ]), 200, $kind.' command'); + } $activity = null; if ($kind === 'activity') { $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; $check($activity['run_id'] === $run['run_id'] && $activity['attempt_number'] === 1, 'Original activity changed'); } + if ($kind === 'signal') { + // PHP accepts the signal and makes its resume task, which stays + // unclaimed until native resumes it after the takeover. + $signalled = $request('POST', '/api/workflows/'.$id.'/signal/go', ['input' => $envelope]); + $check(in_array($signalled[0], [200, 202], true), 'signal: '.json_encode(['status' => $signalled[0], + 'reason' => $signalled[1]['reason'] ?? null, 'errors' => $signalled[1]['errors'] ?? null])); + } $history = $expect($request('GET', '/api/workflows/'.$id.'/runs/'.$run['run_id'].'/history'), 200, 'history')['events']; - $check(array_column($history, 'event_type') === ($kind === 'activity' - ? ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'] - : ['StartAccepted', 'WorkflowStarted', 'TimerScheduled']), 'Original pending history differs'); + fwrite(STDERR, json_encode(['phase' => 'history', 'kind' => $kind, 'events' => array_column($history, 'event_type')], + JSON_THROW_ON_ERROR)."\n"); + $check(array_column($history, 'event_type') === match ($kind) { + 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], + 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], + 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + 'lease' => ['StartAccepted', 'WorkflowStarted'], + }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; } diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index 35b995418..bd2b67148 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -4,14 +4,18 @@ set -euo pipefail test "$#" = 1 || exit 2 backend=$1 test -n "${RESOURCE_SCOPE:?Set the owning isolated network}" -evidence=parity-evidence/upgrade-source -state=parity-upgrade-source-state +label=${DW_PARITY_UPGRADE_LABEL:-} +[[ $label =~ ^[a-z0-9]*$ ]] || exit 2 +evidence=parity-evidence/upgrade-source${label:+-$label} +state=parity-upgrade-source-state${label:+-$label} test -f "$evidence/owner.txt" || exit 0 test "$(cat "$evidence/owner.txt")" = "$RESOURCE_SCOPE" || exit 2 -resource="$RESOURCE_SCOPE-upgrade" +resource="$RESOURCE_SCOPE-upgrade${label:+-$label}" +source=upgrade_source_ref${label:+_$label} +restored=upgrade_restore_ref${label:+_$label} image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 -for role in author control inspect php; do +for role in author control inspect php prepare takeover native resume; do if docker inspect "$resource-$role" >/dev/null 2>&1; then docker stop --signal SIGTERM --time 5 "$resource-$role" >/dev/null docker rm "$resource-$role" >/dev/null @@ -23,17 +27,47 @@ case "$backend" in mysql) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ - -e 'DROP DATABASE IF EXISTS upgrade_source_ref' \ + -e "DROP DATABASE IF EXISTS $source" \ > "$evidence/database-cleanup.txt" 2>&1 ;; pgsql) docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ - -c 'DROP DATABASE IF EXISTS upgrade_source_ref' \ + -c "DROP DATABASE IF EXISTS $source" \ > "$evidence/database-cleanup.txt" 2>&1 ;; sqlite) ;; *) exit 2 ;; esac fi +if [[ -f "$evidence/restore-owned.txt" && "$(cat "$evidence/restore-owned.txt")" == "$RESOURCE_SCOPE" ]]; then +case "$backend" in + mysql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ + -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ + -e "DROP DATABASE IF EXISTS $restored" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + pgsql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS $restored" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + *) ;; +esac +fi +if [[ -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then +case "$backend" in + mysql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ + -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ + -e "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + pgsql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + *) ;; +esac +fi if test -f "$evidence/roles-owned.txt" && test "$(cat "$evidence/roles-owned.txt")" = "$RESOURCE_SCOPE"; then if test "$backend" = mysql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 2c7e58f5c..6d869e1f3 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# A separate published PHP cohort; inventory is not migration or SDK takeover. +# A separate published PHP cohort: inventory and refusals, a verified backup, +# and the takeover of a restored copy that native then resumes. set -euo pipefail test "$#" = 1 || exit 2 backend=$1 @@ -11,9 +12,15 @@ docker network inspect "$RESOURCE_SCOPE" >/dev/null image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 rust_image=mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 -resource="$RESOURCE_SCOPE-upgrade" -state=parity-upgrade-source-state -evidence=parity-evidence/upgrade-source +# A labeled run observes another release beside the default one. Its state, +# evidence, containers and databases are separate, and it names the layout +# inspection must report for that release. +label=${DW_PARITY_UPGRADE_LABEL:-} +[[ $label =~ ^[a-z0-9]*$ ]] || exit 2 +layout=${DW_PARITY_UPGRADE_EXPECTED_LAYOUT:-php-2.8.0} +resource="$RESOURCE_SCOPE-upgrade${label:+-$label}" +state=parity-upgrade-source-state${label:+-$label} +evidence=parity-evidence/upgrade-source${label:+-$label} test -f rust-target/debug/durable-workflow-server git diff --quiet git diff --cached --quiet @@ -26,7 +33,7 @@ mkdir -p "$state" "$evidence" chmod a+rwx "$state" "$evidence" printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/owner.txt" trap 'printf "%s\n" "$?" > "$evidence/profile-status.txt"' EXIT -database=upgrade_source_ref +database=upgrade_source_ref${label:+_$label} password=${DW_PARITY_UPGRADE_DB_PASSWORD:-parity-disposable-password} case "$backend" in mysql) host=${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}; admin=(-e DB_CONNECTION=mysql -e DB_HOST="$host" -e DB_USERNAME=root -e DB_PASSWORD="$password") ;; @@ -37,7 +44,7 @@ esac if test "$backend" = mysql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ -h "$host" -uroot -p"$password" \ - -e 'CREATE DATABASE upgrade_source_ref CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' > "$evidence/create.txt" 2>&1 + -e "CREATE DATABASE $database CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" > "$evidence/create.txt" 2>&1 elif test "$backend" = pgsql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ --entrypoint createdb "$POSTGRES_IMAGE" -h "$host" -U parity "$database" > "$evidence/create.txt" 2>&1 @@ -46,8 +53,11 @@ else chmod 0666 "$state/database.sqlite" fi printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" +# Every backend also takes over a copy and resumes the originals, so the +# timer is short. +timer_seconds=60 configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token - -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) + -e CACHE_STORE=file -e QUEUE_CONNECTION=database) php_run() { docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" "${configuration[@]}" -v "$PWD/$state:/state" -v "$PWD:/source:ro" \ @@ -65,20 +75,34 @@ for attempt in $(seq 1 30); do done test "$ready" = 1 docker run --rm --name "$resource-author" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ - --network "$RESOURCE_SCOPE" -e DW_PARITY_TOKEN=upgrade-fixture-token -v "$PWD:/source:ro" --entrypoint php "$image" \ + --network "$RESOURCE_SCOPE" -e DW_PARITY_TOKEN=upgrade-fixture-token -e DW_PARITY_UPGRADE_TIMER_SECONDS="$timer_seconds" -v "$PWD:/source:ro" --entrypoint php "$image" \ /source/scripts/conformance/server-parity/upgrade-source-author.php http://parity-upgrade-php:8080 upgrade-source \ > "$evidence/originals.json" 2> "$evidence/author.stderr" docker stop --signal SIGTERM --time 10 "$resource-php" >/dev/null docker inspect --format '{{json .State}}' "$resource-php" > "$evidence/stopped-state.json" jq -e '.Running==false and .ExitCode==0 and .OOMKilled==false' "$evidence/stopped-state.json" >/dev/null -if test "$backend" = sqlite; then +if [[ $backend == sqlite ]]; then + # PHP Server keeps SQLite in WAL mode. Once PHP is stopped, the source is + # switched to the rollback journal the upgrade commands need, as an operator + # does first; its rows are unchanged. + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 2 2" ]] + docker run --rm --name "$resource-prepare" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/database.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-prepare-sqlite \ + > "$evidence/prepare.json" 2> "$evidence/prepare.stderr" + jq -e '.status=="rollback_journal" and .previous_journal_mode=="wal" and .source_mutated==true' \ + "$evidence/prepare.json" >/dev/null + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 1 1" ]] +fi +if [[ $backend == sqlite ]]; then docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network=none -v "$PWD/$state:/state:ro" -v "$PWD:/source:ro" --entrypoint php "$image" \ /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==2 and (.tasks|length)==4 and (.activities|length)==1 - and (.history|length)==7 and (.queue_jobs|length)==1 + and (.runs|length)==4 and (.tasks|length)==7 and (.activities|length)==1 + and (.history|length)==13 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -100,7 +124,8 @@ inspect_source() { } snapshot() { if test "$backend" = sqlite; then - find "$state" -maxdepth 1 -type f -print0 | sort -z | xargs -0 -r sha256sum + # The source file and any sidecar it gains; the restored copy lives beside it. + find "$state" -maxdepth 1 -type f -name 'database.sqlite*' -print0 | sort -z | xargs -0 -r sha256sum else local helper=mysql if test "$backend" = pgsql; then helper=postgres; fi @@ -109,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e '.status=="inspected" and (.table_rows|length)==50 and .table_rows.workflow_runs==2 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -120,10 +145,10 @@ jq -e '.status=="inspected" and .queue_inspection.status=="known_subset_bound" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/queue-inspection.json" >/dev/null inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> "$evidence/payload-inspection.stderr" -jq -e '.status=="inspected" and (.payload_inspection.columns|length)==4 +jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==3 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==11 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -132,6 +157,144 @@ jq -e '.status=="inspected" and (.payload_inspection.columns|length)==4 and .payload_inspection.registry_binding_verified==false and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/payload-inspection.json" >/dev/null +fingerprint() { + local target=$1 + docker run --rm --name "$resource-inspect" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network "$RESOURCE_SCOPE" "${reader[@]}" -e DB_DATABASE="$target" -e DW_RUST_EXPERIMENTAL=1 \ + -v "$PWD/$state:/state:ro" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-fingerprint +} +fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" +jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) + and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.jobs==1 + and .source_mutated==false and .takeover_allowed==false and .backup_verified==false + and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null +source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") +fingerprint "$database" > "$evidence/fingerprint-repeat.json" 2>> "$evidence/fingerprint.stderr" +[[ "$(jq -r .fingerprint "$evidence/fingerprint-repeat.json")" == "$source_fingerprint" ]] +# Back up the stopped source and restore it into a separate database, as the +# backup procedure does. The restored copy must fingerprint equal, and one +# changed row in it must change the fingerprint. +restored=upgrade_restore_ref${label:+_$label} +case "$backend" in + mysql) + mysql_admin() { + local tool=$1 + shift + docker run --rm -i --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint "$tool" "$MYSQL_IMAGE" \ + -h "$host" -uroot -p"$password" "$@" + } + mysql_admin mysqldump --single-transaction --quick --no-tablespaces "$database" > "$state/backup.sql" 2> "$evidence/backup.stderr" + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/restore-owned.txt" + mysql_admin mysql -e "CREATE DATABASE $restored CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 2>> "$evidence/backup.stderr" + mysql_admin mysql "$restored" < "$state/backup.sql" 2>> "$evidence/backup.stderr" + mysql_admin mysql -e "GRANT SELECT ON $restored.* TO 'upgrade_source_reader'@'%'" 2>> "$evidence/backup.stderr" + change_restored() { mysql_admin mysql "$restored" -e 'UPDATE jobs SET attempts=attempts+1' 2>> "$evidence/backup.stderr"; } ;; + pgsql) + pg_admin() { + local tool=$1 + shift + docker run --rm -i --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint "$tool" "$POSTGRES_IMAGE" -h "$host" -U parity "$@" + } + pg_admin pg_dump -Fc -d "$database" > "$state/backup.dump" 2> "$evidence/backup.stderr" + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/restore-owned.txt" + pg_admin createdb "$restored" 2>> "$evidence/backup.stderr" + pg_admin pg_restore --no-owner --exit-on-error -d "$restored" < "$state/backup.dump" 2>> "$evidence/backup.stderr" + pg_admin psql -d "$restored" -v ON_ERROR_STOP=1 -c "GRANT CONNECT ON DATABASE $restored TO upgrade_source_reader" \ + -c 'GRANT USAGE ON SCHEMA public TO upgrade_source_reader' \ + -c 'GRANT SELECT ON ALL TABLES IN SCHEMA public TO upgrade_source_reader' > /dev/null 2>> "$evidence/backup.stderr" + change_restored() { pg_admin psql -d "$restored" -v ON_ERROR_STOP=1 -c 'UPDATE jobs SET attempts=attempts+1' > /dev/null 2>> "$evidence/backup.stderr"; } ;; + sqlite) + # A stopped SQLite installation is backed up by copying its file. + cp "$state/database.sqlite" "$state/restored.sqlite" + chmod 0666 "$state/restored.sqlite" + restored=/state/restored.sqlite + change_restored() { + php_run -r '(new PDO("sqlite:/state/restored.sqlite"))->exec("UPDATE jobs SET attempts=attempts+1") === 1 || exit(1);' + } ;; + *) exit 2 ;; +esac +fingerprint "$restored" > "$evidence/restored-fingerprint.json" 2>> "$evidence/fingerprint.stderr" +[[ "$(jq -r .fingerprint "$evidence/restored-fingerprint.json")" == "$source_fingerprint" ]] +change_restored +fingerprint "$restored" > "$evidence/changed-fingerprint.json" 2>> "$evidence/fingerprint.stderr" +changed_fingerprint=$(jq -r .fingerprint "$evidence/changed-fingerprint.json") +[[ $changed_fingerprint =~ ^sha256:[0-9a-f]{64}$ && $changed_fingerprint != "$source_fingerprint" ]] +# Backup-first takeover of a copy of the stopped source, as an operator's +# live database: its digest is the one the restored backup matched. Native +# then resumes the originals, and PHP must refuse the upgraded database. +target=() +case "$backend" in + sqlite) + cp "$state/database.sqlite" "$state/takeover.sqlite" + chmod 0666 "$state/takeover.sqlite" + target=(-e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite) + target_network=none ;; + pgsql) + # The same backup restored once more, taken over with the owner role. + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + pg_admin createdb "$takeover_database" 2>> "$evidence/backup.stderr" + pg_admin pg_restore --no-owner --exit-on-error -d "$takeover_database" < "$state/backup.dump" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=pgsql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=parity -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; + mysql) + # The same backup restored once more, taken over as root, which can see + # every session (the PROCESS privilege). + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + mysql_admin mysql -e "CREATE DATABASE $takeover_database CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 2>> "$evidence/backup.stderr" + mysql_admin mysql "$takeover_database" < "$state/backup.sql" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=mysql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=root -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; + *) ;; +esac +if (( ${#target[@]} )); then + docker run --rm --name "$resource-takeover" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network "$target_network" "${target[@]}" -e DW_RUST_EXPERIMENTAL=1 \ + -e DW_UPGRADE_BACKUP_FINGERPRINT="$source_fingerprint" \ + -e DW_RUST_UPGRADE_QUEUE_CONNECTION=database -e DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION=database \ + -e DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE=default \ + -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-takeover \ + > "$evidence/takeover.json" 2> "$evidence/takeover.stderr" + jq -e --arg backend "$backend" --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" + and (.backend==$backend or ($backend=="mysql" and .backend=="mariadb")) + and .source_layout==$layout and .fingerprint==$fingerprint + and .backup_fingerprint_matched==true + and .converted.incarnations_copied==(if $layout=="frozen" then 0 else 1 end) + and .stored_representation_compatibility_verified==false' "$evidence/takeover.json" >/dev/null + docker run -d --name "$resource-native" --network "$RESOURCE_SCOPE" --network-alias parity-upgrade-native \ + --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m -e DW_RUST_EXPERIMENTAL=1 \ + -e DW_BIND_ADDRESS=0.0.0.0:8080 -e DW_AUTH_DRIVER=token -e DW_AUTH_TOKEN=upgrade-fixture-token \ + "${target[@]}" -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" > "$evidence/native-container.txt" + ready=0 + for attempt in $(seq 1 30); do + if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ + -fsS http://parity-upgrade-native:8080/api/ready > "$evidence/native-readiness.json"; then ready=1; break; fi + sleep 1 + done + [[ $ready == 1 ]] + docker run --rm --name "$resource-resume" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ + --network "$RESOURCE_SCOPE" -e DW_PARITY_TOKEN=upgrade-fixture-token -v "$PWD:/source:ro" --entrypoint php "$image" \ + /source/scripts/conformance/server-parity/upgrade-takeover-resume.php http://parity-upgrade-native:8080 \ + "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" + jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 + and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted" + and .lease.attempt==2 and .lease.redelivered_after_seconds>=0 and .lease.stale_attempt_status==409 + and .lease.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null + if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ + --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ + -e CACHE_STORE=file -e QUEUE_CONNECTION=database -v "$PWD/$state:/state" --entrypoint php "$image" \ + artisan server:assert-php-database > "$evidence/php-refusal.txt" 2>&1; then exit 1; fi + grep -q php_database_refused "$evidence/php-refusal.txt" +fi snapshot > "$evidence/after.json" cmp "$evidence/before.json" "$evidence/after.json" refusal() { @@ -163,9 +326,12 @@ git diff --cached --quiet jq -n --arg revision "$revision" --arg native "$native_tree" \ --arg image "$image" --arg backend "$backend" --arg binary "$binary_sha256" \ --slurpfile inventory "$evidence/inspection.json" --slurpfile originals "$evidence/originals.json" \ + --slurpfile fingerprint "$evidence/fingerprint.json" \ '{outcome:"pass",scope:"read_only_source_inventory_and_refusals",backend:$backend, runner_revision:$revision,native_tree:$native,native_binary_sha256:$binary,php_image:$image, - original_runs:2,source_state_unchanged_by_inspection:true, + original_runs:2,source_state_unchanged_by_inspection:true,source_layout:$inventory[0].source_layout, + source_fingerprint:$fingerprint[0].fingerprint,restored_backup_fingerprint_equal:true, + changed_restored_row_changes_fingerprint:true, catalog_sha256:$inventory[0].catalog_sha256,table_count:($inventory[0].table_rows|length), originals:($originals[0].originals|to_entries|map({kind:.key,workflow_id:.value.workflow_id, run_id:.value.run_id,history_events:(.value.history|length),activity_attempt_id:.value.activity_task.activity_attempt_id})), diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index da17fe171..09da717d5 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 3')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 5')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the two isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-lease', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the four isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php new file mode 100644 index 000000000..41f0281d5 --- /dev/null +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -0,0 +1,178 @@ + rtrim($url, '/'), 'http_errors' => false, 'connect_timeout' => 1, 'timeout' => 10]); +$headers = ['Authorization' => 'Bearer '.getenv('DW_PARITY_TOKEN'), 'X-Namespace' => 'default', + 'X-Durable-Workflow-Control-Plane-Version' => '2', 'X-Durable-Workflow-Protocol-Version' => '1.20']; +$request = static function (string $method, string $path, array $body = []) use ($http, $headers): array { + $response = $http->request($method, $path, ['headers' => $headers] + ($body === [] ? [] : ['json' => $body])); + + return [$response->getStatusCode(), json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; +}; +$check = static function (bool $condition, string $message): void { + if (! $condition) { + throw new RuntimeException($message); + } +}; +$expect = static function (array $response, int $status, string $operation) use ($check): array { + $check($response[0] === $status, $operation.': '.json_encode(['status' => $response[0], + 'reason' => $response[1]['reason'] ?? null], JSON_THROW_ON_ERROR)); + + return $response[1]; +}; +// Complete events, keys sorted at every level: PHP-written history must read +// exactly as PHP returned it before the takeover. +$sorted = static function (mixed $value) use (&$sorted): mixed { + if (! is_array($value)) { + return $value; + } + if (! array_is_list($value)) { + ksort($value); + } + + return array_map($sorted, $value); +}; +$keep = static fn (array $original, array $current): bool => json_encode($sorted($original), JSON_THROW_ON_ERROR) + === json_encode($sorted(array_slice($current, 0, count($original))), JSON_THROW_ON_ERROR); +$types = static fn (array $history): array => array_column($history, 'event_type'); +$history = static function (array $original) use ($request, $expect): array { + return $expect($request('GET', '/api/workflows/'.$original['workflow_id'].'/runs/'.$original['run_id'].'/history'), + 200, 'history')['events']; +}; +$worker = $originals['activity']['activity_task']['lease_owner']; +$queue = $originals['activity']['workflow_task']['task_queue'] ?? 'upgrade-source'; +$result = ['codec' => 'avro', 'blob' => Avro::serialize(['resumed under native'])]; +$outcome = []; +$activity = $originals['activity']; +$timer = $originals['timer']; +$signal = $originals['signal']; +$lease = $originals['lease']; +$complete =static function (array $claimed, string $operation) use ($expect, $request, $result): void { + $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], + 'workflow_task_attempt' => $claimed['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, $operation); +}; + +// The timer scheduled under PHP keeps its history and deadline. +$scheduled = array_values(array_filter($timer['history'], static fn (array $event): bool => $event['event_type'] === 'TimerScheduled'))[0]; +$fireAt = $scheduled['payload']['fire_at']; +$check($keep($timer['history'], $history($timer)), 'Original timer history changed'); +$deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); +$check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); + +// The workflow task PHP leased to a stopped worker keeps its PHP expiry. +$check(is_string($lease['workflow_task']['lease_expires_at'] ?? null), 'The original lease expiry is missing'); +$leaseExpiry = (float) (new DateTimeImmutable($lease['workflow_task']['lease_expires_at']))->format('U.u'); +$check($leaseExpiry - microtime(true) <= (float) $timerBudget, 'The original lease expiry is outside the wait budget'); +$check($keep($lease['history'], $history($lease)), 'Original lease history changed'); + +// The activity leased under PHP completes with its original attempt. +$task = $activity['activity_task']; +$completion = ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], 'result' => $result]; +$completed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'original activity completion'); +$check(($completed['outcome'] ?? null) === 'completed' && ($completed['recorded'] ?? null) === true, 'Original activity completion was not recorded'); +$replayed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'activity completion replay'); +$check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); + +// Each original workflow then gets one workflow task: the activity's and the +// signal's at once, the timer's at its deadline and the lease's when PHP's +// lease expires. On a slow host some are already due, so they may be claimed +// in any order. +$original = $activity['history']; +$claims = []; +$leaseOutcome = null; +$until = max($deadline, $leaseExpiry, microtime(true)) + 30; +while (count($claims) < 4 && microtime(true) < $until) { + $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, + 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; + if (! is_array($claimed)) { + continue; + } + $run = $claimed['run_id']; + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id'], $lease['run_id']], true) && ! isset($claims[$run]), + 'An unexpected workflow task was claimed'); + $claims[$run] = true; + if ($run === $activity['run_id']) { + $check(array_slice($types($claimed['history_events']), 0, count($original)) === $types($original), + 'Original activity history changed in the resumed task'); + $check(end($claimed['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); + $complete($claimed, 'resumed workflow completion'); + } elseif ($run === $signal['run_id']) { + // The signal PHP accepted is delivered from PHP's rows, with its arguments. + $check(array_slice($types($claimed['history_events']), 0, count($signal['history'])) === $types($signal['history']), + 'Original signal history changed in the resumed task'); + $received = array_values(array_filter($claimed['history_events'], + static fn (array $event): bool => $event['event_type'] === 'SignalReceived')); + $check(count($received) === 1 && json_encode($sorted($received[0]['payload']['arguments']), JSON_THROW_ON_ERROR) + === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), + 'The PHP-accepted signal arguments differ under native'); + $complete($claimed, 'signal workflow completion'); + } elseif ($run === $lease['run_id']) { + // After PHP's lease expires, native redelivers the same task as its + // next attempt. The expired PHP attempt cannot complete over it. + $expired = $lease['workflow_task']; + $check($claimed['task_id'] === $expired['task_id'] + && $claimed['workflow_task_attempt'] === $expired['workflow_task_attempt'] + 1, + 'The expired PHP lease was not redelivered as its next attempt'); + $check(microtime(true) >= $leaseExpiry, 'Native redelivered a PHP lease before it expired'); + $check($types($claimed['history_events']) === $types($lease['history']), + 'Original lease history changed in the redelivered task'); + $stale = $request('POST', '/api/worker/workflow-tasks/'.$expired['task_id'].'/complete', ['lease_owner' => $expired['lease_owner'], + 'workflow_task_attempt' => $expired['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]); + $check($stale[0] === 409, 'The expired PHP attempt completed over its redelivery'); + $leaseOutcome = ['attempt' => $claimed['workflow_task_attempt'], + 'redelivered_after_seconds' => round(microtime(true) - $leaseExpiry, 3), + 'stale_attempt_status' => $stale[0], 'stale_attempt_reason' => $stale[1]['reason'] ?? null]; + $complete($claimed, 'redelivered lease completion'); + } else { + $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); + $complete($claimed, 'timer workflow completion'); + } +} +$check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); +$check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); +$check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); +$check(isset($claims[$lease['run_id']]), 'The expired PHP lease was not redelivered'); + +// The workflow task PHP already completed is a stale attempt. +$stale = $activity['workflow_task']; +$refused = $request('POST', '/api/worker/workflow-tasks/'.$stale['task_id'].'/complete', ['lease_owner' => $stale['lease_owner'], + 'workflow_task_attempt' => $stale['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]); +$check($refused[0] === 409, 'A stale PHP workflow task attempt was accepted'); +$closed = $history($activity); +$check($keep($original, $closed) && end($closed)['event_type'] === 'WorkflowCompleted', + 'The resumed activity workflow did not complete on its original history'); +$outcome['activity'] = ['run_id' => $activity['run_id'], 'events' => array_column($closed, 'event_type'), + 'stale_attempt_status' => $refused[0], 'stale_attempt_reason' => $refused[1]['reason'] ?? null]; + +$timerHistory = $history($timer); +$firedPublic = array_values(array_filter($timerHistory, static fn (array $event): bool => $event['event_type'] === 'TimerFired')); +$check(count($firedPublic) === 1, 'The original timer did not fire exactly once'); +$firedAt = (float) (new DateTimeImmutable($firedPublic[0]['timestamp']))->format('U.u'); +$check($firedAt >= $deadline, 'The original timer fired before its deadline'); +$check($keep($timer['history'], $timerHistory) && end($timerHistory)['event_type'] === 'WorkflowCompleted', + 'The timer workflow did not complete on its original history'); +$signalHistory = $history($signal); +$check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', + 'The signal workflow did not complete on its original history'); +$outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; +$leaseHistory = $history($lease); +$check($keep($lease['history'], $leaseHistory) && end($leaseHistory)['event_type'] === 'WorkflowCompleted', + 'The redelivered lease workflow did not complete on its original history'); +$outcome['lease'] = ['run_id' => $lease['run_id'], 'lease_expires_at' => $lease['workflow_task']['lease_expires_at']] + + $leaseOutcome + ['events' => array_column($leaseHistory, 'event_type')]; +$outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, + 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; +echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, + JSON_THROW_ON_ERROR)."\n";