From 68d78be19c445b22e4cd95a404c50c4f3f52f271 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 16:58:01 +0000 Subject: [PATCH 1/6] Add a multi-node kill and restart qualification harness Three native nodes share one MySQL, PostgreSQL or SQLite database behind HAProxy. Published-SDK workers and an offered one-activity workload reach them only through the balancer while nodes are killed and restarted and a worker is paused past its workflow task lease. Every workflow must complete once with its original output and an ordered history, and a database audit must find no duplicate committed outcome. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/multinode/README.md | 77 +++++++ scripts/perf/multinode-haproxy.cfg | 38 ++++ scripts/perf/multinode-workload.php | 176 ++++++++++++++++ scripts/perf/run-server-port-multinode.sh | 191 ++++++++++++++++++ .../perf/server-port-multinode.compose.yml | 123 +++++++++++ 5 files changed, 605 insertions(+) create mode 100644 benchmarks/server-port/multinode/README.md create mode 100644 scripts/perf/multinode-haproxy.cfg create mode 100644 scripts/perf/multinode-workload.php create mode 100755 scripts/perf/run-server-port-multinode.sh create mode 100644 scripts/perf/server-port-multinode.compose.yml diff --git a/benchmarks/server-port/multinode/README.md b/benchmarks/server-port/multinode/README.md new file mode 100644 index 000000000..f875eed92 --- /dev/null +++ b/benchmarks/server-port/multinode/README.md @@ -0,0 +1,77 @@ +# Multi-node kill and restart qualification + +This qualifies native multi-node behavior: no lost durable work, no duplicate +committed outcome, and no stale worker attempt accepted. It runs on each +required database family while server nodes are killed and restarted, and +while a worker loses its lease. It is a correctness check, not a capacity +measurement. + +## Topology + +[`server-port-multinode.compose.yml`](../../../scripts/perf/server-port-multinode.compose.yml) +runs these services: + +- **Database.** One of MySQL 8.4, PostgreSQL 17 or SQLite. With SQLite, the + three nodes share one database file on one host. +- **Native nodes.** Three nodes with the same binary and configuration, one + CPU and 512 MiB each. All three start together against the empty + database, so bootstrap races too. +- **Balancer.** HAProxy round-robins over the nodes. Health checks every + second take a dead node out after two failures. Like a production + balancer, it retries a request on another node only when it could not + connect, so the request never reached a server. A request a node accepted + and then dropped fails back to its client. +- **Clients.** Two SDK workers and the offered workload, from the + [capacity bindings](../../capacity/v1/bindings/php) with the published PHP + SDK. They reach the nodes only through the balancer. A worker that exits is + restarted, as a supervisor would; `container-states.txt` records each + restart. A restart is an SDK finding, not a server failure, but it is + reported with the run. + +## Procedure + +[`run-server-port-multinode.sh`](../../../scripts/perf/run-server-port-multinode.sh) +`OUTPUT_DIR mysql|postgres|sqlite` offers one-activity workflows at +0.5/s in three 180-second windows. Each window drains for up to 420 s, longer +than the five-minute activity lease: an activity whose completion a killed node +dropped is redelivered only when that lease expires. Two +kinds of chaos run during the windows: + +- **Node kills.** Every 15 s, one node in turn is killed with SIGKILL and + restarted 4 s later. +- **Worker pauses.** Every 90 s, one SDK worker is frozen with `docker pause` + for the workflow task lease plus 10 s, then resumed. By then the server has + given its workflow tasks to the other worker, so its completions are stale. + +[`multinode-workload.php`](../../../scripts/perf/multinode-workload.php) +starts each workflow under its own ID. It retries a start, describe or history +read that a dying node never answered, and a retried start finds its own run. +Each window passes only if every planned workflow completed, with: + +- its original output; +- contiguous history sequences; +- exactly one `WorkflowStarted`, `ActivityScheduled`, `ActivityCompleted` and + `WorkflowCompleted`, in that order; +- an `ActivityCompleted` from the last attempt that started. + +Afterwards the database audit must find: + +- every run completed; +- no run with two `WorkflowCompleted` events; +- no activity with two completed attempts; +- no duplicate history sequence. + +The balancer log counts the stale completions the servers refused (`409`) +and the requests a killed node never answered (`5xx`). Node logs are checked +for panics. + +```sh +# The locked SDK and an optimized binary, as the v2 README describes. +export DW_PORT_SOURCE_ROOT=/abs/source DW_PORT_SDK_ROOT=/abs/sdk DW_PORT_NATIVE_BINARY=/abs/native/server +for db in mysql postgres sqlite; do + bash scripts/perf/run-server-port-multinode.sh "results/$db" "$db" +done +``` + +`--smoke` runs one 20-second window to check the setup. It is never a +qualification. diff --git a/scripts/perf/multinode-haproxy.cfg b/scripts/perf/multinode-haproxy.cfg new file mode 100644 index 000000000..a783664d5 --- /dev/null +++ b/scripts/perf/multinode-haproxy.cfg @@ -0,0 +1,38 @@ +# Round-robin over the three native nodes. A node that stops answering health +# checks is taken out within two seconds and returns when it answers again. +# A request that could not connect to a node never reached it, so it is +# retried on another node, as a production balancer does. A request a node +# accepted and then dropped when it was killed is never retried here: it fails +# back to its client, which owns that retry. +global + log stdout format raw local0 info + +resolvers docker + nameserver dns 127.0.0.11:53 + hold valid 1s + +defaults + mode http + log global + # One line per request: the evidence that stale completions were refused. + option httplog + timeout connect 2s + # Long polls hold for up to 30 seconds. + timeout client 75s + timeout server 75s + # The default retry-on is conn-failure only. + retries 3 + option redispatch 1 + +frontend api + bind :8080 + default_backend natives + +backend natives + balance roundrobin + option httpchk GET /api/health + http-check expect status 200 + default-server inter 1s fall 2 rise 2 resolvers docker init-addr none + server native-1 native-1:8080 check + server native-2 native-2:8080 check + server native-3 native-3:8080 check diff --git a/scripts/perf/multinode-workload.php b/scripts/perf/multinode-workload.php new file mode 100644 index 000000000..3d5b14288 --- /dev/null +++ b/scripts/perf/multinode-workload.php @@ -0,0 +1,176 @@ + <5-600 offer seconds> <10-600 drain seconds> +use Composer\InstalledVersions; +use DurableWorkflow\Exception\ServerException; + +require dirname(__DIR__, 2).'/benchmarks/capacity/v1/bindings/php/capacity_adapter.php'; +capacityAutoload(); + +$rate = filter_var($argv[1] ?? null, FILTER_VALIDATE_FLOAT, ['options' => ['min_range' => 0.1, 'max_range' => 5.0]]); +$duration = filter_var($argv[2] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 5, 'max_range' => 600]]); +$drain = filter_var($argv[3] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 10, 'max_range' => 600]]); +$prefix = $argv[4] ?? ''; +if ($rate === false || $duration === false || $drain === false || ! preg_match('/\A[a-z0-9-]{1,40}\z/', $prefix)) { + throw new InvalidArgumentException('Usage: multinode-workload.php <0.1-5 starts/sec> <5-600 offer seconds> <10-600 drain seconds> .'); +} +$queue = capacityEnvironment('DURABLE_WORKFLOW_TASK_QUEUE'); +if ($queue === null) { + throw new InvalidArgumentException('Set DURABLE_WORKFLOW_TASK_QUEUE.'); +} +$client = capacityClient(false); +$blob = str_repeat('m', 1024); +$input = ['blob' => $blob, 'payload_contract' => ['workflow_input_bytes' => 1024, 'workflow_result_bytes' => 1024, + 'activity_input_bytes' => 1024, 'activity_result_bytes' => 1024, 'signal_bytes' => 0]]; +$retries = []; +$errors = []; + +// Only a server's 4xx answer is final. A transport failure, a 5xx answer, or +// a reply cut off when its node was killed (even under a 2xx status) is +// retried; every retried request is safe to repeat under its workflow ID. +$retrying = static function (string $what, callable $call, float $until) use (&$retries) { + $delay = 0.2; + while (true) { + try { + return $call(); + } catch (ServerException $error) { + if ($error->status >= 400 && $error->status < 500 || microtime(true) >= $until) { + throw $error; + } + $class = 'http_'.$error->status; + } catch (Throwable $error) { + if (microtime(true) >= $until) { + throw $error; + } + $class = get_class($error); + } + $retries[$what][$class] = ($retries[$what][$class] ?? 0) + 1; + usleep((int) ($delay * 1_000_000)); + $delay = min(2.0, $delay * 2); + } +}; + +$planned = (int) ceil($rate * $duration); +$began = microtime(true); +$offerEnds = $began + $duration; +$deadline = $offerEnds + $drain; +$ids = []; +$started = []; +for ($n = 0; $n < $planned; $n++) { + $due = $began + $n / $rate; + if (($wait = $due - microtime(true)) > 0) { + usleep((int) ($wait * 1_000_000)); + } + $id = sprintf('%s-%04d', $prefix, $n); + $ids[] = $id; + try { + $retrying('start', static function () use ($client, $id, $queue, $input): void { + try { + $client->startWorkflow('capacity.v1.one_activity', $id, $queue, [$input]); + } catch (ServerException $error) { + // A retried start whose first attempt was accepted finds its own run. + if ($error->status !== 409) { + throw $error; + } + $existing = $client->describeWorkflow($id); + if ($existing->workflowType !== 'capacity.v1.one_activity') { + throw $error; + } + } + }, $due + 60); + $started[$id] = microtime(true); + } catch (Throwable $error) { + $errors[] = $id.': start: '.$error->getMessage(); + } +} + +$completed = []; +$pending = array_keys($started); +while ($pending !== [] && microtime(true) < $deadline) { + $id = array_shift($pending); + try { + $execution = $retrying('describe', static fn () => $client->describeWorkflow($id), $deadline); + if ($execution->status === 'completed') { + $completed[$id] = $execution; + } elseif ($execution->isTerminal === true) { + $errors[] = $id.': terminal status '.$execution->status; + } else { + $pending[] = $id; + usleep(20_000); + } + } catch (Throwable $error) { + $pending[] = $id; + } +} + +$verified = 0; +$attemptsByWorkflow = []; +foreach ($completed as $id => $execution) { + try { + if ($execution->output !== $blob) { + throw new RuntimeException('output differs'); + } + $history = $retrying('history', static fn () => $client->workflowHistory($id, $execution->runId), microtime(true) + 60); + $events = $history['events'] ?? []; + $sequences = array_column($events, 'sequence'); + if ($sequences !== range(1, count($events))) { + throw new RuntimeException('history sequences are not contiguous'); + } + $types = array_column($events, 'event_type'); + $counts = array_count_values($types); + $allowed = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', 'ActivityCompleted', 'WorkflowCompleted']; + if (($unexpected = array_diff(array_keys($counts), $allowed)) !== []) { + throw new RuntimeException('unexpected events '.implode(',', $unexpected)); + } + foreach (['WorkflowStarted', 'ActivityScheduled', 'ActivityCompleted', 'WorkflowCompleted'] as $once) { + if (($counts[$once] ?? 0) !== 1) { + throw new RuntimeException($once.' recorded '.($counts[$once] ?? 0).' times'); + } + } + $starts = array_values(array_filter($events, static fn (array $e): bool => $e['event_type'] === 'ActivityStarted')); + $completion = array_values(array_filter($events, static fn (array $e): bool => $e['event_type'] === 'ActivityCompleted'))[0]; + if ($starts === [] || end($starts)['payload']['activity_attempt_id'] !== $completion['payload']['activity_attempt_id']) { + throw new RuntimeException('the completed activity attempt is not the last one started'); + } + $order = array_values(array_intersect($types, ['WorkflowStarted', 'ActivityScheduled', 'ActivityCompleted', 'WorkflowCompleted'])); + if ($order !== ['WorkflowStarted', 'ActivityScheduled', 'ActivityCompleted', 'WorkflowCompleted']) { + throw new RuntimeException('events out of order'); + } + $attemptsByWorkflow[count($starts)] = ($attemptsByWorkflow[count($starts)] ?? 0) + 1; + $verified++; + } catch (Throwable $error) { + $errors[] = $id.': verification: '.$error->getMessage(); + } +} +foreach ($pending as $id) { + $errors[] = $id.': not completed by the drain deadline'; +} + +echo json_encode([ + 'probe' => 'multinode-offered-one-activity', + 'sdk_php' => InstalledVersions::getPrettyVersion('durable-workflow/sdk'), + 'prefix' => $prefix, + 'offered_rate_per_second' => $rate, + 'offer_seconds' => $duration, + 'drain_limit_seconds' => $drain, + 'started_at_unix_seconds' => $began, + 'finished_at_unix_seconds' => microtime(true), + 'planned_starts' => $planned, + 'accepted_starts' => count($started), + 'completed' => count($completed), + 'verified' => $verified, + 'activity_attempts_per_workflow' => $attemptsByWorkflow, + 'retries' => $retries, + 'errors' => $errors, + 'outcome' => $errors === [] && $verified === $planned ? 'pass' : 'fail', +], JSON_PRETTY_PRINT)."\n"; +exit($errors === [] && $verified === $planned ? 0 : 1); diff --git a/scripts/perf/run-server-port-multinode.sh b/scripts/perf/run-server-port-multinode.sh new file mode 100755 index 000000000..b4241282e --- /dev/null +++ b/scripts/perf/run-server-port-multinode.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Multi-node native kill/restart qualification on one database family. +# +# run-server-port-multinode.sh OUTPUT_DIR mysql|postgres|sqlite [--smoke] +# +# Three native nodes share one database behind HAProxy. Two published-SDK +# workers and the offered workload reach them only through the balancer. +# While work is offered, one node at a time is killed with SIGKILL and +# restarted, and one SDK worker is paused for longer than a workflow task +# lease, so its late completions are stale. Every workflow must complete +# once with its original output and an ordered history, and the database +# audit must find no duplicate committed outcome. +# +# Set DW_PORT_SOURCE_ROOT, DW_PORT_SDK_ROOT and DW_PORT_NATIVE_BINARY as the +# server-port v2 README describes. Run on a host with no other workload. +set -Eeuo pipefail + +out=$1 +db=$2 +mode=${3:-} +case "$db" in mysql|postgres|sqlite) ;; *) echo "Choose mysql, postgres or sqlite." >&2; exit 2 ;; esac +case "$mode" in ""|--smoke) ;; *) echo "Unknown option $mode." >&2; exit 2 ;; esac +: "${DW_PORT_SOURCE_ROOT:?Set the absolute source directory}" +: "${DW_PORT_SDK_ROOT:?Set the normally installed SDK directory}" +: "${DW_PORT_NATIVE_BINARY:?Set the absolute native binary path}" +[[ ! -e $out ]] || { echo "Refusing to overwrite $out." >&2; exit 2; } +mkdir -p "$out" +out=$(realpath "$out") +compose="$DW_PORT_SOURCE_ROOT/scripts/perf/server-port-multinode.compose.yml" +# The SDK vendor directory is mounted over this ignored mount point. +mkdir -p "$DW_PORT_SOURCE_ROOT/benchmarks/capacity/v1/bindings/php/vendor" +log() { printf '%s %s\n' "$(date -u +%FT%T.%3NZ)" "$*" | tee -a "$out/run.log" >&2; } + +if [[ $mode == --smoke ]]; then + rate=0.5 offer=20 drain=120 windows=1 kill_every=8 down=3 pause_every=30 +else + # The drain outlasts the five-minute activity lease: an activity whose + # completion a killed node dropped is redelivered only when that lease expires. + rate=${DW_MN_RATE:-0.5} offer=${DW_MN_OFFER_SECONDS:-180} drain=${DW_MN_DRAIN_SECONDS:-420} + windows=${DW_MN_WINDOWS:-3} kill_every=${DW_MN_KILL_EVERY:-15} down=${DW_MN_DOWN_SECONDS:-4} + pause_every=${DW_MN_PAUSE_EVERY:-90} +fi +lease=${DW_MN_WORKFLOW_TASK_TIMEOUT:-60} +pause=$((lease + 10)) +export DW_MN_WORKFLOW_TASK_TIMEOUT=$lease +export DW_MN_HAPROXY_DIGEST=${DW_MN_HAPROXY_DIGEST:-094401c230b9bf01a6a4dc96e455795e261483f3bd2f42f19d0e6f259bd43837} +export DW_MN_STATE_DIR="$out/state" +mkdir -p "$DW_MN_STATE_DIR" +chmod 0777 "$DW_MN_STATE_DIR" +case "$db" in + mysql) export DW_MN_DB_CONNECTION=mysql DW_MN_DB_HOST=mysql DW_MN_DB_DATABASE=durable_workflow \ + DW_MN_DB_USERNAME=workflow DW_MN_DB_PASSWORD=multinode-mysql; database=(mysql) ;; + postgres) export DW_MN_DB_CONNECTION=pgsql DW_MN_DB_HOST=postgres DW_MN_DB_DATABASE=durable_workflow \ + DW_MN_DB_USERNAME=workflow DW_MN_DB_PASSWORD=multinode-postgres; database=(postgres) ;; + sqlite) export DW_MN_DB_CONNECTION=sqlite DW_MN_DB_DATABASE=/state/native.sqlite; database=() ;; +esac + +project="dw-mn-$db-$(date -u +%Y%m%dt%H%M%S)" +dc() { docker compose -p "$project" -f "$compose" --profile "$db" "$@"; } +container() { dc --profile clients ps -a -q "$1"; } +nodes=(native-1 native-2 native-3) + +stop_file="$out/.stop" +loops=() +cleanup() { + status=$? + touch "$stop_file" + for pid in "${loops[@]}"; do kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true; done + docker unpause "$(container sdk-worker-1)" >/dev/null 2>&1 || true + for node in "${nodes[@]}"; do docker logs "$(container "$node")" > "$out/$node.log" 2>&1 || true; done + docker logs "$(container lb)" > "$out/lb.log" 2>&1 || true + for worker in sdk-worker-1 sdk-worker-2; do docker logs "$(container "$worker")" > "$out/$worker.log" 2>&1 || true; done + dc --profile clients ps -a --format json > "$out/containers-final.jsonl" 2>/dev/null || true + dc --profile clients down --volumes --remove-orphans >> "$out/run.log" 2>&1 || true + rm -rf "$DW_MN_STATE_DIR" + log "finished with status $status" + exit "$status" +} +trap cleanup EXIT + +jq -n --arg db "$db" --arg project "$project" --arg source "$(git -C "$DW_PORT_SOURCE_ROOT" rev-parse HEAD)" \ + --arg dirty "$(git -C "$DW_PORT_SOURCE_ROOT" status --porcelain --untracked-files=no | wc -l)" \ + --arg binary "$(sha256sum "$DW_PORT_NATIVE_BINARY" | cut -d ' ' -f1)" --arg smoke "$mode" \ + --arg kernel "$(uname -r)" --arg cpus "$(nproc)" \ + --arg cpu_model "$(awk -F': ' '/model name/ {print $2; exit}' /proc/cpuinfo)" \ + --argjson schedule "$(jq -n --arg rate "$rate" --arg offer "$offer" --arg drain "$drain" --arg windows "$windows" \ + --arg kill_every "$kill_every" --arg down "$down" --arg pause_every "$pause_every" --arg pause "$pause" --arg lease "$lease" \ + '{rate:($rate|tonumber),offer_seconds:($offer|tonumber),drain_seconds:($drain|tonumber),windows:($windows|tonumber), + node_kill_every_seconds:($kill_every|tonumber),node_down_seconds:($down|tonumber), + worker_pause_every_seconds:($pause_every|tonumber),worker_pause_seconds:($pause|tonumber), + workflow_task_lease_seconds:($lease|tonumber)}')" \ + '{database:$db,compose_project:$project,source_revision:$source,source_modified_files:($dirty|tonumber),native_binary_sha256:$binary,smoke:($smoke=="--smoke"), + schedule:$schedule,machine:{kernel:$kernel,logical_cpus:($cpus|tonumber),cpu_model:$cpu_model}}' > "$out/identity.json" + +log "starting $project" +if (( ${#database[@]} )); then dc up -d --wait "${database[@]}" >> "$out/run.log" 2>&1; fi +# All three nodes start at once against the empty database. +dc up -d --wait "${nodes[@]}" >> "$out/run.log" 2>&1 +dc up -d lb >> "$out/run.log" 2>&1 +dc --profile clients up -d sdk-worker-1 sdk-worker-2 >> "$out/run.log" 2>&1 + +# One node at a time is killed and restarted. +kill_nodes() { + local i=0 node id + while [[ ! -e $stop_file ]]; do + sleep "$kill_every" + [[ -e $stop_file ]] && break + node=${nodes[$((i % 3))]} + id=$(container "$node") + printf '%s\tkill\t%s\n' "$(date -u +%s.%N)" "$node" >> "$out/chaos.tsv" + docker kill -s KILL "$id" >/dev/null 2>&1 || true + sleep "$down" + docker start "$id" >/dev/null 2>&1 || true + printf '%s\tstart\t%s\n' "$(date -u +%s.%N)" "$node" >> "$out/chaos.tsv" + i=$((i + 1)) + done +} +# One SDK worker is frozen for longer than a workflow task lease, then +# resumes with the tasks it held: its completions are stale by then. +pause_worker() { + local id + while [[ ! -e $stop_file ]]; do + sleep "$pause_every" + [[ -e $stop_file ]] && break + id=$(container sdk-worker-1) + printf '%s\tpause\tsdk-worker-1\n' "$(date -u +%s.%N)" >> "$out/chaos.tsv" + docker pause "$id" >/dev/null 2>&1 || true + sleep "$pause" + docker unpause "$id" >/dev/null 2>&1 || true + printf '%s\tunpause\tsdk-worker-1\n' "$(date -u +%s.%N)" >> "$out/chaos.tsv" + done +} +kill_nodes & loops+=($!) +pause_worker & loops+=($!) + +failed=0 +for window in $(seq 1 "$windows"); do + name="window-$window" + log "$name: $rate/s for ${offer}s" + if dc --profile clients run --rm --no-deps probe scripts/perf/multinode-workload.php "$rate" "$offer" "$drain" \ + "mn-$db-w$window" > "$out/$name.json" 2> "$out/$name.stderr"; then + log "$name passed" + else + log "$name failed" + failed=1 + fi +done + +touch "$stop_file" +for pid in "${loops[@]}"; do wait "$pid" 2>/dev/null || true; done +loops=() +for node in "${nodes[@]}"; do docker start "$(container "$node")" >/dev/null 2>&1 || true; done +docker unpause "$(container sdk-worker-1)" >/dev/null 2>&1 || true + +# The balancer's request log shows what the chaos did: stale completions the +# servers refused, and requests a killed node never answered. +docker logs "$(container lb)" > "$out/lb.log" 2>&1 || true +count() { grep -E "$1" "$out/lb.log" | grep -cE "/[0-9-]+ $2 " || true; } +{ + printf 'requests\t%s\n' "$(grep -cE '"[A-Z]+ /api/' "$out/lb.log" || true)" + printf 'workflow_task_completions_409\t%s\n' "$(count '/api/worker/workflow-tasks/[^ ]+/complete HTTP' 409)" + printf 'activity_task_completions_409\t%s\n' "$(count '/api/worker/activity-tasks/[^ ]+/complete HTTP' 409)" + printf 'responses_5xx\t%s\n' "$(count '"[A-Z]+ /api/' '5[0-9][0-9]')" +} > "$out/lb-summary.tsv" + +audit_sql="SELECT 'runs', status, COUNT(*) FROM workflow_runs GROUP BY status; +SELECT 'tasks', task_type, status, COUNT(*) FROM workflow_tasks GROUP BY task_type, status; +SELECT 'attempts', status, COUNT(*) FROM activity_attempts GROUP BY status; +SELECT 'duplicate_workflow_completions', COUNT(*) FROM (SELECT workflow_run_id FROM workflow_history_events WHERE event_type='WorkflowCompleted' GROUP BY workflow_run_id HAVING COUNT(*) > 1) d; +SELECT 'duplicate_activity_completions', COUNT(*) FROM (SELECT activity_execution_id FROM activity_attempts WHERE status='completed' GROUP BY activity_execution_id HAVING COUNT(*) > 1) d; +SELECT 'duplicate_history_sequences', COUNT(*) FROM (SELECT workflow_run_id, sequence FROM workflow_history_events GROUP BY workflow_run_id, sequence HAVING COUNT(*) > 1) d;" +case "$db" in + mysql) dc exec -T mysql sh -c 'MYSQL_PWD=$MYSQL_PASSWORD exec mysql -u"$MYSQL_USER" "$MYSQL_DATABASE" --batch --skip-column-names -e "$1"' audit "$audit_sql" ;; + postgres) dc exec -T postgres psql -U workflow -d durable_workflow -At -F $'\t' -c "$audit_sql" ;; + sqlite) dc --profile clients run --rm --no-deps -v "$DW_MN_STATE_DIR:/state" -e AUDIT_SQL="$audit_sql" probe -r ' + $pdo = new PDO("sqlite:/state/native.sqlite"); + foreach (array_filter(array_map("trim", explode(";", getenv("AUDIT_SQL")))) as $sql) { + foreach ($pdo->query($sql, PDO::FETCH_NUM) as $row) { echo implode("\t", $row), "\n"; } + }' ;; +esac > "$out/database-audit.tsv" 2>> "$out/run.log" +for kind in workflow activity; do + awk -F'\t' -v k="duplicate_${kind}_completions" '$1==k && $2!=0 {bad=1} END {exit bad}' "$out/database-audit.tsv" || failed=1 +done +awk -F'\t' '$1=="duplicate_history_sequences" && $2!=0 {bad=1} END {exit bad}' "$out/database-audit.tsv" || failed=1 +awk -F'\t' '$1=="runs" && $2!="completed" {bad=1} END {exit bad}' "$out/database-audit.tsv" || failed=1 +for node in "${nodes[@]}"; do docker logs "$(container "$node")" 2>&1 | grep -c "panicked" > "$out/$node-panics.txt" || true; done +for id in $(dc --profile clients ps -a -q); do + docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} restarts={{.RestartCount}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}}' "$id" +done > "$out/container-states.txt" +log "audit: $(tr '\n' ' ' < "$out/database-audit.tsv")" +exit "$failed" diff --git a/scripts/perf/server-port-multinode.compose.yml b/scripts/perf/server-port-multinode.compose.yml new file mode 100644 index 000000000..3e78e280a --- /dev/null +++ b/scripts/perf/server-port-multinode.compose.yml @@ -0,0 +1,123 @@ +# Multi-node native qualification: three native nodes share one database +# behind a load balancer, while run-server-port-multinode.sh kills and +# restarts them. Choose exactly one database profile: mysql, postgres or sqlite. +x-native: &native + stop_signal: SIGTERM + image: &php_image durableworkflow/server@sha256:944c384c9875fd82637dd2bdea74dcb881cf983a57599ba0e492bec706296549 + platform: linux/amd64 + user: "1000:1000" + entrypoint: [/native/server] + environment: + DW_RUST_EXPERIMENTAL: "1" + DW_BIND_ADDRESS: 0.0.0.0:8080 + DW_AUTH_DRIVER: token + DW_AUTH_TOKEN: multinode-test-token + DW_WORKFLOW_TASK_TIMEOUT: "${DW_MN_WORKFLOW_TASK_TIMEOUT:-60}" + DB_CONNECTION: "${DW_MN_DB_CONNECTION:?Set the database connection}" + DB_HOST: "${DW_MN_DB_HOST:-}" + DB_DATABASE: "${DW_MN_DB_DATABASE:?Set the database}" + DB_USERNAME: "${DW_MN_DB_USERNAME:-}" + DB_PASSWORD: "${DW_MN_DB_PASSWORD:-}" + volumes: + - "${DW_PORT_NATIVE_BINARY:?Set the absolute native binary path}:/native/server:ro" + - "${DW_MN_STATE_DIR:?Set the shared state directory}:/state" + cpus: 1 + mem_limit: 512m + memswap_limit: 512m + restart: "no" + healthcheck: + test: [CMD, curl, --fail, --silent, http://127.0.0.1:8080/api/ready] + interval: 2s + timeout: 3s + retries: 30 + +x-client: &client + image: *php_image + platform: linux/amd64 + user: "1000:1000" + entrypoint: [php] + working_dir: /source + environment: &client_environment + DURABLE_WORKFLOW_RUNTIME_URL: http://lb:8080 + DURABLE_WORKFLOW_NAMESPACE: default + DURABLE_WORKFLOW_TASK_QUEUE: multinode + DURABLE_WORKFLOW_TOKEN: multinode-test-token + DURABLE_WORKFLOW_CAPACITY_CELL: one-activity + DURABLE_WORKFLOW_WORKER_CONCURRENCY: "1" + volumes: + - "${DW_PORT_SOURCE_ROOT:?Set the absolute source directory}:/source:ro" + - "${DW_PORT_SDK_ROOT:?Set the normally installed SDK directory}/vendor:/source/benchmarks/capacity/v1/bindings/php/vendor:ro" + cpus: 0.5 + mem_limit: 512m + memswap_limit: 512m + restart: "no" + +services: + mysql: + profiles: [mysql] + image: mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 + platform: linux/amd64 + environment: + MYSQL_DATABASE: durable_workflow + MYSQL_USER: workflow + MYSQL_PASSWORD: multinode-mysql + MYSQL_ROOT_PASSWORD: multinode-root + command: ["--innodb-flush-log-at-trx-commit=1", "--sync-binlog=1"] + cpus: 1 + mem_limit: 1024m + memswap_limit: 1024m + restart: "no" + healthcheck: + test: [CMD-SHELL, "MYSQL_PWD=multinode-mysql mysqladmin ping --silent -h 127.0.0.1 -u workflow"] + interval: 2s + timeout: 3s + retries: 30 + + postgres: + profiles: [postgres] + image: mirror.gcr.io/library/postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 + environment: + POSTGRES_DB: durable_workflow + POSTGRES_USER: workflow + POSTGRES_PASSWORD: multinode-postgres + cpus: 1 + mem_limit: 1024m + memswap_limit: 1024m + restart: "no" + healthcheck: + test: [CMD-SHELL, "pg_isready -U workflow -d durable_workflow"] + interval: 2s + timeout: 3s + retries: 30 + + native-1: *native + native-2: *native + native-3: *native + + lb: + image: mirror.gcr.io/library/haproxy@sha256:${DW_MN_HAPROXY_DIGEST:?Set the pinned HAProxy digest} + user: "1000:1000" + volumes: + - "${DW_PORT_SOURCE_ROOT}/scripts/perf/multinode-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro" + cpus: 0.5 + mem_limit: 128m + memswap_limit: 128m + restart: "no" + + sdk-worker-1: + <<: *client + profiles: [clients] + # A supervisor restarts a worker that exits; the run records each restart. + restart: on-failure + stop_signal: SIGTERM + command: [benchmarks/capacity/v1/bindings/php/capacity_adapter.php, worker] + sdk-worker-2: + <<: *client + profiles: [clients] + # A supervisor restarts a worker that exits; the run records each restart. + restart: on-failure + stop_signal: SIGTERM + command: [benchmarks/capacity/v1/bindings/php/capacity_adapter.php, worker] + probe: + <<: *client + profiles: [clients] From 7230d941fceb238ce4db6fd3b8c8bcf8fe1ac199 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 17:32:06 +0000 Subject: [PATCH 2/6] Record the multi-node results on MySQL, PostgreSQL and SQLite All 810 offered workflows completed exactly once while nodes were killed and restarted and a worker was paused past its lease. The audits found no duplicate outcome or history sequence and no node panicked. The paused worker exited 7 times: 6 on native's lost-claim refusals, which lacked the fields PHP's ownership guard returns, and once on SDK 2.0.10's poll handling of a proxy 503. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/multinode/README.md | 4 + .../multinode/results-2026-10-20.md | 57 ++++++++ .../multinode/summary-mysql-2026-10-20.json | 127 ++++++++++++++++++ .../summary-postgres-2026-10-20.json | 126 +++++++++++++++++ .../multinode/summary-sqlite-2026-10-20.json | 121 +++++++++++++++++ 5 files changed, 435 insertions(+) create mode 100644 benchmarks/server-port/multinode/results-2026-10-20.md create mode 100644 benchmarks/server-port/multinode/summary-mysql-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-postgres-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json diff --git a/benchmarks/server-port/multinode/README.md b/benchmarks/server-port/multinode/README.md index f875eed92..4d1d35919 100644 --- a/benchmarks/server-port/multinode/README.md +++ b/benchmarks/server-port/multinode/README.md @@ -75,3 +75,7 @@ done `--smoke` runs one 20-second window to check the setup. It is never a qualification. + +## Results + +- [2026-10-20](results-2026-10-20.md): MySQL, PostgreSQL and SQLite passed. diff --git a/benchmarks/server-port/multinode/results-2026-10-20.md b/benchmarks/server-port/multinode/results-2026-10-20.md new file mode 100644 index 000000000..1870be4d7 --- /dev/null +++ b/benchmarks/server-port/multinode/results-2026-10-20.md @@ -0,0 +1,57 @@ +# Multi-node results, 2026-10-20 + +The full schedule passed on MySQL 8.4, PostgreSQL 17 and SQLite. Every one of +the 810 offered workflows completed exactly once, with its original output +and an ordered history, while server nodes were killed and restarted and a +worker was paused past its lease. The database audits found no duplicate +committed outcome and no duplicate history sequence. No native node panicked. + +| Database | Workflows verified | Node kills | Worker pauses | Duplicate outcomes | Stale workflow completions refused | Paused-worker exits | +| --- | --- | --- | --- | --- | --- | --- | +| MySQL 8.4 | 270 of 270 | 26 | 3 | 0 | 1 | 2 | +| PostgreSQL 17 | 270 of 270 | 26 | 3 | 0 | 0 | 2 | +| SQLite | 270 of 270 | 27 | 3 | 0 | 0 | 3 | + +Each workflow's activity ran once. No activity completion was lost to a +killed node, so none waited for the five-minute activity lease. + +## Identity + +- Harness: `68d78be19`, run from a clean detached checkout. +- Native binary: built from `rust-325-workflow-lease` `ce18934e1`, SHA-256 + `53d634e8fb7d96cab64d768e01d47db6e8c2f5a634cf0d6e67e6047b95ac03a9`. +- Clients: the capacity bindings' locked PHP SDK 2.0.10. +- Machine: one Intel i5-6500 host with 4 logical CPUs and no other workload. +- Raw node, balancer and worker logs were inspected and not kept. Each + database's summary is in `summary--2026-10-20.json`. + +## Worker exits + +The paused worker, `sdk-worker-1`, exited and was restarted 7 times in all. +The other worker never exited. + +- **Native refusal shape (6 exits).** After a pause, the worker sent requests + for workflow tasks the server had given to the other worker. Native refused + the late lease renewal (`task_not_leased`, 5 times) or late completion + (`conflicting_completion`, once) with only a reason and the task fence. + Published PHP refuses the same + requests with the ownership guard's fields: the task status, and whether + the run is closed. A current worker uses those fields to recognize a lost + claim. [#548](https://github.com/durable-workflow/server/pull/548) makes + native return PHP's bodies, with a parity fixture. +- **SDK 2.0.10 poll (1 exit).** A poll got HAProxy's HTML 503 while a killed + node was still marked up. SDK 2.0.10 treats that body as invalid JSON and + exits. The SDK has retried polls on that failure since + [sdk-php#69](https://github.com/durable-workflow/sdk-php/issues/69). + +Current SDKs still exit when the same proxy failure hits a request for a +task that is already claimed: +[sdk-php#197](https://github.com/durable-workflow/sdk-php/issues/197). + +## Earlier attempt + +The first full MySQL run had no balancer retry and no worker restarts. Both +workers exited on HAProxy 503 pages while killed nodes were still marked up, +and window 2 completed nothing. That run measured the clients rather than the +server. It led to the connect-failure retry and the restart policy in the +current harness. diff --git a/benchmarks/server-port/multinode/summary-mysql-2026-10-20.json b/benchmarks/server-port/multinode/summary-mysql-2026-10-20.json new file mode 100644 index 000000000..996097666 --- /dev/null +++ b/benchmarks/server-port/multinode/summary-mysql-2026-10-20.json @@ -0,0 +1,127 @@ +{ + "identity": { + "database": "mysql", + "compose_project": "dw-mn-mysql-20261020t170034", + "source_revision": "68d78be19c445b22e4cd95a404c50c4f3f52f271", + "source_modified_files": 0, + "native_binary_sha256": "53d634e8fb7d96cab64d768e01d47db6e8c2f5a634cf0d6e67e6047b95ac03a9", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + }, + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + }, + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 26, + "pause": 3, + "start": 26, + "unpause": 3 + }, + "balancer": { + "requests": 4044, + "workflow_task_completions_409": 1, + "activity_task_completions_409": 0, + "responses_5xx": 1 + }, + "database_audit": { + "runs completed": 270, + "tasks workflow completed": 540, + "tasks activity completed": 270, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "mysql": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 2, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: conflicting_completion": 1, + "sdk-worker-1: Uncaught JsonException: ": 1 + } +} diff --git a/benchmarks/server-port/multinode/summary-postgres-2026-10-20.json b/benchmarks/server-port/multinode/summary-postgres-2026-10-20.json new file mode 100644 index 000000000..105e5a94d --- /dev/null +++ b/benchmarks/server-port/multinode/summary-postgres-2026-10-20.json @@ -0,0 +1,126 @@ +{ + "identity": { + "database": "postgres", + "compose_project": "dw-mn-postgres-20261020t171122", + "source_revision": "68d78be19c445b22e4cd95a404c50c4f3f52f271", + "source_modified_files": 0, + "native_binary_sha256": "53d634e8fb7d96cab64d768e01d47db6e8c2f5a634cf0d6e67e6047b95ac03a9", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + }, + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + }, + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 26, + "pause": 3, + "start": 26, + "unpause": 3 + }, + "balancer": { + "requests": 3987, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks workflow completed": 540, + "tasks activity completed": 270, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "postgres": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 2, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: task_not_leased": 2 + } +} diff --git a/benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json b/benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json new file mode 100644 index 000000000..27a8564ca --- /dev/null +++ b/benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json @@ -0,0 +1,121 @@ +{ + "identity": { + "database": "sqlite", + "compose_project": "dw-mn-sqlite-20261020t172123", + "source_revision": "68d78be19c445b22e4cd95a404c50c4f3f52f271", + "source_modified_files": 0, + "native_binary_sha256": "53d634e8fb7d96cab64d768e01d47db6e8c2f5a634cf0d6e67e6047b95ac03a9", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + }, + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + }, + { + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 27, + "pause": 3, + "start": 27, + "unpause": 3 + }, + "balancer": { + "requests": 3990, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks activity completed": 270, + "tasks workflow completed": 540, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 3, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: task_not_leased": 3 + } +} From 9898f79cd1e66eb6852b55fbbe31c2e5c60582b0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 18:21:13 +0000 Subject: [PATCH 3/6] Generate the multi-node database secrets per run and tidy the harness The compose file no longer holds fixed database passwords: the driver generates both secrets for each run. The workload throws a dedicated verification exception, loads the capacity adapter once, and the driver's case statements and helpers handle unexpected values and name their arguments. MySQL and SQLite smoke runs passed after the change. Co-Authored-By: Claude Opus 5.5 --- scripts/perf/multinode-workload.php | 19 ++++++++++++------- scripts/perf/run-server-port-multinode.sh | 15 +++++++++++---- .../perf/server-port-multinode.compose.yml | 10 ++++++---- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/scripts/perf/multinode-workload.php b/scripts/perf/multinode-workload.php index 3d5b14288..36da8e661 100644 --- a/scripts/perf/multinode-workload.php +++ b/scripts/perf/multinode-workload.php @@ -13,9 +13,14 @@ use Composer\InstalledVersions; use DurableWorkflow\Exception\ServerException; -require dirname(__DIR__, 2).'/benchmarks/capacity/v1/bindings/php/capacity_adapter.php'; +require_once dirname(__DIR__, 2).'/benchmarks/capacity/v1/bindings/php/capacity_adapter.php'; capacityAutoload(); +// A completed workflow whose output or history breaks the qualification. +final class MultinodeVerificationFailure extends RuntimeException +{ +} + $rate = filter_var($argv[1] ?? null, FILTER_VALIDATE_FLOAT, ['options' => ['min_range' => 0.1, 'max_range' => 5.0]]); $duration = filter_var($argv[2] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 5, 'max_range' => 600]]); $drain = filter_var($argv[3] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 10, 'max_range' => 600]]); @@ -117,33 +122,33 @@ foreach ($completed as $id => $execution) { try { if ($execution->output !== $blob) { - throw new RuntimeException('output differs'); + throw new MultinodeVerificationFailure('output differs'); } $history = $retrying('history', static fn () => $client->workflowHistory($id, $execution->runId), microtime(true) + 60); $events = $history['events'] ?? []; $sequences = array_column($events, 'sequence'); if ($sequences !== range(1, count($events))) { - throw new RuntimeException('history sequences are not contiguous'); + throw new MultinodeVerificationFailure('history sequences are not contiguous'); } $types = array_column($events, 'event_type'); $counts = array_count_values($types); $allowed = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', 'ActivityCompleted', 'WorkflowCompleted']; if (($unexpected = array_diff(array_keys($counts), $allowed)) !== []) { - throw new RuntimeException('unexpected events '.implode(',', $unexpected)); + throw new MultinodeVerificationFailure('unexpected events '.implode(',', $unexpected)); } foreach (['WorkflowStarted', 'ActivityScheduled', 'ActivityCompleted', 'WorkflowCompleted'] as $once) { if (($counts[$once] ?? 0) !== 1) { - throw new RuntimeException($once.' recorded '.($counts[$once] ?? 0).' times'); + throw new MultinodeVerificationFailure($once.' recorded '.($counts[$once] ?? 0).' times'); } } $starts = array_values(array_filter($events, static fn (array $e): bool => $e['event_type'] === 'ActivityStarted')); $completion = array_values(array_filter($events, static fn (array $e): bool => $e['event_type'] === 'ActivityCompleted'))[0]; if ($starts === [] || end($starts)['payload']['activity_attempt_id'] !== $completion['payload']['activity_attempt_id']) { - throw new RuntimeException('the completed activity attempt is not the last one started'); + throw new MultinodeVerificationFailure('the completed activity attempt is not the last one started'); } $order = array_values(array_intersect($types, ['WorkflowStarted', 'ActivityScheduled', 'ActivityCompleted', 'WorkflowCompleted'])); if ($order !== ['WorkflowStarted', 'ActivityScheduled', 'ActivityCompleted', 'WorkflowCompleted']) { - throw new RuntimeException('events out of order'); + throw new MultinodeVerificationFailure('events out of order'); } $attemptsByWorkflow[count($starts)] = ($attemptsByWorkflow[count($starts)] ?? 0) + 1; $verified++; diff --git a/scripts/perf/run-server-port-multinode.sh b/scripts/perf/run-server-port-multinode.sh index b4241282e..24d444682 100755 --- a/scripts/perf/run-server-port-multinode.sh +++ b/scripts/perf/run-server-port-multinode.sh @@ -47,17 +47,23 @@ export DW_MN_HAPROXY_DIGEST=${DW_MN_HAPROXY_DIGEST:-094401c230b9bf01a6a4dc96e455 export DW_MN_STATE_DIR="$out/state" mkdir -p "$DW_MN_STATE_DIR" chmod 0777 "$DW_MN_STATE_DIR" +# Each run gets its own database secrets; the compose file requires them even +# for profiles it does not start. +secret() { od -An -N16 -tx1 /dev/urandom | tr -d ' \n'; } +DW_MN_DB_PASSWORD=$(secret) DW_MN_DB_ROOT_PASSWORD=$(secret) +export DW_MN_DB_PASSWORD DW_MN_DB_ROOT_PASSWORD case "$db" in mysql) export DW_MN_DB_CONNECTION=mysql DW_MN_DB_HOST=mysql DW_MN_DB_DATABASE=durable_workflow \ - DW_MN_DB_USERNAME=workflow DW_MN_DB_PASSWORD=multinode-mysql; database=(mysql) ;; + DW_MN_DB_USERNAME=workflow; database=(mysql) ;; postgres) export DW_MN_DB_CONNECTION=pgsql DW_MN_DB_HOST=postgres DW_MN_DB_DATABASE=durable_workflow \ - DW_MN_DB_USERNAME=workflow DW_MN_DB_PASSWORD=multinode-postgres; database=(postgres) ;; + DW_MN_DB_USERNAME=workflow; database=(postgres) ;; sqlite) export DW_MN_DB_CONNECTION=sqlite DW_MN_DB_DATABASE=/state/native.sqlite; database=() ;; + *) echo "Choose mysql, postgres or sqlite." >&2; exit 2 ;; esac project="dw-mn-$db-$(date -u +%Y%m%dt%H%M%S)" dc() { docker compose -p "$project" -f "$compose" --profile "$db" "$@"; } -container() { dc --profile clients ps -a -q "$1"; } +container() { local service=$1; dc --profile clients ps -a -q "$service"; } nodes=(native-1 native-2 native-3) stop_file="$out/.stop" @@ -155,7 +161,7 @@ docker unpause "$(container sdk-worker-1)" >/dev/null 2>&1 || true # The balancer's request log shows what the chaos did: stale completions the # servers refused, and requests a killed node never answered. docker logs "$(container lb)" > "$out/lb.log" 2>&1 || true -count() { grep -E "$1" "$out/lb.log" | grep -cE "/[0-9-]+ $2 " || true; } +count() { local path=$1 status=$2; grep -E "$path" "$out/lb.log" | grep -cE "/[0-9-]+ $status " || true; } { printf 'requests\t%s\n' "$(grep -cE '"[A-Z]+ /api/' "$out/lb.log" || true)" printf 'workflow_task_completions_409\t%s\n' "$(count '/api/worker/workflow-tasks/[^ ]+/complete HTTP' 409)" @@ -177,6 +183,7 @@ case "$db" in foreach (array_filter(array_map("trim", explode(";", getenv("AUDIT_SQL")))) as $sql) { foreach ($pdo->query($sql, PDO::FETCH_NUM) as $row) { echo implode("\t", $row), "\n"; } }' ;; + *) echo "Unknown database $db." >&2; exit 2 ;; esac > "$out/database-audit.tsv" 2>> "$out/run.log" for kind in workflow activity; do awk -F'\t' -v k="duplicate_${kind}_completions" '$1==k && $2!=0 {bad=1} END {exit bad}' "$out/database-audit.tsv" || failed=1 diff --git a/scripts/perf/server-port-multinode.compose.yml b/scripts/perf/server-port-multinode.compose.yml index 3e78e280a..2c103e9cc 100644 --- a/scripts/perf/server-port-multinode.compose.yml +++ b/scripts/perf/server-port-multinode.compose.yml @@ -60,15 +60,16 @@ services: environment: MYSQL_DATABASE: durable_workflow MYSQL_USER: workflow - MYSQL_PASSWORD: multinode-mysql - MYSQL_ROOT_PASSWORD: multinode-root + # The driver generates both secrets for each run. + MYSQL_PASSWORD: ${DW_MN_DB_PASSWORD:?Run through run-server-port-multinode.sh} + MYSQL_ROOT_PASSWORD: ${DW_MN_DB_ROOT_PASSWORD:?Run through run-server-port-multinode.sh} command: ["--innodb-flush-log-at-trx-commit=1", "--sync-binlog=1"] cpus: 1 mem_limit: 1024m memswap_limit: 1024m restart: "no" healthcheck: - test: [CMD-SHELL, "MYSQL_PWD=multinode-mysql mysqladmin ping --silent -h 127.0.0.1 -u workflow"] + test: [CMD-SHELL, "MYSQL_PWD=$$MYSQL_PASSWORD mysqladmin ping --silent -h 127.0.0.1 -u workflow"] interval: 2s timeout: 3s retries: 30 @@ -79,7 +80,8 @@ services: environment: POSTGRES_DB: durable_workflow POSTGRES_USER: workflow - POSTGRES_PASSWORD: multinode-postgres + # The driver generates the secret for each run. + POSTGRES_PASSWORD: ${DW_MN_DB_PASSWORD:?Run through run-server-port-multinode.sh} cpus: 1 mem_limit: 1024m memswap_limit: 1024m From eeb510a3a59b18e0ad530e0a837325ac1d84d4fe Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 18:28:28 +0000 Subject: [PATCH 4/6] Add a MariaDB 10.11 profile to the multi-node qualification The shared fixtures qualify MariaDB 10.11 separately from MySQL, so the multi-node harness runs on it too, with the same image digest. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/multinode/README.md | 8 +++---- scripts/perf/run-server-port-multinode.sh | 9 ++++--- .../perf/server-port-multinode.compose.yml | 24 ++++++++++++++++++- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/benchmarks/server-port/multinode/README.md b/benchmarks/server-port/multinode/README.md index 4d1d35919..81eb032bc 100644 --- a/benchmarks/server-port/multinode/README.md +++ b/benchmarks/server-port/multinode/README.md @@ -11,8 +11,8 @@ measurement. [`server-port-multinode.compose.yml`](../../../scripts/perf/server-port-multinode.compose.yml) runs these services: -- **Database.** One of MySQL 8.4, PostgreSQL 17 or SQLite. With SQLite, the - three nodes share one database file on one host. +- **Database.** One of MySQL 8.4, MariaDB 10.11, PostgreSQL 17 or SQLite. + With SQLite, the three nodes share one database file on one host. - **Native nodes.** Three nodes with the same binary and configuration, one CPU and 512 MiB each. All three start together against the empty database, so bootstrap races too. @@ -31,7 +31,7 @@ runs these services: ## Procedure [`run-server-port-multinode.sh`](../../../scripts/perf/run-server-port-multinode.sh) -`OUTPUT_DIR mysql|postgres|sqlite` offers one-activity workflows at +`OUTPUT_DIR mysql|mariadb|postgres|sqlite` offers one-activity workflows at 0.5/s in three 180-second windows. Each window drains for up to 420 s, longer than the five-minute activity lease: an activity whose completion a killed node dropped is redelivered only when that lease expires. Two @@ -68,7 +68,7 @@ for panics. ```sh # The locked SDK and an optimized binary, as the v2 README describes. export DW_PORT_SOURCE_ROOT=/abs/source DW_PORT_SDK_ROOT=/abs/sdk DW_PORT_NATIVE_BINARY=/abs/native/server -for db in mysql postgres sqlite; do +for db in mysql mariadb postgres sqlite; do bash scripts/perf/run-server-port-multinode.sh "results/$db" "$db" done ``` diff --git a/scripts/perf/run-server-port-multinode.sh b/scripts/perf/run-server-port-multinode.sh index 24d444682..93fd36c4a 100755 --- a/scripts/perf/run-server-port-multinode.sh +++ b/scripts/perf/run-server-port-multinode.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Multi-node native kill/restart qualification on one database family. # -# run-server-port-multinode.sh OUTPUT_DIR mysql|postgres|sqlite [--smoke] +# run-server-port-multinode.sh OUTPUT_DIR mysql|mariadb|postgres|sqlite [--smoke] # # Three native nodes share one database behind HAProxy. Two published-SDK # workers and the offered workload reach them only through the balancer. @@ -18,7 +18,7 @@ set -Eeuo pipefail out=$1 db=$2 mode=${3:-} -case "$db" in mysql|postgres|sqlite) ;; *) echo "Choose mysql, postgres or sqlite." >&2; exit 2 ;; esac +case "$db" in mysql|mariadb|postgres|sqlite) ;; *) echo "Choose mysql, mariadb, postgres or sqlite." >&2; exit 2 ;; esac case "$mode" in ""|--smoke) ;; *) echo "Unknown option $mode." >&2; exit 2 ;; esac : "${DW_PORT_SOURCE_ROOT:?Set the absolute source directory}" : "${DW_PORT_SDK_ROOT:?Set the normally installed SDK directory}" @@ -55,10 +55,12 @@ export DW_MN_DB_PASSWORD DW_MN_DB_ROOT_PASSWORD case "$db" in mysql) export DW_MN_DB_CONNECTION=mysql DW_MN_DB_HOST=mysql DW_MN_DB_DATABASE=durable_workflow \ DW_MN_DB_USERNAME=workflow; database=(mysql) ;; + mariadb) export DW_MN_DB_CONNECTION=mariadb DW_MN_DB_HOST=mariadb DW_MN_DB_DATABASE=durable_workflow \ + DW_MN_DB_USERNAME=workflow; database=(mariadb) ;; postgres) export DW_MN_DB_CONNECTION=pgsql DW_MN_DB_HOST=postgres DW_MN_DB_DATABASE=durable_workflow \ DW_MN_DB_USERNAME=workflow; database=(postgres) ;; sqlite) export DW_MN_DB_CONNECTION=sqlite DW_MN_DB_DATABASE=/state/native.sqlite; database=() ;; - *) echo "Choose mysql, postgres or sqlite." >&2; exit 2 ;; + *) echo "Choose mysql, mariadb, postgres or sqlite." >&2; exit 2 ;; esac project="dw-mn-$db-$(date -u +%Y%m%dt%H%M%S)" @@ -177,6 +179,7 @@ SELECT 'duplicate_activity_completions', COUNT(*) FROM (SELECT activity_executio SELECT 'duplicate_history_sequences', COUNT(*) FROM (SELECT workflow_run_id, sequence FROM workflow_history_events GROUP BY workflow_run_id, sequence HAVING COUNT(*) > 1) d;" case "$db" in mysql) dc exec -T mysql sh -c 'MYSQL_PWD=$MYSQL_PASSWORD exec mysql -u"$MYSQL_USER" "$MYSQL_DATABASE" --batch --skip-column-names -e "$1"' audit "$audit_sql" ;; + mariadb) dc exec -T mariadb sh -c 'MYSQL_PWD=$MARIADB_PASSWORD exec mariadb -u"$MARIADB_USER" "$MARIADB_DATABASE" --batch --skip-column-names -e "$1"' audit "$audit_sql" ;; postgres) dc exec -T postgres psql -U workflow -d durable_workflow -At -F $'\t' -c "$audit_sql" ;; sqlite) dc --profile clients run --rm --no-deps -v "$DW_MN_STATE_DIR:/state" -e AUDIT_SQL="$audit_sql" probe -r ' $pdo = new PDO("sqlite:/state/native.sqlite"); diff --git a/scripts/perf/server-port-multinode.compose.yml b/scripts/perf/server-port-multinode.compose.yml index 2c103e9cc..f2158fda3 100644 --- a/scripts/perf/server-port-multinode.compose.yml +++ b/scripts/perf/server-port-multinode.compose.yml @@ -1,6 +1,7 @@ # Multi-node native qualification: three native nodes share one database # behind a load balancer, while run-server-port-multinode.sh kills and -# restarts them. Choose exactly one database profile: mysql, postgres or sqlite. +# restarts them. Choose exactly one database profile: mysql, mariadb, postgres or +# sqlite. x-native: &native stop_signal: SIGTERM image: &php_image durableworkflow/server@sha256:944c384c9875fd82637dd2bdea74dcb881cf983a57599ba0e492bec706296549 @@ -74,6 +75,27 @@ services: timeout: 3s retries: 30 + mariadb: + profiles: [mariadb] + # The digest the shared fixtures' MariaDB 10.11 jobs use. + image: mirror.gcr.io/library/mariadb@sha256:7db29378d4fdab73f8123bbc2b48905c90d1a4b00cf848b028f1e81e623257f2 + environment: + MARIADB_DATABASE: durable_workflow + MARIADB_USER: workflow + # The driver generates both secrets for each run. + MARIADB_PASSWORD: ${DW_MN_DB_PASSWORD:?Run through run-server-port-multinode.sh} + MARIADB_ROOT_PASSWORD: ${DW_MN_DB_ROOT_PASSWORD:?Run through run-server-port-multinode.sh} + command: ["--innodb-flush-log-at-trx-commit=1", "--sync-binlog=1"] + cpus: 1 + mem_limit: 1024m + memswap_limit: 1024m + restart: "no" + healthcheck: + test: [CMD, healthcheck.sh, --connect, --innodb_initialized] + interval: 2s + timeout: 3s + retries: 30 + postgres: profiles: [postgres] image: mirror.gcr.io/library/postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 From 6b676949af70caf4f5b685d0466ccf278c1577c0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 20:31:49 +0000 Subject: [PATCH 5/6] Accept and count the server's expired-lease recovery in multi-node histories A current SDK worker that dies holding a workflow task lease leaves the task to the server's expired-lease recovery, which records RepairRequested, as published PHP's does. The workload accepted only the happy-path events and flagged that recovered workflow. It now accepts RepairRequested and reports the recoveries per workflow. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/multinode/README.md | 7 +++++-- scripts/perf/multinode-workload.php | 10 +++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/benchmarks/server-port/multinode/README.md b/benchmarks/server-port/multinode/README.md index 81eb032bc..a70775f93 100644 --- a/benchmarks/server-port/multinode/README.md +++ b/benchmarks/server-port/multinode/README.md @@ -52,7 +52,10 @@ Each window passes only if every planned workflow completed, with: - contiguous history sequences; - exactly one `WorkflowStarted`, `ActivityScheduled`, `ActivityCompleted` and `WorkflowCompleted`, in that order; -- an `ActivityCompleted` from the last attempt that started. +- an `ActivityCompleted` from the last attempt that started; +- no other events except `ActivityStarted` and `RepairRequested`. The server + records `RepairRequested` when it recovers the expired lease of a worker + that died holding a workflow task; the workload counts these recoveries. Afterwards the database audit must find: @@ -78,4 +81,4 @@ qualification. ## Results -- [2026-10-20](results-2026-10-20.md): MySQL, PostgreSQL and SQLite passed. +- [2026-10-20](results-2026-10-20.md): MySQL, MariaDB, PostgreSQL and SQLite passed. diff --git a/scripts/perf/multinode-workload.php b/scripts/perf/multinode-workload.php index 36da8e661..9829e7631 100644 --- a/scripts/perf/multinode-workload.php +++ b/scripts/perf/multinode-workload.php @@ -119,6 +119,7 @@ final class MultinodeVerificationFailure extends RuntimeException $verified = 0; $attemptsByWorkflow = []; +$repairedWorkflows = []; foreach ($completed as $id => $execution) { try { if ($execution->output !== $blob) { @@ -132,7 +133,10 @@ final class MultinodeVerificationFailure extends RuntimeException } $types = array_column($events, 'event_type'); $counts = array_count_values($types); - $allowed = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', 'ActivityCompleted', 'WorkflowCompleted']; + // A worker that dies holding a workflow task lease leaves the task to + // the server's expired-lease recovery, which records RepairRequested. + $allowed = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', 'ActivityCompleted', + 'WorkflowCompleted', 'RepairRequested']; if (($unexpected = array_diff(array_keys($counts), $allowed)) !== []) { throw new MultinodeVerificationFailure('unexpected events '.implode(',', $unexpected)); } @@ -151,6 +155,9 @@ final class MultinodeVerificationFailure extends RuntimeException throw new MultinodeVerificationFailure('events out of order'); } $attemptsByWorkflow[count($starts)] = ($attemptsByWorkflow[count($starts)] ?? 0) + 1; + if (($counts['RepairRequested'] ?? 0) > 0) { + $repairedWorkflows[$id] = $counts['RepairRequested']; + } $verified++; } catch (Throwable $error) { $errors[] = $id.': verification: '.$error->getMessage(); @@ -174,6 +181,7 @@ final class MultinodeVerificationFailure extends RuntimeException 'completed' => count($completed), 'verified' => $verified, 'activity_attempts_per_workflow' => $attemptsByWorkflow, + 'repairs_by_workflow' => (object) $repairedWorkflows, 'retries' => $retries, 'errors' => $errors, 'outcome' => $errors === [] && $verified === $planned ? 'pass' : 'fail', From 947f6fc055a8414a006b77242d09046215b4383f Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 20:59:49 +0000 Subject: [PATCH 6/6] Record MariaDB, the ownership-guard confirmation and a current SDK The multi-node schedule also passed on MariaDB 10.11. A confirmation run with the #548 binary shows every late request refused with PHP's ownership guard body; SDK 2.0.10's remaining exits are its own lack of lost-claim handling. With PHP SDK 2.5.3, all 1,080 workflows completed once on the four databases and no worker exited. Summaries now record the SDK version, and the record describes the concurrent bootstrap race found on the way. Co-Authored-By: Claude Opus 5.5 --- .../multinode/results-2026-10-20.md | 114 +++++++++++---- .../summary-guard-mariadb-2026-10-20.json | 132 +++++++++++++++++ .../summary-guard-mysql-2026-10-20.json | 137 ++++++++++++++++++ .../summary-guard-postgres-2026-10-20.json | 132 +++++++++++++++++ .../summary-guard-sqlite-2026-10-20.json | 127 ++++++++++++++++ .../multinode/summary-mariadb-2026-10-20.json | 134 +++++++++++++++++ .../multinode/summary-mysql-2026-10-20.json | 6 + .../summary-postgres-2026-10-20.json | 6 + .../summary-sdk253-mariadb-2026-10-20.json | 130 +++++++++++++++++ .../summary-sdk253-mysql-2026-10-20.json | 130 +++++++++++++++++ .../summary-sdk253-postgres-2026-10-20.json | 130 +++++++++++++++++ .../summary-sdk253-sqlite-2026-10-20.json | 125 ++++++++++++++++ .../multinode/summary-sqlite-2026-10-20.json | 6 + 13 files changed, 1284 insertions(+), 25 deletions(-) create mode 100644 benchmarks/server-port/multinode/summary-guard-mariadb-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-guard-mysql-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-guard-postgres-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-guard-sqlite-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-mariadb-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-sdk253-mariadb-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-sdk253-mysql-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-sdk253-postgres-2026-10-20.json create mode 100644 benchmarks/server-port/multinode/summary-sdk253-sqlite-2026-10-20.json diff --git a/benchmarks/server-port/multinode/results-2026-10-20.md b/benchmarks/server-port/multinode/results-2026-10-20.md index 1870be4d7..794e15716 100644 --- a/benchmarks/server-port/multinode/results-2026-10-20.md +++ b/benchmarks/server-port/multinode/results-2026-10-20.md @@ -1,14 +1,16 @@ # Multi-node results, 2026-10-20 -The full schedule passed on MySQL 8.4, PostgreSQL 17 and SQLite. Every one of -the 810 offered workflows completed exactly once, with its original output -and an ordered history, while server nodes were killed and restarted and a -worker was paused past its lease. The database audits found no duplicate -committed outcome and no duplicate history sequence. No native node panicked. +The full schedule passed on MySQL 8.4, MariaDB 10.11, PostgreSQL 17 and +SQLite. Every one of the 1,080 offered workflows completed exactly once, with +its original output and an ordered history, while server nodes were killed +and restarted and a worker was paused past its lease. The database audits +found no duplicate committed outcome and no duplicate history sequence. No +native node panicked. -| Database | Workflows verified | Node kills | Worker pauses | Duplicate outcomes | Stale workflow completions refused | Paused-worker exits | +| Database | Workflows verified | Node kills | Worker pauses | Duplicate outcomes | Stale workflow completions refused | Worker exits | | --- | --- | --- | --- | --- | --- | --- | | MySQL 8.4 | 270 of 270 | 26 | 3 | 0 | 1 | 2 | +| MariaDB 10.11 | 270 of 270 | 25 | 3 | 0 | 1 | 3 | | PostgreSQL 17 | 270 of 270 | 26 | 3 | 0 | 0 | 2 | | SQLite | 270 of 270 | 27 | 3 | 0 | 0 | 3 | @@ -17,29 +19,31 @@ killed node, so none waited for the five-minute activity lease. ## Identity -- Harness: `68d78be19`, run from a clean detached checkout. -- Native binary: built from `rust-325-workflow-lease` `ce18934e1`, SHA-256 - `53d634e8fb7d96cab64d768e01d47db6e8c2f5a634cf0d6e67e6047b95ac03a9`. -- Clients: the capacity bindings' locked PHP SDK 2.0.10. +- Harness: `68d78be19` (MariaDB: `eeb510a3a`, which adds its profile), each + run from a clean detached checkout. +- Native binary: an unoptimized dev-profile build of `rust-325-workflow-lease` + `ce18934e1`, SHA-256 + `53d634e8fb7d96cab64d768e01d47db6e8c2f5a634cf0d6e67e6047b95ac03a9`. This + checks correctness, not speed. +- Clients: the capacity bindings' locked PHP SDK 2.0.10. A later run used SDK 2.5.3 (below). - Machine: one Intel i5-6500 host with 4 logical CPUs and no other workload. -- Raw node, balancer and worker logs were inspected and not kept. Each - database's summary is in `summary--2026-10-20.json`. +- Raw node, balancer and worker logs were inspected and not kept. Each run's + summary is in `summary--2026-10-20.json`. ## Worker exits -The paused worker, `sdk-worker-1`, exited and was restarted 7 times in all. -The other worker never exited. - -- **Native refusal shape (6 exits).** After a pause, the worker sent requests - for workflow tasks the server had given to the other worker. Native refused - the late lease renewal (`task_not_leased`, 5 times) or late completion - (`conflicting_completion`, once) with only a reason and the task fence. - Published PHP refuses the same - requests with the ownership guard's fields: the task status, and whether - the run is closed. A current worker uses those fields to recognize a lost - claim. [#548](https://github.com/durable-workflow/server/pull/548) makes - native return PHP's bodies, with a parity fixture. -- **SDK 2.0.10 poll (1 exit).** A poll got HAProxy's HTML 503 while a killed +Workers exited and were restarted 10 times in all: + +- **Native refusal shape (8 exits, all by the paused worker).** After a pause, + the worker sent requests for workflow tasks the server had given to the + other worker. Native refused the late lease renewals (`task_not_leased`) and + completions (`conflicting_completion`) with only a reason and the task + fence. Published PHP refuses the same requests with its ownership guard's + fields: the task status, and whether the run is closed. A current worker + reads those fields to drop a lost claim. + [#548](https://github.com/durable-workflow/server/pull/548) makes native + return PHP's bodies, with a parity fixture. +- **SDK 2.0.10 poll (2 exits).** A poll got HAProxy's HTML 503 while a killed node was still marked up. SDK 2.0.10 treats that body as invalid JSON and exits. The SDK has retried polls on that failure since [sdk-php#69](https://github.com/durable-workflow/sdk-php/issues/69). @@ -48,6 +52,66 @@ Current SDKs still exit when the same proxy failure hits a request for a task that is already claimed: [sdk-php#197](https://github.com/durable-workflow/sdk-php/issues/197). +## Confirmation with the ownership guard + +The same schedule ran again with native built from +[#548](https://github.com/durable-workflow/server/pull/548) (`f43e6fae2`, +unoptimized dev profile, SHA-256 +`4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f`) and the +harness at `1a8add148` (MariaDB: `eeb510a3a`). Summaries are in +`summary-guard--2026-10-20.json`. + +| Database | Workflows verified | Node kills | Duplicate outcomes | Worker exits | +| --- | --- | --- | --- | --- | +| MySQL 8.4 | 270 of 270 | 27 | 0 | 4 | +| MariaDB 10.11 | 270 of 270 | 25 | 0 | 3 | +| PostgreSQL 17 | 270 of 270 | 27 | 0 | 3 | +| SQLite | 270 of 270 | 27 | 0 | 2 | + +No native node panicked. Every late request from the paused worker was now +refused with PHP's ownership guard body. Eleven exits followed a late lease +renewal after the run had closed, refused with `run_closed` and +`can_continue: false`, the body published PHP returns. SDK 2.0.10 has no +lost-claim handling (the SDK added it in 2.2.6), so it exits on that refusal +from either server. The twelfth exit was the SDK 2.0.10 poll case above. + +## Current SDK + +The schedule ran a third time with the published PHP SDK 2.5.3 installed in +place of the bindings' locked 2.0.10, and native from #548 as above. Summaries +are in `summary-sdk253--2026-10-20.json`. + +| Database | Workflows verified | Node kills | Duplicate outcomes | Worker exits | +| --- | --- | --- | --- | --- | +| MySQL 8.4 | 270 of 270 | 25 | 0 | 0 | +| MariaDB 10.11 | 270 of 270 | 26 | 0 | 0 | +| PostgreSQL 17 | 270 of 270 | 24 | 0 | 0 | +| SQLite | 270 of 270 | 25 | 0 | 0 | + +With a current SDK, the workers stayed up through every node kill and pause. +Across the four runs the paused worker got 7 lost-claim refusals from native +(`409` on a late lease renewal or completion) and dropped each claim without +exiting. + +An earlier PostgreSQL run with the same SDK had one worker exit. A node was +killed while answering that worker's lease renewal, and the balancer returned +502, which the SDK does not retry while a task is claimed +([sdk-php#197](https://github.com/durable-workflow/sdk-php/issues/197)). The +exited worker still held its lease. Native recovered the expired lease as PHP +does, recording `RepairRequested`, and the workflow completed once. The +workload then accepted only the happy-path events and flagged that history. +It now accepts and counts the recovery, and the PostgreSQL row above is the +rerun. + +## Concurrent bootstrap + +All three nodes of each run start together against the empty database. In a +later smoke run on MySQL, one node exited with `unsupported_rust_schema`: it +started while another node was between creating the native marker table and +recording its marker. MySQL and MariaDB commit DDL on their own, so that state +is visible. The correction lets the node wait for the bootstrap lock, and its +kill qualification covers the window. + ## Earlier attempt The first full MySQL run had no balancer retry and no worker restarts. Both diff --git a/benchmarks/server-port/multinode/summary-guard-mariadb-2026-10-20.json b/benchmarks/server-port/multinode/summary-guard-mariadb-2026-10-20.json new file mode 100644 index 000000000..31c1ecae5 --- /dev/null +++ b/benchmarks/server-port/multinode/summary-guard-mariadb-2026-10-20.json @@ -0,0 +1,132 @@ +{ + "identity": { + "database": "mariadb", + "compose_project": "dw-mn-mariadb-20261020t190725", + "source_revision": "eeb510a3a59b18e0ad530e0a837325ac1d84d4fe", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 25, + "pause": 3, + "start": 25, + "unpause": 3 + }, + "balancer": { + "requests": 3978, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks activity completed": 270, + "tasks workflow completed": 540, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "mariadb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 3, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: ": 3 + } +} diff --git a/benchmarks/server-port/multinode/summary-guard-mysql-2026-10-20.json b/benchmarks/server-port/multinode/summary-guard-mysql-2026-10-20.json new file mode 100644 index 000000000..37adf4c45 --- /dev/null +++ b/benchmarks/server-port/multinode/summary-guard-mysql-2026-10-20.json @@ -0,0 +1,137 @@ +{ + "identity": { + "database": "mysql", + "compose_project": "dw-mn-mysql-20261020t182603", + "source_revision": "1a8add148c293f72243b9dbbe44845f0675d93f9", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": { + "start": { + "http_0": 1 + } + }, + "errors": [] + } + ], + "chaos": { + "kill": 27, + "pause": 3, + "start": 27, + "unpause": 3 + }, + "balancer": { + "requests": 4009, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 2 + }, + "database_audit": { + "runs completed": 270, + "tasks workflow completed": 540, + "tasks activity completed": 270, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "mysql": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 4, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: ": 3, + "sdk-worker-1: Uncaught JsonException: ": 1 + } +} diff --git a/benchmarks/server-port/multinode/summary-guard-postgres-2026-10-20.json b/benchmarks/server-port/multinode/summary-guard-postgres-2026-10-20.json new file mode 100644 index 000000000..95431d0e1 --- /dev/null +++ b/benchmarks/server-port/multinode/summary-guard-postgres-2026-10-20.json @@ -0,0 +1,132 @@ +{ + "identity": { + "database": "postgres", + "compose_project": "dw-mn-postgres-20261020t183624", + "source_revision": "1a8add148c293f72243b9dbbe44845f0675d93f9", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 27, + "pause": 3, + "start": 27, + "unpause": 3 + }, + "balancer": { + "requests": 3983, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks workflow completed": 540, + "tasks activity completed": 270, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "postgres": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 3, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: ": 3 + } +} diff --git a/benchmarks/server-port/multinode/summary-guard-sqlite-2026-10-20.json b/benchmarks/server-port/multinode/summary-guard-sqlite-2026-10-20.json new file mode 100644 index 000000000..36cc13ed6 --- /dev/null +++ b/benchmarks/server-port/multinode/summary-guard-sqlite-2026-10-20.json @@ -0,0 +1,127 @@ +{ + "identity": { + "database": "sqlite", + "compose_project": "dw-mn-sqlite-20261020t184615", + "source_revision": "1a8add148c293f72243b9dbbe44845f0675d93f9", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 27, + "pause": 3, + "start": 27, + "unpause": 3 + }, + "balancer": { + "requests": 3988, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks activity completed": 270, + "tasks workflow completed": 540, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 2, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: ": 2 + } +} diff --git a/benchmarks/server-port/multinode/summary-mariadb-2026-10-20.json b/benchmarks/server-port/multinode/summary-mariadb-2026-10-20.json new file mode 100644 index 000000000..75959bb5f --- /dev/null +++ b/benchmarks/server-port/multinode/summary-mariadb-2026-10-20.json @@ -0,0 +1,134 @@ +{ + "identity": { + "database": "mariadb", + "compose_project": "dw-mn-mariadb-20261020t185630", + "source_revision": "eeb510a3a59b18e0ad530e0a837325ac1d84d4fe", + "source_modified_files": 0, + "native_binary_sha256": "53d634e8fb7d96cab64d768e01d47db6e8c2f5a634cf0d6e67e6047b95ac03a9", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.0.10", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 25, + "pause": 3, + "start": 25, + "unpause": 3 + }, + "balancer": { + "requests": 3982, + "workflow_task_completions_409": 1, + "activity_task_completions_409": 0, + "responses_5xx": 1 + }, + "database_audit": { + "runs completed": 270, + "tasks activity completed": 270, + "tasks workflow completed": 540, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "mariadb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 2, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 1, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": { + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: conflicting_completion": 1, + "sdk-worker-1: Uncaught DurableWorkflow\\Exception\\TransportException: task_not_leased": 1, + "sdk-worker-2: Uncaught JsonException: ": 1 + } +} diff --git a/benchmarks/server-port/multinode/summary-mysql-2026-10-20.json b/benchmarks/server-port/multinode/summary-mysql-2026-10-20.json index 996097666..f65b4dad7 100644 --- a/benchmarks/server-port/multinode/summary-mysql-2026-10-20.json +++ b/benchmarks/server-port/multinode/summary-mysql-2026-10-20.json @@ -25,6 +25,7 @@ }, "windows": [ { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -33,10 +34,12 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] }, { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -45,10 +48,12 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] }, { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -57,6 +62,7 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] } diff --git a/benchmarks/server-port/multinode/summary-postgres-2026-10-20.json b/benchmarks/server-port/multinode/summary-postgres-2026-10-20.json index 105e5a94d..f09a174d3 100644 --- a/benchmarks/server-port/multinode/summary-postgres-2026-10-20.json +++ b/benchmarks/server-port/multinode/summary-postgres-2026-10-20.json @@ -25,6 +25,7 @@ }, "windows": [ { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -33,10 +34,12 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] }, { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -45,10 +48,12 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] }, { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -57,6 +62,7 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] } diff --git a/benchmarks/server-port/multinode/summary-sdk253-mariadb-2026-10-20.json b/benchmarks/server-port/multinode/summary-sdk253-mariadb-2026-10-20.json new file mode 100644 index 000000000..823f2ae1c --- /dev/null +++ b/benchmarks/server-port/multinode/summary-sdk253-mariadb-2026-10-20.json @@ -0,0 +1,130 @@ +{ + "identity": { + "database": "mariadb", + "compose_project": "dw-mn-mariadb-20261020t200655", + "source_revision": "eeb510a3a59b18e0ad530e0a837325ac1d84d4fe", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 26, + "pause": 3, + "start": 26, + "unpause": 3 + }, + "balancer": { + "requests": 4437, + "workflow_task_completions_409": 1, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks activity completed": 270, + "tasks workflow completed": 540, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "mariadb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": {} +} diff --git a/benchmarks/server-port/multinode/summary-sdk253-mysql-2026-10-20.json b/benchmarks/server-port/multinode/summary-sdk253-mysql-2026-10-20.json new file mode 100644 index 000000000..9bd30d9ac --- /dev/null +++ b/benchmarks/server-port/multinode/summary-sdk253-mysql-2026-10-20.json @@ -0,0 +1,130 @@ +{ + "identity": { + "database": "mysql", + "compose_project": "dw-mn-mysql-20261020t195521", + "source_revision": "eeb510a3a59b18e0ad530e0a837325ac1d84d4fe", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 25, + "pause": 3, + "start": 25, + "unpause": 3 + }, + "balancer": { + "requests": 4263, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 1 + }, + "database_audit": { + "runs completed": 270, + "tasks workflow completed": 540, + "tasks activity completed": 270, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "mysql": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": {} +} diff --git a/benchmarks/server-port/multinode/summary-sdk253-postgres-2026-10-20.json b/benchmarks/server-port/multinode/summary-sdk253-postgres-2026-10-20.json new file mode 100644 index 000000000..f47975969 --- /dev/null +++ b/benchmarks/server-port/multinode/summary-sdk253-postgres-2026-10-20.json @@ -0,0 +1,130 @@ +{ + "identity": { + "database": "postgres", + "compose_project": "dw-mn-postgres-20261020t204232", + "source_revision": "6b676949af70caf4f5b685d0466ccf278c1577c0", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": {}, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": {}, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": {}, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 24, + "pause": 3, + "start": 24, + "unpause": 3 + }, + "balancer": { + "requests": 4262, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks activity completed": 270, + "tasks workflow completed": 540, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "postgres": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": {} +} diff --git a/benchmarks/server-port/multinode/summary-sdk253-sqlite-2026-10-20.json b/benchmarks/server-port/multinode/summary-sdk253-sqlite-2026-10-20.json new file mode 100644 index 000000000..de431eea5 --- /dev/null +++ b/benchmarks/server-port/multinode/summary-sdk253-sqlite-2026-10-20.json @@ -0,0 +1,125 @@ +{ + "identity": { + "database": "sqlite", + "compose_project": "dw-mn-sqlite-20261020t202729", + "source_revision": "eeb510a3a59b18e0ad530e0a837325ac1d84d4fe", + "source_modified_files": 0, + "native_binary_sha256": "4c19c325cbd175d58d0b2710bb0450241f3760730514788eedf5185bab312e8f", + "smoke": false, + "schedule": { + "rate": 0.5, + "offer_seconds": 180, + "drain_seconds": 420, + "windows": 3, + "node_kill_every_seconds": 15, + "node_down_seconds": 4, + "worker_pause_every_seconds": 90, + "worker_pause_seconds": 70, + "workflow_task_lease_seconds": 60 + }, + "machine": { + "kernel": "5.15.0-198-generic", + "logical_cpus": 4, + "cpu_model": "Intel(R) Core(TM) i5-6500 CPU @ 3.20GHz" + } + }, + "windows": [ + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + }, + { + "sdk_php": "2.5.3", + "outcome": "pass", + "planned_starts": 90, + "accepted_starts": 90, + "completed": 90, + "verified": 90, + "activity_attempts_per_workflow": { + "1": 90 + }, + "repairs_by_workflow": null, + "retries": [], + "errors": [] + } + ], + "chaos": { + "kill": 25, + "pause": 3, + "start": 25, + "unpause": 3 + }, + "balancer": { + "requests": 4351, + "workflow_task_completions_409": 0, + "activity_task_completions_409": 0, + "responses_5xx": 0 + }, + "database_audit": { + "runs completed": 270, + "tasks activity completed": 270, + "tasks workflow completed": 540, + "attempts completed": 270, + "duplicate_workflow_completions": 0, + "duplicate_activity_completions": 0, + "duplicate_history_sequences": 0 + }, + "containers": { + "lb": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "native-3": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-1": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + }, + "sdk-worker-2": { + "restarts": 0, + "oom_killed": false, + "exit_code": 0 + } + }, + "native_panics": 0, + "sdk_worker_exits": {} +} diff --git a/benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json b/benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json index 27a8564ca..a22e141b2 100644 --- a/benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json +++ b/benchmarks/server-port/multinode/summary-sqlite-2026-10-20.json @@ -25,6 +25,7 @@ }, "windows": [ { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -33,10 +34,12 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] }, { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -45,10 +48,12 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] }, { + "sdk_php": "2.0.10", "outcome": "pass", "planned_starts": 90, "accepted_starts": 90, @@ -57,6 +62,7 @@ "activity_attempts_per_workflow": { "1": 90 }, + "repairs_by_workflow": null, "retries": [], "errors": [] }