Add 437 clickhouse upgrade test backup

scripts/clickhouse-upgrade-test/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+
+# Generated test output -- keep the directory (see results/.gitkeep) but not
+# whatever a local run drops into it.
+results/report.md
+results/report.json
+results/steps/
+results/backups.json

scripts/clickhouse-upgrade-test/Makefile

@@ -0,0 +1,21 @@
+.PHONY: test test-staged test-direct config down clean
+
+# Full test: staged (recommended) upgrade path, then the naive direct-jump path
+test:
+	python3 run_test.py --scenario both
+
+test-staged:
+	python3 run_test.py --scenario staged
+
+test-direct:
+	python3 run_test.py --scenario direct
+
+# Validate docker-compose.yml without needing registry access
+config:
+	docker compose config
+
+down:
+	docker compose down -v
+
+clean: down
+	rm -rf results/report.md results/report.json

scripts/clickhouse-upgrade-test/config/backup/s3.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!--
+  Credentials for the backup bucket on the `minio` service
+  (docker-compose.backup.yml), mounted on ch1/ch2/ch3 and chr. Same mechanism
+  as production's config.d/ooni_backup_s3.xml (ansible role clickhouse_backup):
+  BACKUP/RESTORE and the s3() function pick up keys for any URL under this
+  endpoint, so queries never carry them. That matters because an incremental
+  backup stores its base_backup argument verbatim in its .backup file.
+-->
+<clickhouse>
+    <s3>
+        <ooni_backup>
+            <endpoint>http://minio:9000/backups/</endpoint>
+            <access_key_id>minioadmin</access_key_id>
+            <secret_access_key>minioadmin</secret_access_key>
+        </ooni_backup>
+    </s3>
+</clickhouse>

scripts/clickhouse-upgrade-test/config/ch1/node.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>01</replica>
+        <cluster>oonidata_cluster</cluster>
+    </macros>
+
+    <!-- Embedded Keeper: this node is Keeper server id=1 -->
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>information</raft_logs_level>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>ch1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>ch2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>ch3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</clickhouse>

scripts/clickhouse-upgrade-test/config/ch2/node.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>02</replica>
+        <cluster>oonidata_cluster</cluster>
+    </macros>
+
+    <!-- Embedded Keeper: this node is Keeper server id=2 -->
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>2</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>information</raft_logs_level>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>ch1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>ch2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>ch3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</clickhouse>

scripts/clickhouse-upgrade-test/config/ch3/node.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>03</replica>
+        <cluster>oonidata_cluster</cluster>
+    </macros>
+
+    <!-- Embedded Keeper: this node is Keeper server id=3 -->
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>3</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>information</raft_logs_level>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>ch1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>ch2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>ch3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</clickhouse>

scripts/clickhouse-upgrade-test/config/common/common.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+<!--
+  Settings shared by all three nodes. Mirrors the shape of OONI's production
+  config (ooni/devops ansible/group_vars/clickhouse/vars.yml -> idealista.clickhouse_role)
+  trimmed down to what's needed to reproduce the replication/Keeper behaviour
+  we care about for the upgrade test.
+-->
+<clickhouse>
+    <listen_host>0.0.0.0</listen_host>
+    <listen_host>::</listen_host>
+    <listen_try>1</listen_try>
+
+    <logger>
+        <level>information</level>
+        <console>1</console>
+    </logger>
+
+    <!-- Same three-node ensemble on every node: each ClickHouse server also
+         runs the embedded Keeper role, exactly like data1/data2/data3 in prod. -->
+    <zookeeper>
+        <node>
+            <host>ch1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>ch2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>ch3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+
+    <!-- Distributed DDL queue, same path/profile as prod so ON CLUSTER works -->
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+        <profile>default</profile>
+        <pool_size>1</pool_size>
+        <task_max_lifetime>604800</task_max_lifetime>
+        <cleanup_delay_period>60</cleanup_delay_period>
+        <max_tasks_in_queue>1000</max_tasks_in_queue>
+    </distributed_ddl>
+
+    <remote_servers>
+        <oonidata_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>ch1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>ch2</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>ch3</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </oonidata_cluster>
+    </remote_servers>
+</clickhouse>

scripts/clickhouse-upgrade-test/config/common/users.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!-- Deliberately simple auth (no password): this cluster only ever exists
+     on a private docker network for the duration of the test. -->
+<clickhouse>
+    <profiles>
+        <default>
+            <max_memory_usage>2000000000</max_memory_usage>
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+            <load_balancing>random</load_balancing>
+        </default>
+    </profiles>
+
+    <users>
+        <default>
+            <password></password>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+            <access_management>1</access_management>
+        </default>
+    </users>
+
+    <quotas>
+        <default>
+            <interval>
+                <duration>3600</duration>
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</clickhouse>

scripts/clickhouse-upgrade-test/config/restore/node.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<!--
+  chr: a standalone node backups are restored onto (docker-compose.backup.yml).
+  It runs its own single-node Keeper, so restoring replicated tables never
+  touches the ZooKeeper paths of the ch1/ch2/ch3 cluster, the same
+  situation as restoring onto a freshly built server after losing the
+  cluster. Macros match production's, since the replicated tables' Keeper
+  paths use {cluster} and {shard}.
+-->
+<clickhouse>
+    <listen_host>0.0.0.0</listen_host>
+    <listen_host>::</listen_host>
+    <listen_try>1</listen_try>
+
+    <logger>
+        <level>information</level>
+        <console>1</console>
+    </logger>
+
+    <macros>
+        <shard>01</shard>
+        <replica>01</replica>
+        <cluster>oonidata_cluster</cluster>
+    </macros>
+
+    <zookeeper>
+        <node>
+            <host>chr</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>information</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>chr</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</clickhouse>

scripts/clickhouse-upgrade-test/docker-compose.backup.yml

@@ -0,0 +1,56 @@
+# Overlay for the backup/restore checks (harness/backup.py): an S3-compatible
+# store to back the cluster up to, and a standalone node to restore onto.
+#
+# Used together with the base file, always both flags:
+#   docker compose -f docker-compose.yml -f docker-compose.backup.yml <cmd>
+#
+# MinIO Inc. stopped publishing images, so this uses pgsty/minio, the
+# community-maintained fork (AGPLv3, same `server` command and env vars;
+# its image also ships `mc`, used to create the bucket). Production backs up
+# to Hetzner Object Storage, which speaks the same S3 API.
+#
+# ch1/ch2/ch3 get the bucket credentials (config/backup/s3.xml) from the base
+# docker-compose.yml, not from here: harness/compose.py recreates nodes
+# during upgrades with the base file only, and the mount must survive that.
+
+services:
+  minio:
+    image: "docker.io/pgsty/minio:RELEASE.2026-08-04T00-00-00Z"
+    container_name: ooni-minio
+    hostname: minio
+    command: ["server", "/data"]
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    networks:
+      - chnet
+    volumes:
+      - minio_data:/data
+    healthcheck:
+      test: ["CMD", "curl", "-fs", "http://localhost:9000/minio/health/live"]
+      interval: 3s
+      timeout: 3s
+      retries: 20
+
+  # Recreated (with fresh anonymous volumes) for every restore check, on
+  # whatever version CHR_IMAGE names -- see harness/backup.py.
+  chr:
+    image: "clickhouse/clickhouse-server:${CHR_IMAGE:-24.8.6.70}"
+    container_name: ooni-chr
+    hostname: chr
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    mem_limit: 1500m
+    networks:
+      - chnet
+    ports:
+      - "${CHR_HTTP_PORT:-8126}:8123"
+    volumes:
+      - ./config/restore/node.xml:/etc/clickhouse-server/config.d/node.xml:ro
+      - ./config/common/users.xml:/etc/clickhouse-server/users.d/users.xml:ro
+      - ./config/backup/s3.xml:/etc/clickhouse-server/config.d/backup_s3.xml:ro
+
+volumes:
+  minio_data: