From 503b3115c9ebfd856df48e607a1d2ddf5f3633ce Mon Sep 17 00:00:00 2001 From: Dimitrios Vasilas Date: Wed, 22 Apr 2026 16:41:35 +0300 Subject: [PATCH 1/5] WKBCH-25: Complete CRR workflow PR #49 added the CRR plumbing but didn't create the IAM role backbeat assumes for replication. Add it in the data account via vault accountSeeds so role-issued credentials authorize source and destination S3 calls. Ship scripts/enable-crr.sh to create the buckets and apply the replication policy idempotently. --- cmd/configure.go | 1 + scripts/enable-crr.sh | 102 +++++++++++++++++++ templates/backbeat/admin-backbeat.json | 1 + templates/backbeat/config.json | 8 +- templates/global/docker-compose.yaml | 5 +- templates/kafka/setup.sh | 4 + templates/vault/config.json | 32 ++++++ templates/vault/create-management-account.sh | 9 +- 8 files changed, 157 insertions(+), 5 deletions(-) create mode 100755 scripts/enable-crr.sh create mode 100644 templates/backbeat/admin-backbeat.json diff --git a/cmd/configure.go b/cmd/configure.go index 6c3dcc0..386e015 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -145,6 +145,7 @@ func generateBackbeatConfig(cfg EnvironmentConfig, path string) error { "config.json", "config.notification.json", "notificationCredentials.json", + "admin-backbeat.json", } return renderTemplates(cfg, "templates/backbeat", filepath.Join(path, "backbeat"), templates) diff --git a/scripts/enable-crr.sh b/scripts/enable-crr.sh new file mode 100755 index 0000000..f994f3f --- /dev/null +++ b/scripts/enable-crr.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# enable-crr.sh — create source + destination buckets and configure replication. +# +# Usage: +# scripts/enable-crr.sh --source --destination \ +# [--prefix ] [--endpoint ] +# +# Defaults: +# --endpoint http://127.0.0.1:8000 +# --prefix "" (replicate everything) +# +# Idempotent: re-running is a no-op once resources exist. + +set -eu + +# Pinned to match templates/vault/create-management-account.sh and the +# replication-role accountSeed in templates/vault/config.json. +ROLE_ARN="arn:aws:iam::123456789012:role/scality-internal/replication-role" + +SOURCE="" +DESTINATION="" +PREFIX="" +ENDPOINT="http://127.0.0.1:8000" + +while [ $# -gt 0 ]; do + case "$1" in + --source) SOURCE="$2"; shift 2 ;; + --destination) DESTINATION="$2"; shift 2 ;; + --prefix) PREFIX="$2"; shift 2 ;; + --endpoint) ENDPOINT="$2"; shift 2 ;; + -h|--help) + sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac +done + +if [ -z "$SOURCE" ] || [ -z "$DESTINATION" ]; then + echo "error: --source and --destination are required" >&2 + exit 2 +fi + +# testaccount credentials are fixed in templates/vault/create-management-account.sh +export AWS_ACCESS_KEY_ID="WBTKACCESSI9O3YKIRQ0" +export AWS_SECRET_ACCESS_KEY="ICxmNTBbOqijy4rMq/MOP1EPlTMqfsEBLjROcAbN" +export AWS_DEFAULT_REGION="us-east-1" + +AWS="aws --endpoint-url $ENDPOINT" + +create_bucket() { + local bucket="$1" + if $AWS s3api create-bucket --bucket "$bucket" >/dev/null 2>&1; then + echo "[crr] created bucket $bucket" + else + # swallow "already exists and owned by you" — treat anything else as fatal + if $AWS s3api head-bucket --bucket "$bucket" >/dev/null 2>&1; then + echo "[crr] bucket $bucket already exists" + else + echo "error: failed to create bucket $bucket" >&2 + $AWS s3api create-bucket --bucket "$bucket" + exit 1 + fi + fi +} + +enable_versioning() { + local bucket="$1" + $AWS s3api put-bucket-versioning \ + --bucket "$bucket" \ + --versioning-configuration Status=Enabled + echo "[crr] versioning enabled on $bucket" +} + +create_bucket "$SOURCE" +create_bucket "$DESTINATION" +enable_versioning "$SOURCE" +enable_versioning "$DESTINATION" + +REPLICATION_CONFIG=$(cat < $DESTINATION (prefix='$PREFIX')" diff --git a/templates/backbeat/admin-backbeat.json b/templates/backbeat/admin-backbeat.json new file mode 100644 index 0000000..e8c82a1 --- /dev/null +++ b/templates/backbeat/admin-backbeat.json @@ -0,0 +1 @@ +{"D4IT2AWSB588GO5J9T00":"UEEu8tYlsOGGrgf4DAiSZD6apVNPUWqRiPG0nTB6"} diff --git a/templates/backbeat/config.json b/templates/backbeat/config.json index 72ab457..065a3c5 100644 --- a/templates/backbeat/config.json +++ b/templates/backbeat/config.json @@ -21,6 +21,10 @@ "host": "127.0.0.1", "port": 8500 }, + "redis": { + "host": "127.0.0.1", + "port": 6379 + }, "replicationGroupId": "RG001 ", "queuePopulator": { "cronRule": "*/5 * * * * *", @@ -59,7 +63,7 @@ "host": "127.0.0.1", "port": 8500, "adminPort": 8600, - "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json" + "adminCredentialsFile": "/conf/admin-backbeat.json" } } }, @@ -74,7 +78,7 @@ "host": "127.0.0.1", "port": 8500, "adminPort": 8600, - "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json" + "adminCredentialsFile": "/conf/admin-backbeat.json" } } }, diff --git a/templates/global/docker-compose.yaml b/templates/global/docker-compose.yaml index bf5b032..3ef0c17 100644 --- a/templates/global/docker-compose.yaml +++ b/templates/global/docker-compose.yaml @@ -181,6 +181,8 @@ services: depends_on: setup-vault: condition: service_completed_successfully + setup-kafka: + condition: service_completed_successfully environment: SUPERVISORD_CONF: supervisord.conf BACKBEAT_CONFIG_FILE: /conf/config.json @@ -188,6 +190,7 @@ services: - ./config/backbeat/supervisord.conf:/conf/supervisord.conf:ro - ./config/backbeat/config.json:/conf/config.json:ro - ./config/backbeat/config.notification.json:/conf/config.notification.json:ro + - ./config/backbeat/admin-backbeat.json:/conf/admin-backbeat.json:ro - ./config/backbeat/env:/conf/env:ro - ./logs/backbeat:/logs profiles: @@ -260,7 +263,7 @@ services: backbeat-data-mover backbeat-replication-status backbeat-replication-failed - backbeat-metrics-group-crr + backbeat-metrics CREATE_ZOOKEEPER_PATHS: 'true' ZOOKEEPER_ENDPOINT: 127.0.0.1:2181/backbeat depends_on: diff --git a/templates/kafka/setup.sh b/templates/kafka/setup.sh index 1e843f7..0c2aa50 100644 --- a/templates/kafka/setup.sh +++ b/templates/kafka/setup.sh @@ -46,6 +46,9 @@ if [[ "$CREATE_ZOOKEEPER_PATHS" == "true" ]]; then fi echo "[setup] Creating Zookeeper paths..." + # zookeeper-shell.sh exits non-zero when any 'create' returns NodeExists, + # which is the expected outcome on re-runs against a persisted volume. + set +e zookeeper-shell.sh localhost:2181/backbeat <&1) || { if echo "$resp" | grep -q "EntityAlreadyExists"; then @@ -221,6 +225,7 @@ if [ -f "$BACKBEAT_CONFIG_FILE" ]; then mv /tmp/backbeat-config.updated.json "$BACKBEAT_CONFIG_FILE" echo "[setup] Backbeat config.json updated with lifecycle credentials" + fi echo "[setup] Setup completed successfully" From 705326dd790b5d07b96c50b96fd2998d3642da6d Mon Sep 17 00:00:00 2001 From: Dimitrios Vasilas Date: Tue, 28 Apr 2026 18:00:19 +0300 Subject: [PATCH 2/5] WKBCH-25: Run setup-vault and setup-scuba as host user Both containers write into host-mounted directories. Becuase the run as root, they left root-owned files. --- cmd/config.go | 5 +++++ templates/global/defaults.env | 3 +++ templates/global/docker-compose.yaml | 2 ++ 3 files changed, 10 insertions(+) diff --git a/cmd/config.go b/cmd/config.go index 65237ba..649fbdc 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -60,6 +60,9 @@ type EnvironmentConfig struct { Clickhouse ClickhouseConfig `yaml:"clickhouse"` Fluentbit FluentbitConfig `yaml:"fluentbit"` Nginx NginxConfig `yaml:"nginx"` + + HostUID int `yaml:"-"` + HostGID int `yaml:"-"` } type GlobalConfig struct { @@ -319,6 +322,8 @@ func DefaultEnvironmentConfig() EnvironmentConfig { func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) { cfg := DefaultEnvironmentConfig() + cfg.HostUID = os.Getuid() + cfg.HostGID = os.Getgid() if path == "" { return cfg, nil diff --git a/templates/global/defaults.env b/templates/global/defaults.env index 126ee95..6a81553 100644 --- a/templates/global/defaults.env +++ b/templates/global/defaults.env @@ -15,3 +15,6 @@ NGINX_IMAGE="{{ .Nginx.Image }}" METADATA_S3_DB_VERSION="{{ .S3Metadata.VFormat }}" CLOUDSERVER_ENABLE_NULL_VERSION_COMPAT_MODE="{{ .Cloudserver.EnableNullVersionCompatMode }}" + +HOST_UID="{{ .HostUID }}" +HOST_GID="{{ .HostGID }}" diff --git a/templates/global/docker-compose.yaml b/templates/global/docker-compose.yaml index 3ef0c17..e5a3c77 100644 --- a/templates/global/docker-compose.yaml +++ b/templates/global/docker-compose.yaml @@ -125,6 +125,7 @@ services: BASE_IMAGE: ${VAULT_IMAGE} container_name: workbench-setup-vault network_mode: host + user: "${HOST_UID}:${HOST_GID}" volumes: - ./config/vault/management-creds.json:/conf/management-creds.json:ro - ./config/backbeat:/conf/backbeat:rw @@ -166,6 +167,7 @@ services: BASE_IMAGE: ${SCUBA_IMAGE} container_name: workbench-setup-scuba network_mode: host + user: "${HOST_UID}:${HOST_GID}" depends_on: setup-vault: condition: service_completed_successfully From 316d4b2b4f8f914e264eda68eec1370c7fe91eef Mon Sep 17 00:00:00 2001 From: Dimitrios Vasilas Date: Thu, 30 Apr 2026 18:51:19 +0300 Subject: [PATCH 3/5] WKBCH-25: Disable lifecycle conductor backlogControl WKBCH-25 added redis to backbeat config, which activates backlogControl (previously a no-op since redis was unreachable). It then throttles the conductor whenever the bucket-tasks topic has any consumer lag, which causes test flakiness under bursty parallel load: lifecycle stalls long enough to blow expiration timeouts. --- templates/backbeat/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/backbeat/config.json b/templates/backbeat/config.json index 065a3c5..46da47e 100644 --- a/templates/backbeat/config.json +++ b/templates/backbeat/config.json @@ -180,7 +180,7 @@ "port": 8500 } }, - "backlogControl": { "enabled": true }, + "backlogControl": { "enabled": false }, "cronRule": "*/5 * * * * *", "concurrency": 10, "bucketSource": "bucketd", From f9a33d1636a580a077dc5382e4695175f1ce5cff Mon Sep 17 00:00:00 2001 From: Dimitrios Vasilas Date: Thu, 30 Apr 2026 23:22:23 +0300 Subject: [PATCH 4/5] WKBCH-25: Provision queue-populator for raft sessions 1, 2, 3 --- templates/kafka/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/kafka/setup.sh b/templates/kafka/setup.sh index 0c2aa50..d856897 100644 --- a/templates/kafka/setup.sh +++ b/templates/kafka/setup.sh @@ -64,9 +64,9 @@ create /queue-populator/raft-id-dispatcher create /queue-populator/raft-id-dispatcher/owners create /queue-populator/raft-id-dispatcher/leaders create /queue-populator/raft-id-dispatcher/provisions -create /queue-populator/raft-id-dispatcher/provisions/0 create /queue-populator/raft-id-dispatcher/provisions/1 create /queue-populator/raft-id-dispatcher/provisions/2 +create /queue-populator/raft-id-dispatcher/provisions/3 create /lifecycle create /lifecycle/conductor create /lifecycle/conductor/election From d098b2c90e61392e3e5ab197773a765e1a38a10b Mon Sep 17 00:00:00 2001 From: Dimitrios Vasilas Date: Mon, 4 May 2026 12:32:09 +0300 Subject: [PATCH 5/5] WKBCH-25: Wait for redis to be healthy before starting backbeat --- templates/global/docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/global/docker-compose.yaml b/templates/global/docker-compose.yaml index e5a3c77..6df165a 100644 --- a/templates/global/docker-compose.yaml +++ b/templates/global/docker-compose.yaml @@ -185,6 +185,8 @@ services: condition: service_completed_successfully setup-kafka: condition: service_completed_successfully + redis: + condition: service_healthy environment: SUPERVISORD_CONF: supervisord.conf BACKBEAT_CONFIG_FILE: /conf/config.json