WKBCH-25: Complete CRR support

cmd/config.go

@@ -60,6 +60,9 @@ type EnvironmentConfig struct {
 	Clickhouse     ClickhouseConfig     `yaml:"clickhouse"`
 	Fluentbit      FluentbitConfig      `yaml:"fluentbit"`
 	Nginx          NginxConfig          `yaml:"nginx"`
+
+	HostUID int `yaml:"-"`
+	HostGID int `yaml:"-"`
 }
 
 type GlobalConfig struct {
@@ -319,6 +322,8 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
 
 func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) {
 	cfg := DefaultEnvironmentConfig()
+	cfg.HostUID = os.Getuid()
+	cfg.HostGID = os.Getgid()
 
 	if path == "" {
 		return cfg, nil

cmd/configure.go

@@ -145,6 +145,7 @@ func generateBackbeatConfig(cfg EnvironmentConfig, path string) error {
 		"config.json",
 		"config.notification.json",
 		"notificationCredentials.json",
+		"admin-backbeat.json",
 	}
 
 	return renderTemplates(cfg, "templates/backbeat", filepath.Join(path, "backbeat"), templates)

scripts/enable-crr.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# enable-crr.sh — create source + destination buckets and configure replication.
+#
+# Usage:
+#   scripts/enable-crr.sh --source <bucket> --destination <bucket> \
+#       [--prefix <pfx>] [--endpoint <url>]
+#
+# Defaults:
+#   --endpoint  http://127.0.0.1:8000
+#   --prefix    ""   (replicate everything)
+#
+# Idempotent: re-running is a no-op once resources exist.
+
+set -eu
+
+# Pinned to match templates/vault/create-management-account.sh and the
+# replication-role accountSeed in templates/vault/config.json.
+ROLE_ARN="arn:aws:iam::123456789012:role/scality-internal/replication-role"
+
+SOURCE=""
+DESTINATION=""
+PREFIX=""
+ENDPOINT="http://127.0.0.1:8000"
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --source)       SOURCE="$2"; shift 2 ;;
+        --destination)  DESTINATION="$2"; shift 2 ;;
+        --prefix)       PREFIX="$2"; shift 2 ;;
+        --endpoint)     ENDPOINT="$2"; shift 2 ;;
+        -h|--help)
+            sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'
+            exit 0
+            ;;
+        *) echo "unknown flag: $1" >&2; exit 2 ;;
+    esac
+done
+
+if [ -z "$SOURCE" ] || [ -z "$DESTINATION" ]; then
+    echo "error: --source and --destination are required" >&2
+    exit 2
+fi
+
+# testaccount credentials are fixed in templates/vault/create-management-account.sh
+export AWS_ACCESS_KEY_ID="WBTKACCESSI9O3YKIRQ0"
+export AWS_SECRET_ACCESS_KEY="ICxmNTBbOqijy4rMq/MOP1EPlTMqfsEBLjROcAbN"
+export AWS_DEFAULT_REGION="us-east-1"
+
+AWS="aws --endpoint-url $ENDPOINT"
+
+create_bucket() {
+    local bucket="$1"
+    if $AWS s3api create-bucket --bucket "$bucket" >/dev/null 2>&1; then
+        echo "[crr] created bucket $bucket"
+    else
+        # swallow "already exists and owned by you" — treat anything else as fatal
+        if $AWS s3api head-bucket --bucket "$bucket" >/dev/null 2>&1; then
+            echo "[crr] bucket $bucket already exists"
+        else
+            echo "error: failed to create bucket $bucket" >&2
+            $AWS s3api create-bucket --bucket "$bucket"
+            exit 1
+        fi
+    fi
+}
+
+enable_versioning() {
+    local bucket="$1"
+    $AWS s3api put-bucket-versioning \
+        --bucket "$bucket" \
+        --versioning-configuration Status=Enabled
+    echo "[crr] versioning enabled on $bucket"
+}
+
+create_bucket "$SOURCE"
+create_bucket "$DESTINATION"
+enable_versioning "$SOURCE"
+enable_versioning "$DESTINATION"
+
+REPLICATION_CONFIG=$(cat <<EOF
+{
+  "Role": "${ROLE_ARN},${ROLE_ARN}",
+  "Rules": [
+    {
+      "ID": "workbench-crr",
+      "Status": "Enabled",
+      "Prefix": "${PREFIX}",
+      "Destination": {
+        "Bucket": "arn:aws:s3:::${DESTINATION}",
+        "StorageClass": "sf"
+      }
+    }
+  ]
+}
+EOF
+)
+
+$AWS s3api put-bucket-replication \
+    --bucket "$SOURCE" \
+    --replication-configuration "$REPLICATION_CONFIG"
+
+echo "[crr] replication configured: $SOURCE -> $DESTINATION (prefix='$PREFIX')"

templates/backbeat/admin-backbeat.json

@@ -0,0 +1 @@
+{"D4IT2AWSB588GO5J9T00":"UEEu8tYlsOGGrgf4DAiSZD6apVNPUWqRiPG0nTB6"}

templates/backbeat/config.json

@@ -21,6 +21,10 @@
         "host": "127.0.0.1",
         "port": 8500
     },
+    "redis": {
+        "host": "127.0.0.1",
+        "port": 6379
+    },
     "replicationGroupId": "RG001  ",
     "queuePopulator": {
         "cronRule": "*/5 * * * * *",
@@ -59,7 +63,7 @@
                         "host": "127.0.0.1",
                         "port": 8500,
                         "adminPort": 8600,
-                        "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
+                        "adminCredentialsFile": "/conf/admin-backbeat.json"
                     }
                 }
             },
@@ -74,7 +78,7 @@
                         "host": "127.0.0.1",
                         "port": 8500,
                         "adminPort": 8600,
-                        "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
+                        "adminCredentialsFile": "/conf/admin-backbeat.json"
                     }
                 }
             },
@@ -176,7 +180,7 @@
                         "port": 8500
                     }
                 },
-                "backlogControl": { "enabled": true },
+                "backlogControl": { "enabled": false },
                 "cronRule": "*/5 * * * * *",
                 "concurrency": 10,
                 "bucketSource": "bucketd",

templates/global/defaults.env

@@ -15,3 +15,6 @@ NGINX_IMAGE="{{ .Nginx.Image }}"
 
 METADATA_S3_DB_VERSION="{{ .S3Metadata.VFormat }}"
 CLOUDSERVER_ENABLE_NULL_VERSION_COMPAT_MODE="{{ .Cloudserver.EnableNullVersionCompatMode }}"
+
+HOST_UID="{{ .HostUID }}"
+HOST_GID="{{ .HostGID }}"

templates/global/docker-compose.yaml

@@ -125,6 +125,7 @@ services:
         BASE_IMAGE: ${VAULT_IMAGE}
     container_name: workbench-setup-vault
     network_mode: host
+    user: "${HOST_UID}:${HOST_GID}"
     volumes:
       - ./config/vault/management-creds.json:/conf/management-creds.json:ro
       - ./config/backbeat:/conf/backbeat:rw
@@ -166,6 +167,7 @@ services:
         BASE_IMAGE: ${SCUBA_IMAGE}
     container_name: workbench-setup-scuba
     network_mode: host
+    user: "${HOST_UID}:${HOST_GID}"
     depends_on:
       setup-vault:
         condition: service_completed_successfully
@@ -181,13 +183,18 @@ services:
     depends_on:
       setup-vault:
         condition: service_completed_successfully
+      setup-kafka:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
     environment:
       SUPERVISORD_CONF: supervisord.conf
       BACKBEAT_CONFIG_FILE: /conf/config.json
     volumes:
       - ./config/backbeat/supervisord.conf:/conf/supervisord.conf:ro
       - ./config/backbeat/config.json:/conf/config.json:ro
       - ./config/backbeat/config.notification.json:/conf/config.notification.json:ro
+      - ./config/backbeat/admin-backbeat.json:/conf/admin-backbeat.json:ro
       - ./config/backbeat/env:/conf/env:ro
       - ./logs/backbeat:/logs
     profiles:
@@ -260,7 +267,7 @@ services:
         backbeat-data-mover
         backbeat-replication-status
         backbeat-replication-failed
-        backbeat-metrics-group-crr
+        backbeat-metrics
       CREATE_ZOOKEEPER_PATHS: 'true'
       ZOOKEEPER_ENDPOINT: 127.0.0.1:2181/backbeat
     depends_on:

templates/kafka/setup.sh

@@ -46,6 +46,9 @@ if [[ "$CREATE_ZOOKEEPER_PATHS" == "true" ]]; then
     fi
 
     echo "[setup] Creating Zookeeper paths..."
+    # zookeeper-shell.sh exits non-zero when any 'create' returns NodeExists,
+    # which is the expected outcome on re-runs against a persisted volume.
+    set +e
     zookeeper-shell.sh localhost:2181/backbeat <<EOF
 create /
 create /bucket-notification
@@ -61,14 +64,15 @@ create /queue-populator/raft-id-dispatcher
 create /queue-populator/raft-id-dispatcher/owners
 create /queue-populator/raft-id-dispatcher/leaders
 create /queue-populator/raft-id-dispatcher/provisions
-create /queue-populator/raft-id-dispatcher/provisions/0
 create /queue-populator/raft-id-dispatcher/provisions/1
 create /queue-populator/raft-id-dispatcher/provisions/2
+create /queue-populator/raft-id-dispatcher/provisions/3
 create /lifecycle
 create /lifecycle/conductor
 create /lifecycle/conductor/election
 quit
 EOF
+    set -e
     echo "[setup] Zookeeper paths created."
     echo
 fi

templates/vault/config.json

@@ -115,6 +115,38 @@
                     ]
                 }
             }
+        },
+        {
+            "role": {
+                "roleName": "replication-role",
+                "trustPolicy": {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Effect": "Allow",
+                            "Principal": {
+                                "AWS": "arn:aws:iam::000000000000:user/root"
+                            },
+                            "Action": "sts:AssumeRole",
+                            "Condition": {}
+                        }
+                    ]
+                }
+            },
+            "permissionPolicy": {
+                "policyName": "replication-policy",
+                "policyDocument": {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Sid": "ReplicationFullAccess",
+                            "Effect": "Allow",
+                            "Action": ["s3:*"],
+                            "Resource": ["*"]
+                        }
+                    ]
+                }
+            }
         }
     ],
     "utapi": {

templates/vault/create-management-account.sh

@@ -56,16 +56,20 @@ fi
 echo "[setup] Management account and access key setup completed successfully"
 
 # === Create test data account ===
-# This account is seeded with the lifecycle role via vault's accountSeeds,
-# which allows backbeat to AssumeRole into it for lifecycle operations.
+# This account is seeded with the lifecycle and replication roles via vault's
+# accountSeeds. We pin the account ID so downstream tooling (e.g. the
+# enable-crr.sh helper) can reference role ARNs in this account by a stable
+# value.
 TEST_ACCOUNT_ACCESS_KEY="WBTKACCESSI9O3YKIRQ0"
 TEST_ACCOUNT_SECRET_KEY="ICxmNTBbOqijy4rMq/MOP1EPlTMqfsEBLjROcAbN"
+TEST_ACCOUNT_ID="123456789012"
 
 echo "[setup] Creating test data account..."
 resp=$(./node_modules/vaultclient/bin/vaultclient \
         create-account \
         --name testaccount \
         --email testaccount@test.com \
+        --accountid "$TEST_ACCOUNT_ID" \
         --host 127.0.0.1 \
         --port 8600 2>&1) || {
   if echo "$resp" | grep -q "EntityAlreadyExists"; then
@@ -221,6 +225,7 @@ if [ -f "$BACKBEAT_CONFIG_FILE" ]; then
 
   mv /tmp/backbeat-config.updated.json "$BACKBEAT_CONFIG_FILE"
   echo "[setup] Backbeat config.json updated with lifecycle credentials"
+
 fi
 
 echo "[setup] Setup completed successfully"