From 926790e8fdcc0550429fe2a7d3215562ddb600a1 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Wed, 3 Dec 2025 15:33:21 +0100 Subject: [PATCH 1/5] added backbeat apis ISSUE: CLDSRVCLT-5 --- .github/authdata.json | 21 ++ .github/docker-compose.backbeat.yml | 191 +++++++++++++++++ .github/workflows/test.yml | 39 ++++ .../proxyBackbeatApis/checkConnection.smithy | 13 ++ .../proxyBackbeatApis/commonStructures.smithy | 21 ++ .../proxyBackbeatApis/getFailedObject.smithy | 29 +++ .../getLocationsIngestionStatus.smithy | 16 ++ .../getLocationsStatus.smithy | 16 ++ models/proxyBackbeatApis/listFailed.smithy | 29 +++ .../pauseAllIngestionSites.smithy | 18 ++ models/proxyBackbeatApis/pauseAllSites.smithy | 18 ++ .../pauseIngestionSite.smithy | 22 ++ models/proxyBackbeatApis/pauseSite.smithy | 22 ++ .../resumeAllIngestionSites.smithy | 18 ++ .../proxyBackbeatApis/resumeAllSites.smithy | 18 ++ .../resumeIngestionSite.smithy | 22 ++ models/proxyBackbeatApis/resumeSite.smithy | 22 ++ .../retryFailedObjects.smithy | 19 ++ .../scheduleIngestionSiteResume.smithy | 22 ++ .../scheduleSiteResume.smithy | 22 ++ package.json | 4 +- service/cloudserverProxyBackbeatApis.smithy | 32 +++ smithy-build.json | 9 + src/clients/proxyBackbeatApis.ts | 64 ++++++ src/index.ts | 1 + tests/testBackbeatProxyApis.test.ts | 194 ++++++++++++++++++ tests/testHelpers.ts | 8 + tests/testSetup.ts | 7 +- 28 files changed, 915 insertions(+), 2 deletions(-) create mode 100644 .github/authdata.json create mode 100644 .github/docker-compose.backbeat.yml create mode 100644 models/proxyBackbeatApis/checkConnection.smithy create mode 100644 models/proxyBackbeatApis/commonStructures.smithy create mode 100644 models/proxyBackbeatApis/getFailedObject.smithy create mode 100644 models/proxyBackbeatApis/getLocationsIngestionStatus.smithy create mode 100644 models/proxyBackbeatApis/getLocationsStatus.smithy create mode 100644 models/proxyBackbeatApis/listFailed.smithy create mode 100644 models/proxyBackbeatApis/pauseAllIngestionSites.smithy create mode 100644 models/proxyBackbeatApis/pauseAllSites.smithy create mode 100644 models/proxyBackbeatApis/pauseIngestionSite.smithy create mode 100644 models/proxyBackbeatApis/pauseSite.smithy create mode 100644 models/proxyBackbeatApis/resumeAllIngestionSites.smithy create mode 100644 models/proxyBackbeatApis/resumeAllSites.smithy create mode 100644 models/proxyBackbeatApis/resumeIngestionSite.smithy create mode 100644 models/proxyBackbeatApis/resumeSite.smithy create mode 100644 models/proxyBackbeatApis/retryFailedObjects.smithy create mode 100644 models/proxyBackbeatApis/scheduleIngestionSiteResume.smithy create mode 100644 models/proxyBackbeatApis/scheduleSiteResume.smithy create mode 100644 service/cloudserverProxyBackbeatApis.smithy create mode 100644 src/clients/proxyBackbeatApis.ts create mode 100644 tests/testBackbeatProxyApis.test.ts diff --git a/.github/authdata.json b/.github/authdata.json new file mode 100644 index 00000000..36b2bf85 --- /dev/null +++ b/.github/authdata.json @@ -0,0 +1,21 @@ +{ + "accounts": [{ + "name": "bart", + "arn": "aws::iam:123456789012:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "displayName": "bart", + "keys": { + "access": "accessKey1", + "secret": "verySecretKey1" + } + }, { + "name": "lisa", + "arn": "aws::iam:123456789013:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", + "displayName": "lisa", + "keys": { + "access": "accessKey2", + "secret": "verySecretKey2" + } + }] +} diff --git a/.github/docker-compose.backbeat.yml b/.github/docker-compose.backbeat.yml new file mode 100644 index 00000000..8807bed1 --- /dev/null +++ b/.github/docker-compose.backbeat.yml @@ -0,0 +1,191 @@ +version: '3.8' + +services: + zookeeper: + image: zookeeper:3.8 + platform: linux/amd64 + ports: + - "127.0.0.1:2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: confluentinc/cp-kafka:7.4.0 + platform: linux/amd64 + ports: + - "127.0.0.1:9092:9092" + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_BROKER_ID: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + depends_on: + - zookeeper + + mongodb: + image: mongo:4.4 + platform: linux/amd64 + ports: + - "127.0.0.1:27018:27018" + command: mongod --port 27018 --replSet rs0 --bind_ip_all + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo --port 27018 --quiet + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + mongodb-init: + image: mongo:4.4 + platform: linux/amd64 + depends_on: + mongodb: + condition: service_healthy + command: > + mongo --host mongodb:27018 --eval + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})' + restart: "no" + + redis: + image: redis:7-alpine + platform: linux/amd64 + ports: + - "127.0.0.1:6379:6379" + + cloudserver: + image: ghcr.io/scality/cloudserver:9.1.4 + platform: linux/amd64 + ports: + - "127.0.0.1:8000:8000" + environment: + - S3VAULT=mem + - S3METADATA=mongodb + - S3DATA=mem + - MONGODB_HOSTS=mongodb:27018 + - MONGODB_RS=rs0 + - REMOTE_MANAGEMENT_DISABLE=true + - LOG_LEVEL=info + - CRR_METRICS_HOST=backbeat-api + - CRR_METRICS_PORT=8900 + depends_on: + mongodb-init: + condition: service_completed_successfully + backbeat-api: + condition: service_started + + backbeat-api: + image: ghcr.io/scality/backbeat:9.1.3 + platform: linux/amd64 + ports: + - "127.0.0.1:8900:8900" + environment: + # Kafka configuration + KAFKA_HOSTS: kafka:9092 + KAFKA_COMPRESSION_TYPE: snappy + + # Zookeeper configuration + ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat + ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true" + + # Redis configuration + REDIS_HOST: redis + REDIS_PORT: 6379 + + # MongoDB configuration + MONGODB_HOSTS: mongodb:27018 + MONGODB_DATABASE: metadata + + # Management backend + MANAGEMENT_BACKEND: operator + REMOTE_MANAGEMENT_DISABLE: "true" + + # Cloudserver configuration + CLOUDSERVER_HOST: cloudserver + CLOUDSERVER_PORT: 8000 + + # Replication source S3 + EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver + EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000 + EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service + EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication + + # Replication destination + EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service + EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001 + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}' + + # Lifecycle + EXTENSIONS_LIFECYCLE_AUTH_TYPE: service + EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT: service-lifecycle + + # Healthchecks + HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0" + + # Logging + LOG_LEVEL: info + depends_on: + mongodb-init: + condition: service_completed_successfully + kafka: + condition: service_started + redis: + condition: service_started + command: node bin/backbeat.js + + backbeat-queue-processor: + image: ghcr.io/scality/backbeat:9.1.3 + platform: linux/amd64 + ports: + - "127.0.0.1:4043:4043" + environment: + # Kafka configuration + KAFKA_HOSTS: kafka:9092 + KAFKA_COMPRESSION_TYPE: snappy + + # Zookeeper configuration + ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat + ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true" + + # Redis configuration + REDIS_HOST: redis + REDIS_PORT: 6379 + + # MongoDB configuration + MONGODB_HOSTS: mongodb:27018 + MONGODB_DATABASE: metadata + + # Management backend + MANAGEMENT_BACKEND: operator + REMOTE_MANAGEMENT_DISABLE: "true" + + # Cloudserver configuration + CLOUDSERVER_HOST: cloudserver + CLOUDSERVER_PORT: 8000 + + # Replication source S3 + EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver + EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000 + EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service + EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication + + # Replication destination + EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service + EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001 + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}' + + # Probe server port + LIVENESS_PROBE_PORT: 4043 + + # Logging + LOG_LEVEL: info + depends_on: + mongodb-init: + condition: service_completed_successfully + cloudserver: + condition: service_started + backbeat-api: + condition: service_started + command: node extensions/replication/queueProcessor/task.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index baf7c6cf..c0af3a37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -92,3 +92,42 @@ jobs: - name: Stop Cloudserver if: always() run: docker compose -f .github/docker-compose.cloudserver-metadata.yml down + + test-backbeat-apis: + name: Test backbeat apis + runs-on: ubuntu-24.04 + needs: lint + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup and Build + uses: ./.github/actions/setup-and-build + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Start Backbeat with CloudServer + run: docker compose -f .github/docker-compose.backbeat.yml up -d + + - name: Wait for Cloudserver to be ready + run: | + set -o pipefail + bash .github/scripts/wait_for_local_port.bash 8000 40 + + - name: Wait for Backbeat API to be ready + run: | + set -o pipefail + bash .github/scripts/wait_for_local_port.bash 8900 60 + + - name: Run backbeat apis tests + run: yarn test:backbeat-apis + + - name: Stop Backbeat and Cloudserver + if: always() + run: docker compose -f .github/docker-compose.backbeat.yml down \ No newline at end of file diff --git a/models/proxyBackbeatApis/checkConnection.smithy b/models/proxyBackbeatApis/checkConnection.smithy new file mode 100644 index 00000000..0dda8173 --- /dev/null +++ b/models/proxyBackbeatApis/checkConnection.smithy @@ -0,0 +1,13 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/healthcheck") +operation CheckConnection { + input: CheckConnectionInput, + output: CheckConnectionOutput +} + +structure CheckConnectionInput {} + +structure CheckConnectionOutput {} diff --git a/models/proxyBackbeatApis/commonStructures.smithy b/models/proxyBackbeatApis/commonStructures.smithy new file mode 100644 index 00000000..2a9aeef5 --- /dev/null +++ b/models/proxyBackbeatApis/commonStructures.smithy @@ -0,0 +1,21 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +map LocationStatusMap { + key: String, + value: Document +} + +structure FailedObjectVersion { + Bucket: String, + + Key: String, + + VersionId: String, + + StorageClass: String, + + Size: Integer, + + LastModified: Timestamp +} \ No newline at end of file diff --git a/models/proxyBackbeatApis/getFailedObject.smithy b/models/proxyBackbeatApis/getFailedObject.smithy new file mode 100644 index 00000000..a39491ad --- /dev/null +++ b/models/proxyBackbeatApis/getFailedObject.smithy @@ -0,0 +1,29 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/failed/{Bucket}/{Key+}") +operation GetFailedObject { + input: GetFailedObjectInput, + output: GetFailedObjectOutput +} + +structure GetFailedObjectInput { + @required + @httpLabel + Bucket: String, + + @required + @httpLabel + Key: String, + + @required + @httpQuery("versionId") + VersionId: String +} + +structure GetFailedObjectOutput { + IsTruncated: Boolean, + + Versions: FailedObjectVersions +} diff --git a/models/proxyBackbeatApis/getLocationsIngestionStatus.smithy b/models/proxyBackbeatApis/getLocationsIngestionStatus.smithy new file mode 100644 index 00000000..31ad0db5 --- /dev/null +++ b/models/proxyBackbeatApis/getLocationsIngestionStatus.smithy @@ -0,0 +1,16 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/ingestion/status") +operation GetLocationsIngestionStatus { + input: GetLocationsIngestionStatusInput, + output: GetLocationsIngestionStatusOutput +} + +structure GetLocationsIngestionStatusInput {} + +structure GetLocationsIngestionStatusOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/getLocationsStatus.smithy b/models/proxyBackbeatApis/getLocationsStatus.smithy new file mode 100644 index 00000000..b2585950 --- /dev/null +++ b/models/proxyBackbeatApis/getLocationsStatus.smithy @@ -0,0 +1,16 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/status") +operation GetLocationsStatus { + input: GetLocationsStatusInput, + output: GetLocationsStatusOutput +} + +structure GetLocationsStatusInput {} + +structure GetLocationsStatusOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/listFailed.smithy b/models/proxyBackbeatApis/listFailed.smithy new file mode 100644 index 00000000..eeb4b627 --- /dev/null +++ b/models/proxyBackbeatApis/listFailed.smithy @@ -0,0 +1,29 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/failed") +operation ListFailed { + input: ListFailedInput, + output: ListFailedOutput +} + +structure ListFailedInput { + @httpQuery("marker") + Marker: String, + + @httpQuery("sitename") + Sitename: String +} + +structure ListFailedOutput { + IsTruncated: Boolean, + + NextMarker: String, + + Versions: FailedObjectVersions +} + +list FailedObjectVersions { + member: FailedObjectVersion +} diff --git a/models/proxyBackbeatApis/pauseAllIngestionSites.smithy b/models/proxyBackbeatApis/pauseAllIngestionSites.smithy new file mode 100644 index 00000000..5ebc40d2 --- /dev/null +++ b/models/proxyBackbeatApis/pauseAllIngestionSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/pause") +operation PauseAllIngestionSites { + input: PauseAllIngestionSitesInput, + output: PauseAllIngestionSitesOutput +} + +structure PauseAllIngestionSitesInput { + @httpPayload + Body: Blob +} + +structure PauseAllIngestionSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/pauseAllSites.smithy b/models/proxyBackbeatApis/pauseAllSites.smithy new file mode 100644 index 00000000..ea1a809b --- /dev/null +++ b/models/proxyBackbeatApis/pauseAllSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/pause") +operation PauseAllSites { + input: PauseAllSitesInput, + output: PauseAllSitesOutput +} + +structure PauseAllSitesInput { + @httpPayload + Body: Blob +} + +structure PauseAllSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/pauseIngestionSite.smithy b/models/proxyBackbeatApis/pauseIngestionSite.smithy new file mode 100644 index 00000000..8201e469 --- /dev/null +++ b/models/proxyBackbeatApis/pauseIngestionSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/pause/{Site}") +operation PauseIngestionSite { + input: PauseIngestionSiteInput, + output: PauseIngestionSiteOutput +} + +structure PauseIngestionSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure PauseIngestionSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/pauseSite.smithy b/models/proxyBackbeatApis/pauseSite.smithy new file mode 100644 index 00000000..1e8d3bb2 --- /dev/null +++ b/models/proxyBackbeatApis/pauseSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/pause/{Site}") +operation PauseSite { + input: PauseSiteInput, + output: PauseSiteOutput +} + +structure PauseSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure PauseSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeAllIngestionSites.smithy b/models/proxyBackbeatApis/resumeAllIngestionSites.smithy new file mode 100644 index 00000000..ba1690ae --- /dev/null +++ b/models/proxyBackbeatApis/resumeAllIngestionSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume") +operation ResumeAllIngestionSites { + input: ResumeAllIngestionSitesInput, + output: ResumeAllIngestionSitesOutput +} + +structure ResumeAllIngestionSitesInput { + @httpPayload + Body: Blob +} + +structure ResumeAllIngestionSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeAllSites.smithy b/models/proxyBackbeatApis/resumeAllSites.smithy new file mode 100644 index 00000000..b84d0d16 --- /dev/null +++ b/models/proxyBackbeatApis/resumeAllSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/resume") +operation ResumeAllSites { + input: ResumeAllSitesInput, + output: ResumeAllSitesOutput +} + +structure ResumeAllSitesInput { + @httpPayload + Body: Blob +} + +structure ResumeAllSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeIngestionSite.smithy b/models/proxyBackbeatApis/resumeIngestionSite.smithy new file mode 100644 index 00000000..e3fcc38b --- /dev/null +++ b/models/proxyBackbeatApis/resumeIngestionSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume/{Site}") +operation ResumeIngestionSite { + input: ResumeIngestionSiteInput, + output: ResumeIngestionSiteOutput +} + +structure ResumeIngestionSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ResumeIngestionSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeSite.smithy b/models/proxyBackbeatApis/resumeSite.smithy new file mode 100644 index 00000000..6f2ee7e1 --- /dev/null +++ b/models/proxyBackbeatApis/resumeSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/resume/{Site}") +operation ResumeSite { + input: ResumeSiteInput, + output: ResumeSiteOutput +} + +structure ResumeSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ResumeSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/retryFailedObjects.smithy b/models/proxyBackbeatApis/retryFailedObjects.smithy new file mode 100644 index 00000000..98f52cb3 --- /dev/null +++ b/models/proxyBackbeatApis/retryFailedObjects.smithy @@ -0,0 +1,19 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/failed") +operation RetryFailedObjects { + input: RetryFailedObjectsInput, + output: RetryFailedObjectsOutput +} + +structure RetryFailedObjectsInput { + @required + @httpPayload + Body: Blob +} + +structure RetryFailedObjectsOutput { + @httpPayload + Results: Document +} diff --git a/models/proxyBackbeatApis/scheduleIngestionSiteResume.smithy b/models/proxyBackbeatApis/scheduleIngestionSiteResume.smithy new file mode 100644 index 00000000..a4878a24 --- /dev/null +++ b/models/proxyBackbeatApis/scheduleIngestionSiteResume.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume/{Site}/schedule") +operation ScheduleIngestionSiteResume { + input: ScheduleIngestionSiteResumeInput, + output: ScheduleIngestionSiteResumeOutput +} + +structure ScheduleIngestionSiteResumeInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ScheduleIngestionSiteResumeOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/scheduleSiteResume.smithy b/models/proxyBackbeatApis/scheduleSiteResume.smithy new file mode 100644 index 00000000..ef37da42 --- /dev/null +++ b/models/proxyBackbeatApis/scheduleSiteResume.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/resume/{Site}/schedule") +operation ScheduleSiteResume { + input: ScheduleSiteResumeInput, + output: ScheduleSiteResumeOutput +} + +structure ScheduleSiteResumeInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ScheduleSiteResumeOutput { + @httpPayload + status: Document +} diff --git a/package.json b/package.json index ec82a98c..576e0799 100644 --- a/package.json +++ b/package.json @@ -30,11 +30,13 @@ "build:smithy": "smithy build", "build:generated:backbeatRoutes": "cd build/smithy/cloudserverBackbeatRoutes/typescript-codegen && yarn install && yarn build", "build:generated:bucketQuota": "cd build/smithy/cloudserverBucketQuota/typescript-codegen && yarn install && yarn build", + "build:generated:proxyBackbeatApis": "cd build/smithy/cloudserverProxyBackbeatApis/typescript-codegen && yarn install && yarn build", "build:wrapper": "tsc", - "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated:backbeatRoutes && yarn build:generated:bucketQuota && yarn build:wrapper", + "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated:backbeatRoutes && yarn build:generated:bucketQuota && yarn build:generated:proxyBackbeatApis && yarn build:wrapper", "test": "jest", "test:mongo-backend": "BACKEND_TYPE=mongo jest", "test:metadata-backend": "BACKEND_TYPE=metadata jest", + "test:backbeat-apis": "BACKBEAT_SETUP=true yarn jest tests/testBackbeatProxyApis.test.ts", "lint": "eslint src tests", "typecheck": "tsc --noEmit" }, diff --git a/service/cloudserverProxyBackbeatApis.smithy b/service/cloudserverProxyBackbeatApis.smithy new file mode 100644 index 00000000..31b6425c --- /dev/null +++ b/service/cloudserverProxyBackbeatApis.smithy @@ -0,0 +1,32 @@ +$version: "2.0" + +namespace cloudserver.proxyBackbeatApis + +use aws.protocols#restJson1 +use aws.auth#sigv4 +use aws.api#service + +@restJson1 +@sigv4(name: "s3") +@service(sdkId: "CloudserverProxyBackbeatApis") +service CloudserverProxyBackbeatApis { + version: "2017-07-01", + operations: [ + CheckConnection, + GetFailedObject, + GetLocationsIngestionStatus, + GetLocationsStatus, + ListFailed, + PauseAllIngestionSites, + PauseAllSites, + PauseIngestionSite, + PauseSite, + ResumeAllIngestionSites, + ResumeAllSites, + ResumeIngestionSite, + ResumeSite, + RetryFailedObjects, + ScheduleIngestionSiteResume, + ScheduleSiteResume, + ] +} diff --git a/smithy-build.json b/smithy-build.json index 00589e82..78886c2f 100644 --- a/smithy-build.json +++ b/smithy-build.json @@ -25,6 +25,15 @@ "packageVersion": "1.0.0" } } + }, + "cloudserverProxyBackbeatApis": { + "plugins": { + "typescript-codegen": { + "service": "cloudserver.proxyBackbeatApis#CloudserverProxyBackbeatApis", + "package": "@scality/cloudserverclient-proxy-backbeat", + "packageVersion": "1.0.0" + } + } } } } diff --git a/src/clients/proxyBackbeatApis.ts b/src/clients/proxyBackbeatApis.ts new file mode 100644 index 00000000..2328baa8 --- /dev/null +++ b/src/clients/proxyBackbeatApis.ts @@ -0,0 +1,64 @@ +import { + CloudserverProxyBackbeatApisClient, + CloudserverProxyBackbeatApisClientConfig, +} from '../../build/smithy/cloudserverProxyBackbeatApis/typescript-codegen'; + +export { CloudserverProxyBackbeatApisClientConfig }; +export { + CheckConnectionCommand, + GetFailedObjectCommand, + GetLocationsIngestionStatusCommand, + GetLocationsStatusCommand, + ListFailedCommand, + PauseAllIngestionSitesCommand, + PauseAllSitesCommand, + PauseIngestionSiteCommand, + PauseSiteCommand, + ResumeAllIngestionSitesCommand, + ResumeAllSitesCommand, + ResumeIngestionSiteCommand, + ResumeSiteCommand, + RetryFailedObjectsCommand, + ScheduleIngestionSiteResumeCommand, + ScheduleSiteResumeCommand, +} from '../../build/smithy/cloudserverProxyBackbeatApis/typescript-codegen'; +export type { + CheckConnectionCommandInput, + CheckConnectionCommandOutput, + GetFailedObjectCommandInput, + GetFailedObjectCommandOutput, + GetLocationsIngestionStatusCommandInput, + GetLocationsIngestionStatusCommandOutput, + GetLocationsStatusCommandInput, + GetLocationsStatusCommandOutput, + ListFailedCommandInput, + ListFailedCommandOutput, + PauseAllIngestionSitesCommandInput, + PauseAllIngestionSitesCommandOutput, + PauseAllSitesCommandInput, + PauseAllSitesCommandOutput, + PauseIngestionSiteCommandInput, + PauseIngestionSiteCommandOutput, + PauseSiteCommandInput, + PauseSiteCommandOutput, + ResumeAllIngestionSitesCommandInput, + ResumeAllIngestionSitesCommandOutput, + ResumeAllSitesCommandInput, + ResumeAllSitesCommandOutput, + ResumeIngestionSiteCommandInput, + ResumeIngestionSiteCommandOutput, + ResumeSiteCommandInput, + ResumeSiteCommandOutput, + RetryFailedObjectsCommandInput, + RetryFailedObjectsCommandOutput, + ScheduleIngestionSiteResumeCommandInput, + ScheduleIngestionSiteResumeCommandOutput, + ScheduleSiteResumeCommandInput, + ScheduleSiteResumeCommandOutput, +} from '../../build/smithy/cloudserverProxyBackbeatApis/typescript-codegen'; + +export class ProxyBackbeatApisClient extends CloudserverProxyBackbeatApisClient { + constructor(config: CloudserverProxyBackbeatApisClientConfig) { + super(config); + } +} diff --git a/src/index.ts b/src/index.ts index e377ec7d..85656164 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ export * from './clients/backbeatRoutes'; export * from './clients/bucketQuota'; +export * from './clients/proxyBackbeatApis'; export * from './clients/s3Extended'; export { CloudserverClient, CloudserverClientConfig } from './clients/cloudserver'; export * from './utils'; diff --git a/tests/testBackbeatProxyApis.test.ts b/tests/testBackbeatProxyApis.test.ts new file mode 100644 index 00000000..cf4242e0 --- /dev/null +++ b/tests/testBackbeatProxyApis.test.ts @@ -0,0 +1,194 @@ +import { + ProxyBackbeatApisClient, + CheckConnectionCommandInput, + CheckConnectionCommand, + GetLocationsStatusCommandInput, + GetLocationsStatusCommand, + GetLocationsIngestionStatusCommandInput, + GetLocationsIngestionStatusCommand, + ListFailedCommandInput, + ListFailedCommand, + GetFailedObjectCommandInput, + GetFailedObjectCommand, + RetryFailedObjectsCommand, + PauseAllSitesCommandInput, + PauseAllSitesCommand, + PauseAllIngestionSitesCommandInput, + PauseAllIngestionSitesCommand, + PauseSiteCommandInput, + PauseSiteCommand, + PauseIngestionSiteCommandInput, + PauseIngestionSiteCommand, + ResumeAllSitesCommandInput, + ResumeAllSitesCommand, + ResumeAllIngestionSitesCommandInput, + ResumeAllIngestionSitesCommand, + ResumeSiteCommandInput, + ResumeSiteCommand, + ResumeIngestionSiteCommandInput, + ResumeIngestionSiteCommand, + ScheduleSiteResumeCommandInput, + ScheduleSiteResumeCommand, + ScheduleIngestionSiteResumeCommandInput, + ScheduleIngestionSiteResumeCommand, +} from '../src/clients/proxyBackbeatApis'; +import { createTestClient, testConfig } from './testSetup'; +import { describeForBackbeatSetup } from './testHelpers'; +import assert from 'assert'; + +// Note : these tests are relatively simple, as it's not straightforward to setup +// backbeat with locations, and real pause/resume tests scenarios. +// The tests in Zenko are also using these apis, and will represent an opportunity to +// test this client better : https://scality.atlassian.net/browse/ZENKO-5102 +describeForBackbeatSetup('CloudServer Check Status API Tests', () => { + let proxyBackbeatApisClient: ProxyBackbeatApisClient; + + beforeAll(() => { + ({proxyBackbeatApisClient} = createTestClient()); + }); + + it('should test checkConnection', async () => { + const input: CheckConnectionCommandInput = {}; + const cmd = new CheckConnectionCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test GetLocationsStatus', async () => { + const input: GetLocationsStatusCommandInput = {}; + const cmd = new GetLocationsStatusCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual((result.status as Record)?.['wontwork-location'], 'enabled'); + }); + + it('should test GetLocationsIngestionStatus', async () => { + const input: GetLocationsIngestionStatusCommandInput = {}; + const cmd = new GetLocationsIngestionStatusCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test listFailed', async () => { + const input: ListFailedCommandInput = {}; + const cmd = new ListFailedCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test getFailedObject', async () => { + const input: GetFailedObjectCommandInput = { + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + VersionId: testConfig.versionID, + }; + const cmd = new GetFailedObjectCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test retryFailedObjects', async () => { + const bodyData = JSON.stringify([ + { + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + VersionId: testConfig.versionID, + StorageClass: 'STANDARD', + } + ]); + const cmd = new RetryFailedObjectsCommand({ + Body: new TextEncoder().encode(bodyData) + }); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseAllSites', async () => { + const input: PauseAllSitesCommandInput = {}; + const cmd = new PauseAllSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseAllIngestionSites', async () => { + const input: PauseAllIngestionSitesCommandInput = {}; + const cmd = new PauseAllIngestionSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseSite', async () => { + const input: PauseSiteCommandInput = { + Site: 'zenko', // Value from backbeat default config.json, destination.bootstrapList + }; + const cmd = new PauseSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test pauseIngestionSite', async () => { + const input: PauseIngestionSiteCommandInput = { + Site: 'a-zenko-location', // Value from backbeat default config.json, extensions.ingestion.sources + }; + const cmd = new PauseIngestionSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeAllSites', async () => { + const input: ResumeAllSitesCommandInput = {}; + const cmd = new ResumeAllSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeAllIngestionSites', async () => { + const input: ResumeAllIngestionSitesCommandInput = {}; + const cmd = new ResumeAllIngestionSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeSite', async () => { + const input: ResumeSiteCommandInput = { + Site: 'zenko', + }; + const cmd = new ResumeSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test resumeIngestionSite', async () => { + const input: ResumeIngestionSiteCommandInput = { + Site: 'a-zenko-location', + }; + const cmd = new ResumeIngestionSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test scheduleSiteResume', async () => { + const bodyData = JSON.stringify({ hours: 24 }); + const input: ScheduleSiteResumeCommandInput = { + Site: 'zenko', + Body: new TextEncoder().encode(bodyData) + }; + const cmd = new ScheduleSiteResumeCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test scheduleIngestionSiteResume', async () => { + const bodyData = JSON.stringify({ hours: 24 }); + const input: ScheduleIngestionSiteResumeCommandInput = { + Site: 'zenko', + Body: new TextEncoder().encode(bodyData) + }; + const cmd = new ScheduleIngestionSiteResumeCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); +}); diff --git a/tests/testHelpers.ts b/tests/testHelpers.ts index 70be4765..23edf0a1 100644 --- a/tests/testHelpers.ts +++ b/tests/testHelpers.ts @@ -18,3 +18,11 @@ export function describeForMetadataBackend(name: string, fn: () => void): void { describe.skip(`${name} (tests skipped: metadata backend only)`, fn); } } + +export function describeForBackbeatSetup(name: string, fn: () => void): void { + if (process.env.BACKBEAT_SETUP === 'true') { + describe(name, fn); + } else { + describe.skip(`${name} (tests skipped: backbeat setup required)`, fn); + } +} diff --git a/tests/testSetup.ts b/tests/testSetup.ts index 383a76e2..d2d51dd6 100644 --- a/tests/testSetup.ts +++ b/tests/testSetup.ts @@ -4,6 +4,7 @@ import { BackbeatRoutesClient, CloudserverBackbeatRoutesClientConfig } from '../ import { S3Client, PutObjectCommand, CreateBucketCommand, PutBucketVersioningCommand } from '@aws-sdk/client-s3'; import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@aws-sdk/types'; import { BucketQuotaClient } from '../src/clients/bucketQuota'; +import { ProxyBackbeatApisClient } from '../src/clients/proxyBackbeatApis'; jest.setTimeout(30000); const credentialsProvider: AwsCredentialIdentityProvider = async (): Promise => ({ @@ -48,6 +49,7 @@ export const testConfig = { objectKey: `test-cloudserverclient-object-sla/sh${randomId()}`, objectData: 'iAmSomeData', canonicalID: '39383234313039353433383937313939393939395247303031202036353034352e30', + versionID: '' }; async function initBucketForTests() { @@ -70,7 +72,8 @@ async function initBucketForTests() { Key: testConfig.objectKey, Body: testConfig.objectData, }); - await s3client.send(putObjectCommand); + const result = await s3client.send(putObjectCommand); + testConfig.versionID = result.VersionId!; } catch (error: any) { assert.fail(`Failed to initialize bucket for tests: ${error}`); } @@ -79,11 +82,13 @@ async function initBucketForTests() { export function createTestClient(): { backbeatRoutesClient: BackbeatRoutesClient, bucketQuotaClient: BucketQuotaClient, + proxyBackbeatApisClient: ProxyBackbeatApisClient, s3client: S3Client } { return { backbeatRoutesClient: new BackbeatRoutesClient(config), bucketQuotaClient: new BucketQuotaClient(config), + proxyBackbeatApisClient: new ProxyBackbeatApisClient(config), s3client, }; } From 486a3d2b679111bfd81d9a9ec5a9ee6442dadbfc Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Fri, 16 Jan 2026 16:51:20 +0100 Subject: [PATCH 2/5] use profile compose, refactoring, remove queue processor ISSUE: CLDSRVCLT-5 --- .github/authdata.json | 21 -- .github/docker-compose.backbeat.yml | 191 ------------------ .../docker-compose.cloudserver-metadata.yml | 20 -- .github/docker-compose.cloudserver-mongo.yml | 46 ----- .github/docker-compose.yml | 182 +++++++++++++++++ .github/scripts/wait_for_local_port.bash | 0 .github/workflows/test.yml | 32 +-- 7 files changed, 188 insertions(+), 304 deletions(-) delete mode 100644 .github/authdata.json delete mode 100644 .github/docker-compose.backbeat.yml delete mode 100644 .github/docker-compose.cloudserver-metadata.yml delete mode 100644 .github/docker-compose.cloudserver-mongo.yml create mode 100644 .github/docker-compose.yml mode change 100755 => 100644 .github/scripts/wait_for_local_port.bash diff --git a/.github/authdata.json b/.github/authdata.json deleted file mode 100644 index 36b2bf85..00000000 --- a/.github/authdata.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "accounts": [{ - "name": "bart", - "arn": "aws::iam:123456789012:root", - "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", - "displayName": "bart", - "keys": { - "access": "accessKey1", - "secret": "verySecretKey1" - } - }, { - "name": "lisa", - "arn": "aws::iam:123456789013:root", - "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", - "displayName": "lisa", - "keys": { - "access": "accessKey2", - "secret": "verySecretKey2" - } - }] -} diff --git a/.github/docker-compose.backbeat.yml b/.github/docker-compose.backbeat.yml deleted file mode 100644 index 8807bed1..00000000 --- a/.github/docker-compose.backbeat.yml +++ /dev/null @@ -1,191 +0,0 @@ -version: '3.8' - -services: - zookeeper: - image: zookeeper:3.8 - platform: linux/amd64 - ports: - - "127.0.0.1:2181:2181" - environment: - - ALLOW_ANONYMOUS_LOGIN=yes - - kafka: - image: confluentinc/cp-kafka:7.4.0 - platform: linux/amd64 - ports: - - "127.0.0.1:9092:9092" - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_BROKER_ID: 1 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - depends_on: - - zookeeper - - mongodb: - image: mongo:4.4 - platform: linux/amd64 - ports: - - "127.0.0.1:27018:27018" - command: mongod --port 27018 --replSet rs0 --bind_ip_all - healthcheck: - test: echo 'db.runCommand("ping").ok' | mongo --port 27018 --quiet - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - - mongodb-init: - image: mongo:4.4 - platform: linux/amd64 - depends_on: - mongodb: - condition: service_healthy - command: > - mongo --host mongodb:27018 --eval - 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})' - restart: "no" - - redis: - image: redis:7-alpine - platform: linux/amd64 - ports: - - "127.0.0.1:6379:6379" - - cloudserver: - image: ghcr.io/scality/cloudserver:9.1.4 - platform: linux/amd64 - ports: - - "127.0.0.1:8000:8000" - environment: - - S3VAULT=mem - - S3METADATA=mongodb - - S3DATA=mem - - MONGODB_HOSTS=mongodb:27018 - - MONGODB_RS=rs0 - - REMOTE_MANAGEMENT_DISABLE=true - - LOG_LEVEL=info - - CRR_METRICS_HOST=backbeat-api - - CRR_METRICS_PORT=8900 - depends_on: - mongodb-init: - condition: service_completed_successfully - backbeat-api: - condition: service_started - - backbeat-api: - image: ghcr.io/scality/backbeat:9.1.3 - platform: linux/amd64 - ports: - - "127.0.0.1:8900:8900" - environment: - # Kafka configuration - KAFKA_HOSTS: kafka:9092 - KAFKA_COMPRESSION_TYPE: snappy - - # Zookeeper configuration - ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat - ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true" - - # Redis configuration - REDIS_HOST: redis - REDIS_PORT: 6379 - - # MongoDB configuration - MONGODB_HOSTS: mongodb:27018 - MONGODB_DATABASE: metadata - - # Management backend - MANAGEMENT_BACKEND: operator - REMOTE_MANAGEMENT_DISABLE: "true" - - # Cloudserver configuration - CLOUDSERVER_HOST: cloudserver - CLOUDSERVER_PORT: 8000 - - # Replication source S3 - EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver - EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000 - EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service - EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication - - # Replication destination - EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service - EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001 - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}' - - # Lifecycle - EXTENSIONS_LIFECYCLE_AUTH_TYPE: service - EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT: service-lifecycle - - # Healthchecks - HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0" - - # Logging - LOG_LEVEL: info - depends_on: - mongodb-init: - condition: service_completed_successfully - kafka: - condition: service_started - redis: - condition: service_started - command: node bin/backbeat.js - - backbeat-queue-processor: - image: ghcr.io/scality/backbeat:9.1.3 - platform: linux/amd64 - ports: - - "127.0.0.1:4043:4043" - environment: - # Kafka configuration - KAFKA_HOSTS: kafka:9092 - KAFKA_COMPRESSION_TYPE: snappy - - # Zookeeper configuration - ZOOKEEPER_CONNECTION_STRING: zookeeper:2181/backbeat - ZOOKEEPER_AUTO_CREATE_NAMESPACE: "true" - - # Redis configuration - REDIS_HOST: redis - REDIS_PORT: 6379 - - # MongoDB configuration - MONGODB_HOSTS: mongodb:27018 - MONGODB_DATABASE: metadata - - # Management backend - MANAGEMENT_BACKEND: operator - REMOTE_MANAGEMENT_DISABLE: "true" - - # Cloudserver configuration - CLOUDSERVER_HOST: cloudserver - CLOUDSERVER_PORT: 8000 - - # Replication source S3 - EXTENSIONS_REPLICATION_SOURCE_S3_HOST: cloudserver - EXTENSIONS_REPLICATION_SOURCE_S3_PORT: 8000 - EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: service - EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: service-replication - - # Replication destination - EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: service - EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: service-replication - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: cloudserver:8001 - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{"site": "wontwork-location", "type": "aws_s3"}, {"site": "aws-location", "type": "aws_s3"}' - - # Probe server port - LIVENESS_PROBE_PORT: 4043 - - # Logging - LOG_LEVEL: info - depends_on: - mongodb-init: - condition: service_completed_successfully - cloudserver: - condition: service_started - backbeat-api: - condition: service_started - command: node extensions/replication/queueProcessor/task.js diff --git a/.github/docker-compose.cloudserver-metadata.yml b/.github/docker-compose.cloudserver-metadata.yml deleted file mode 100644 index 52abc17f..00000000 --- a/.github/docker-compose.cloudserver-metadata.yml +++ /dev/null @@ -1,20 +0,0 @@ -services: - metadata-standalone: - image: ghcr.io/scality/metadata:8.11.0-standalone - platform: linux/amd64 - network_mode: 'host' - volumes: - - ./md-config.json:/mnt/standalone_workdir/config.json:ro - - cloudserver-metadata: - image: ghcr.io/scality/cloudserver:9.3.0-preview.1 - platform: linux/amd64 - network_mode: 'host' - environment: - - S3VAULT=mem - - S3METADATA=scality - - S3DATA=mem - - REMOTE_MANAGEMENT_DISABLE=true - - LOG_LEVEL=info - depends_on: - - metadata-standalone diff --git a/.github/docker-compose.cloudserver-mongo.yml b/.github/docker-compose.cloudserver-mongo.yml deleted file mode 100644 index 7f208901..00000000 --- a/.github/docker-compose.cloudserver-mongo.yml +++ /dev/null @@ -1,46 +0,0 @@ -services: - mongodb: - image: mongo:5.0 - platform: linux/amd64 - command: --replSet rs0 --port 27018 --bind_ip_all - ports: - - "27018:27018" - healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh --port 27018 --quiet - interval: 5s - timeout: 10s - retries: 5 - start_period: 10s - - mongo-init: - image: mongo:5.0 - platform: linux/amd64 - depends_on: - mongodb: - condition: service_healthy - command: > - mongosh --host mongodb:27018 --eval - 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})' - restart: "no" - - cloudserver: - image: ghcr.io/scality/cloudserver:9.3.0-preview.1 - platform: linux/amd64 - ports: - - "8000:8000" - environment: - - S3VAULT=mem - - S3METADATA=mongodb - - S3DATA=mem - - MONGODB_HOSTS=mongodb:27018 - - MONGODB_RS=rs0 - - REMOTE_MANAGEMENT_DISABLE=true - - LOG_LEVEL=info - depends_on: - mongo-init: - condition: service_completed_successfully - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/"] - interval: 5s - timeout: 5s - retries: 12 diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml new file mode 100644 index 00000000..099c9e6a --- /dev/null +++ b/.github/docker-compose.yml @@ -0,0 +1,182 @@ +services: + # =================== + # MongoDB + # =================== + mongodb: + image: mongo:4.4 + command: mongod --replSet rs0 --port 27018 --bind_ip_all + ports: + - "127.0.0.1:27018:27018" + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo --port 27018 --quiet + interval: 5s + timeout: 10s + retries: 5 + start_period: 10s + profiles: + - mongo + - backbeat + + mongodb-init: + image: mongo:4.4 + depends_on: + mongodb: + condition: service_healthy + command: > + mongo --host mongodb:27018 --eval + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})' + restart: "no" + profiles: + - mongo + - backbeat + + # =================== + # Cloudserver with MongoDB backend + # =================== + cloudserver: + image: ghcr.io/scality/cloudserver:9.3.0-preview.1 + ports: + - "127.0.0.1:8000:8000" + environment: + - S3VAULT=mem + - S3METADATA=mongodb + - S3DATA=mem + - MONGODB_HOSTS=mongodb:27018 + - MONGODB_RS=rs0 + - REMOTE_MANAGEMENT_DISABLE=true + - LOG_LEVEL=info + - CRR_METRICS_HOST=backbeat-api + - CRR_METRICS_PORT=8900 + depends_on: + mongodb-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:8000/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] + interval: 5s + timeout: 5s + retries: 12 + profiles: + - mongo + - backbeat + + # =================== + # Metadata backend services + # =================== + metadata-standalone: + image: ghcr.io/scality/metadata:8.11.0-standalone + network_mode: 'host' + volumes: + - ./md-config.json:/mnt/standalone_workdir/config.json:ro + profiles: + - metadata + + cloudserver-metadata: + image: ghcr.io/scality/cloudserver:9.3.0-preview.1 + network_mode: 'host' + environment: + - S3VAULT=mem + - S3METADATA=scality + - S3DATA=mem + - REMOTE_MANAGEMENT_DISABLE=true + - LOG_LEVEL=info + depends_on: + - metadata-standalone + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:8000/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] + interval: 5s + timeout: 5s + retries: 12 + profiles: + - metadata + + # =================== + # Backbeat services + # =================== + zookeeper: + image: zookeeper:3.8 + ports: + - "127.0.0.1:2181:2181" + healthcheck: + test: ["CMD", "zkServer.sh", "status"] + interval: 10s + timeout: 5s + retries: 5 + profiles: + - backbeat + + zk-init: + image: zookeeper:3.8 + depends_on: + zookeeper: + condition: service_healthy + command: > + bash -c " + sleep 2 && + zkCli.sh -server zookeeper:2181 create /backbeat '' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication '' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/state '' && + zkCli.sh -server zookeeper:2181 create /backbeat/ingestion '' && + zkCli.sh -server zookeeper:2181 create /backbeat/ingestion/state '' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/us-east-1 '{\"paused\":false}' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/us-east-2 '{\"paused\":false}' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/wontwork-location '{\"paused\":false}' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/location-dmf-v1 '{\"paused\":false}' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/zenko '{\"paused\":false}' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/aws-location '{\"paused\":false}' && + zkCli.sh -server zookeeper:2181 create /backbeat/replication/replayState '' && + echo 'Zookeeper paths initialized successfully' + " + restart: "no" + profiles: + - backbeat + + redis: + image: redis:7-alpine + ports: + - "127.0.0.1:6379:6379" + profiles: + - backbeat + + kafka: + image: wurstmeister/kafka:latest + ports: + - "127.0.0.1:9092:9092" + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + depends_on: + zookeeper: + condition: service_healthy + profiles: + - backbeat + + backbeat-api: + image: ghcr.io/scality/backbeat:9.1.3 + ports: + - "127.0.0.1:8900:8900" + environment: + ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 + REDIS_HOST: redis + KAFKA_HOSTS: kafka:9092 + MONGODB_HOSTS: mongodb:27018 + REMOTE_MANAGEMENT_DISABLE: "1" + HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0" + depends_on: + mongodb-init: + condition: service_completed_successfully + zk-init: + condition: service_completed_successfully + kafka: + condition: service_started + redis: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:8900/_/healthcheck', r => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))\""] + interval: 5s + timeout: 5s + retries: 12 + command: node bin/backbeat.js + profiles: + - backbeat diff --git a/.github/scripts/wait_for_local_port.bash b/.github/scripts/wait_for_local_port.bash old mode 100755 new mode 100644 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c0af3a37..3d09c02d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,19 +40,14 @@ jobs: uses: ./.github/actions/setup-and-build - name: Start Cloudserver with MongoDB backend - run: docker compose -f .github/docker-compose.cloudserver-mongo.yml up -d - - - name: Wait for Cloudserver to be ready - run: | - set -o pipefail - bash .github/scripts/wait_for_local_port.bash 8000 40 + run: docker compose -f .github/docker-compose.yml --profile mongo up -d --wait - name: Run MongoDB backend tests run: yarn test:mongo-backend - name: Stop Cloudserver if: always() - run: docker compose -f .github/docker-compose.cloudserver-mongo.yml down + run: docker compose -f .github/docker-compose.yml --profile mongo down test-metadata-backend: name: Test with Scality metadata backend @@ -74,24 +69,19 @@ jobs: password: ${{ github.token }} - name: Start Cloudserver with Scality metadata backend - run: docker compose -f .github/docker-compose.cloudserver-metadata.yml up -d + run: docker compose -f .github/docker-compose.yml --profile metadata up -d --wait - name: Wait for metadata to be ready run: | set -o pipefail bash .github/scripts/wait_for_local_port.bash 9000 40 - - name: Wait for Cloudserver to be ready - run: | - set -o pipefail - bash .github/scripts/wait_for_local_port.bash 8000 60 - - name: Run metadata backend tests run: yarn test:metadata-backend - name: Stop Cloudserver if: always() - run: docker compose -f .github/docker-compose.cloudserver-metadata.yml down + run: docker compose -f .github/docker-compose.yml --profile metadata down test-backbeat-apis: name: Test backbeat apis @@ -113,21 +103,11 @@ jobs: password: ${{ github.token }} - name: Start Backbeat with CloudServer - run: docker compose -f .github/docker-compose.backbeat.yml up -d - - - name: Wait for Cloudserver to be ready - run: | - set -o pipefail - bash .github/scripts/wait_for_local_port.bash 8000 40 - - - name: Wait for Backbeat API to be ready - run: | - set -o pipefail - bash .github/scripts/wait_for_local_port.bash 8900 60 + run: docker compose -f .github/docker-compose.yml --profile backbeat up -d --wait - name: Run backbeat apis tests run: yarn test:backbeat-apis - name: Stop Backbeat and Cloudserver if: always() - run: docker compose -f .github/docker-compose.backbeat.yml down \ No newline at end of file + run: docker compose -f .github/docker-compose.yml --profile backbeat down \ No newline at end of file From 388f816840c3ea3ec154db9b6307e039a987d6be Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Mon, 19 Jan 2026 17:12:31 +0100 Subject: [PATCH 3/5] fixup --- .github/scripts/wait_for_local_port.bash | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/scripts/wait_for_local_port.bash diff --git a/.github/scripts/wait_for_local_port.bash b/.github/scripts/wait_for_local_port.bash old mode 100644 new mode 100755 From 736c492d0bcf54eb624fea808b403e1f471b085e Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Mon, 19 Jan 2026 17:02:52 +0100 Subject: [PATCH 4/5] move variables to env file ISSUE: CLDSRVCLT-5 --- .github/docker-compose.yml | 86 +++++++++++++++++++------------------- .github/docker.env | 27 ++++++++++++ .github/workflows/test.yml | 12 +++--- 3 files changed, 76 insertions(+), 49 deletions(-) create mode 100644 .github/docker.env diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml index 099c9e6a..f113037b 100644 --- a/.github/docker-compose.yml +++ b/.github/docker-compose.yml @@ -3,12 +3,12 @@ services: # MongoDB # =================== mongodb: - image: mongo:4.4 - command: mongod --replSet rs0 --port 27018 --bind_ip_all + image: ${MONGODB_IMAGE} + command: mongod --replSet rs0 --port ${MONGODB_PORT} --bind_ip_all ports: - - "127.0.0.1:27018:27018" + - "127.0.0.1:${MONGODB_PORT}:${MONGODB_PORT}" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongo --port 27018 --quiet + test: echo 'db.runCommand("ping").ok' | mongo --port ${MONGODB_PORT} --quiet interval: 5s timeout: 10s retries: 5 @@ -18,13 +18,13 @@ services: - backbeat mongodb-init: - image: mongo:4.4 + image: ${MONGODB_IMAGE} depends_on: mongodb: condition: service_healthy command: > - mongo --host mongodb:27018 --eval - 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})' + mongo --host mongodb:${MONGODB_PORT} --eval + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:${MONGODB_PORT}"}]})' restart: "no" profiles: - mongo @@ -34,24 +34,24 @@ services: # Cloudserver with MongoDB backend # =================== cloudserver: - image: ghcr.io/scality/cloudserver:9.3.0-preview.1 + image: ${CLOUDSERVER_IMAGE} ports: - - "127.0.0.1:8000:8000" + - "127.0.0.1:${CLOUDSERVER_PORT}:${CLOUDSERVER_PORT}" environment: - S3VAULT=mem - S3METADATA=mongodb - S3DATA=mem - - MONGODB_HOSTS=mongodb:27018 + - MONGODB_HOSTS=mongodb:${MONGODB_PORT} - MONGODB_RS=rs0 - REMOTE_MANAGEMENT_DISABLE=true - LOG_LEVEL=info - CRR_METRICS_HOST=backbeat-api - - CRR_METRICS_PORT=8900 + - CRR_METRICS_PORT=${BACKBEAT_PORT} depends_on: mongodb-init: condition: service_completed_successfully healthcheck: - test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:8000/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${CLOUDSERVER_PORT}/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] interval: 5s timeout: 5s retries: 12 @@ -63,7 +63,7 @@ services: # Metadata backend services # =================== metadata-standalone: - image: ghcr.io/scality/metadata:8.11.0-standalone + image: ${METADATA_IMAGE} network_mode: 'host' volumes: - ./md-config.json:/mnt/standalone_workdir/config.json:ro @@ -71,7 +71,7 @@ services: - metadata cloudserver-metadata: - image: ghcr.io/scality/cloudserver:9.3.0-preview.1 + image: ${CLOUDSERVER_IMAGE} network_mode: 'host' environment: - S3VAULT=mem @@ -82,7 +82,7 @@ services: depends_on: - metadata-standalone healthcheck: - test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:8000/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${CLOUDSERVER_PORT}/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] interval: 5s timeout: 5s retries: 12 @@ -93,9 +93,9 @@ services: # Backbeat services # =================== zookeeper: - image: zookeeper:3.8 + image: ${ZOOKEEPER_IMAGE} ports: - - "127.0.0.1:2181:2181" + - "127.0.0.1:${ZOOKEEPER_PORT}:${ZOOKEEPER_PORT}" healthcheck: test: ["CMD", "zkServer.sh", "status"] interval: 10s @@ -105,25 +105,25 @@ services: - backbeat zk-init: - image: zookeeper:3.8 + image: ${ZOOKEEPER_IMAGE} depends_on: zookeeper: condition: service_healthy command: > bash -c " sleep 2 && - zkCli.sh -server zookeeper:2181 create /backbeat '' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication '' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/state '' && - zkCli.sh -server zookeeper:2181 create /backbeat/ingestion '' && - zkCli.sh -server zookeeper:2181 create /backbeat/ingestion/state '' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/us-east-1 '{\"paused\":false}' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/us-east-2 '{\"paused\":false}' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/wontwork-location '{\"paused\":false}' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/location-dmf-v1 '{\"paused\":false}' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/zenko '{\"paused\":false}' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/state/aws-location '{\"paused\":false}' && - zkCli.sh -server zookeeper:2181 create /backbeat/replication/replayState '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/ingestion '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/ingestion/state '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/us-east-1 '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/us-east-2 '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/wontwork-location '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/location-dmf-v1 '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/zenko '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/aws-location '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/replayState '' && echo 'Zookeeper paths initialized successfully' " restart: "no" @@ -131,20 +131,20 @@ services: - backbeat redis: - image: redis:7-alpine + image: ${REDIS_IMAGE} ports: - - "127.0.0.1:6379:6379" + - "127.0.0.1:${REDIS_PORT}:${REDIS_PORT}" profiles: - backbeat kafka: - image: wurstmeister/kafka:latest + image: ${KAFKA_IMAGE} ports: - - "127.0.0.1:9092:9092" + - "127.0.0.1:${KAFKA_PORT}:${KAFKA_PORT}" environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:${ZOOKEEPER_PORT} + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:${KAFKA_PORT} + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:${KAFKA_PORT} KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 depends_on: zookeeper: @@ -153,14 +153,14 @@ services: - backbeat backbeat-api: - image: ghcr.io/scality/backbeat:9.1.3 + image: ${BACKBEAT_IMAGE} ports: - - "127.0.0.1:8900:8900" + - "127.0.0.1:${BACKBEAT_PORT}:${BACKBEAT_PORT}" environment: - ZOOKEEPER_CONNECTION_STRING: zookeeper:2181 + ZOOKEEPER_CONNECTION_STRING: zookeeper:${ZOOKEEPER_PORT} REDIS_HOST: redis - KAFKA_HOSTS: kafka:9092 - MONGODB_HOSTS: mongodb:27018 + KAFKA_HOSTS: kafka:${KAFKA_PORT} + MONGODB_HOSTS: mongodb:${MONGODB_PORT} REMOTE_MANAGEMENT_DISABLE: "1" HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0" depends_on: @@ -173,7 +173,7 @@ services: redis: condition: service_started healthcheck: - test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:8900/_/healthcheck', r => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))\""] + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${BACKBEAT_PORT}/_/healthcheck', r => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))\""] interval: 5s timeout: 5s retries: 12 diff --git a/.github/docker.env b/.github/docker.env new file mode 100644 index 00000000..18a1cc6f --- /dev/null +++ b/.github/docker.env @@ -0,0 +1,27 @@ +# MongoDB +MONGODB_IMAGE=mongo:4.4 +MONGODB_PORT=27018 + +# Cloudserver +CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.0-preview.1 +CLOUDSERVER_PORT=8000 + +# Metadata +METADATA_IMAGE=ghcr.io/scality/metadata:8.11.0-standalone +METADATA_PORT=9000 + +# Zookeeper +ZOOKEEPER_IMAGE=zookeeper:3.8 +ZOOKEEPER_PORT=2181 + +# Kafka +KAFKA_IMAGE=wurstmeister/kafka:latest +KAFKA_PORT=9092 + +# Redis +REDIS_IMAGE=redis:7-alpine +REDIS_PORT=6379 + +# Backbeat +BACKBEAT_IMAGE=ghcr.io/scality/backbeat:9.1.3 +BACKBEAT_PORT=8900 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d09c02d..ac0a0e5d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,14 +40,14 @@ jobs: uses: ./.github/actions/setup-and-build - name: Start Cloudserver with MongoDB backend - run: docker compose -f .github/docker-compose.yml --profile mongo up -d --wait + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile mongo up -d --wait - name: Run MongoDB backend tests run: yarn test:mongo-backend - name: Stop Cloudserver if: always() - run: docker compose -f .github/docker-compose.yml --profile mongo down + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile mongo down test-metadata-backend: name: Test with Scality metadata backend @@ -69,7 +69,7 @@ jobs: password: ${{ github.token }} - name: Start Cloudserver with Scality metadata backend - run: docker compose -f .github/docker-compose.yml --profile metadata up -d --wait + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile metadata up -d --wait - name: Wait for metadata to be ready run: | @@ -81,7 +81,7 @@ jobs: - name: Stop Cloudserver if: always() - run: docker compose -f .github/docker-compose.yml --profile metadata down + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile metadata down test-backbeat-apis: name: Test backbeat apis @@ -103,11 +103,11 @@ jobs: password: ${{ github.token }} - name: Start Backbeat with CloudServer - run: docker compose -f .github/docker-compose.yml --profile backbeat up -d --wait + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile backbeat up -d --wait - name: Run backbeat apis tests run: yarn test:backbeat-apis - name: Stop Backbeat and Cloudserver if: always() - run: docker compose -f .github/docker-compose.yml --profile backbeat down \ No newline at end of file + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile backbeat down \ No newline at end of file From a3b366612a0ffd3d12db13465b2a394c4aa654fe Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Wed, 14 Jan 2026 18:47:17 +0100 Subject: [PATCH 5/5] bump version to 1.0.2 ISSUE: CLDSRVCLT-5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 576e0799..78486a65 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@scality/cloudserverclient", - "version": "1.0.1", + "version": "1.0.2", "engines": { "node": ">=20" },