diff --git a/.github/docker-compose.cloudserver-metadata.yml b/.github/docker-compose.cloudserver-metadata.yml deleted file mode 100644 index 52abc17f..00000000 --- a/.github/docker-compose.cloudserver-metadata.yml +++ /dev/null @@ -1,20 +0,0 @@ -services: - metadata-standalone: - image: ghcr.io/scality/metadata:8.11.0-standalone - platform: linux/amd64 - network_mode: 'host' - volumes: - - ./md-config.json:/mnt/standalone_workdir/config.json:ro - - cloudserver-metadata: - image: ghcr.io/scality/cloudserver:9.3.0-preview.1 - platform: linux/amd64 - network_mode: 'host' - environment: - - S3VAULT=mem - - S3METADATA=scality - - S3DATA=mem - - REMOTE_MANAGEMENT_DISABLE=true - - LOG_LEVEL=info - depends_on: - - metadata-standalone diff --git a/.github/docker-compose.cloudserver-mongo.yml b/.github/docker-compose.cloudserver-mongo.yml deleted file mode 100644 index 7f208901..00000000 --- a/.github/docker-compose.cloudserver-mongo.yml +++ /dev/null @@ -1,46 +0,0 @@ -services: - mongodb: - image: mongo:5.0 - platform: linux/amd64 - command: --replSet rs0 --port 27018 --bind_ip_all - ports: - - "27018:27018" - healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh --port 27018 --quiet - interval: 5s - timeout: 10s - retries: 5 - start_period: 10s - - mongo-init: - image: mongo:5.0 - platform: linux/amd64 - depends_on: - mongodb: - condition: service_healthy - command: > - mongosh --host mongodb:27018 --eval - 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:27018"}]})' - restart: "no" - - cloudserver: - image: ghcr.io/scality/cloudserver:9.3.0-preview.1 - platform: linux/amd64 - ports: - - "8000:8000" - environment: - - S3VAULT=mem - - S3METADATA=mongodb - - S3DATA=mem - - MONGODB_HOSTS=mongodb:27018 - - MONGODB_RS=rs0 - - REMOTE_MANAGEMENT_DISABLE=true - - LOG_LEVEL=info - depends_on: - mongo-init: - condition: service_completed_successfully - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/"] - interval: 5s - timeout: 5s - retries: 12 diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml new file mode 100644 index 00000000..f113037b --- /dev/null +++ b/.github/docker-compose.yml @@ -0,0 +1,182 @@ +services: + # =================== + # MongoDB + # =================== + mongodb: + image: ${MONGODB_IMAGE} + command: mongod --replSet rs0 --port ${MONGODB_PORT} --bind_ip_all + ports: + - "127.0.0.1:${MONGODB_PORT}:${MONGODB_PORT}" + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo --port ${MONGODB_PORT} --quiet + interval: 5s + timeout: 10s + retries: 5 + start_period: 10s + profiles: + - mongo + - backbeat + + mongodb-init: + image: ${MONGODB_IMAGE} + depends_on: + mongodb: + condition: service_healthy + command: > + mongo --host mongodb:${MONGODB_PORT} --eval + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:${MONGODB_PORT}"}]})' + restart: "no" + profiles: + - mongo + - backbeat + + # =================== + # Cloudserver with MongoDB backend + # =================== + cloudserver: + image: ${CLOUDSERVER_IMAGE} + ports: + - "127.0.0.1:${CLOUDSERVER_PORT}:${CLOUDSERVER_PORT}" + environment: + - S3VAULT=mem + - S3METADATA=mongodb + - S3DATA=mem + - MONGODB_HOSTS=mongodb:${MONGODB_PORT} + - MONGODB_RS=rs0 + - REMOTE_MANAGEMENT_DISABLE=true + - LOG_LEVEL=info + - CRR_METRICS_HOST=backbeat-api + - CRR_METRICS_PORT=${BACKBEAT_PORT} + depends_on: + mongodb-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${CLOUDSERVER_PORT}/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] + interval: 5s + timeout: 5s + retries: 12 + profiles: + - mongo + - backbeat + + # =================== + # Metadata backend services + # =================== + metadata-standalone: + image: ${METADATA_IMAGE} + network_mode: 'host' + volumes: + - ./md-config.json:/mnt/standalone_workdir/config.json:ro + profiles: + - metadata + + cloudserver-metadata: + image: ${CLOUDSERVER_IMAGE} + network_mode: 'host' + environment: + - S3VAULT=mem + - S3METADATA=scality + - S3DATA=mem + - REMOTE_MANAGEMENT_DISABLE=true + - LOG_LEVEL=info + depends_on: + - metadata-standalone + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${CLOUDSERVER_PORT}/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""] + interval: 5s + timeout: 5s + retries: 12 + profiles: + - metadata + + # =================== + # Backbeat services + # =================== + zookeeper: + image: ${ZOOKEEPER_IMAGE} + ports: + - "127.0.0.1:${ZOOKEEPER_PORT}:${ZOOKEEPER_PORT}" + healthcheck: + test: ["CMD", "zkServer.sh", "status"] + interval: 10s + timeout: 5s + retries: 5 + profiles: + - backbeat + + zk-init: + image: ${ZOOKEEPER_IMAGE} + depends_on: + zookeeper: + condition: service_healthy + command: > + bash -c " + sleep 2 && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/ingestion '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/ingestion/state '' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/us-east-1 '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/us-east-2 '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/wontwork-location '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/location-dmf-v1 '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/zenko '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/aws-location '{\"paused\":false}' && + zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/replayState '' && + echo 'Zookeeper paths initialized successfully' + " + restart: "no" + profiles: + - backbeat + + redis: + image: ${REDIS_IMAGE} + ports: + - "127.0.0.1:${REDIS_PORT}:${REDIS_PORT}" + profiles: + - backbeat + + kafka: + image: ${KAFKA_IMAGE} + ports: + - "127.0.0.1:${KAFKA_PORT}:${KAFKA_PORT}" + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:${ZOOKEEPER_PORT} + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:${KAFKA_PORT} + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:${KAFKA_PORT} + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + depends_on: + zookeeper: + condition: service_healthy + profiles: + - backbeat + + backbeat-api: + image: ${BACKBEAT_IMAGE} + ports: + - "127.0.0.1:${BACKBEAT_PORT}:${BACKBEAT_PORT}" + environment: + ZOOKEEPER_CONNECTION_STRING: zookeeper:${ZOOKEEPER_PORT} + REDIS_HOST: redis + KAFKA_HOSTS: kafka:${KAFKA_PORT} + MONGODB_HOSTS: mongodb:${MONGODB_PORT} + REMOTE_MANAGEMENT_DISABLE: "1" + HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0" + depends_on: + mongodb-init: + condition: service_completed_successfully + zk-init: + condition: service_completed_successfully + kafka: + condition: service_started + redis: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${BACKBEAT_PORT}/_/healthcheck', r => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))\""] + interval: 5s + timeout: 5s + retries: 12 + command: node bin/backbeat.js + profiles: + - backbeat diff --git a/.github/docker.env b/.github/docker.env new file mode 100644 index 00000000..18a1cc6f --- /dev/null +++ b/.github/docker.env @@ -0,0 +1,27 @@ +# MongoDB +MONGODB_IMAGE=mongo:4.4 +MONGODB_PORT=27018 + +# Cloudserver +CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.0-preview.1 +CLOUDSERVER_PORT=8000 + +# Metadata +METADATA_IMAGE=ghcr.io/scality/metadata:8.11.0-standalone +METADATA_PORT=9000 + +# Zookeeper +ZOOKEEPER_IMAGE=zookeeper:3.8 +ZOOKEEPER_PORT=2181 + +# Kafka +KAFKA_IMAGE=wurstmeister/kafka:latest +KAFKA_PORT=9092 + +# Redis +REDIS_IMAGE=redis:7-alpine +REDIS_PORT=6379 + +# Backbeat +BACKBEAT_IMAGE=ghcr.io/scality/backbeat:9.1.3 +BACKBEAT_PORT=8900 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index baf7c6cf..ac0a0e5d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,19 +40,14 @@ jobs: uses: ./.github/actions/setup-and-build - name: Start Cloudserver with MongoDB backend - run: docker compose -f .github/docker-compose.cloudserver-mongo.yml up -d - - - name: Wait for Cloudserver to be ready - run: | - set -o pipefail - bash .github/scripts/wait_for_local_port.bash 8000 40 + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile mongo up -d --wait - name: Run MongoDB backend tests run: yarn test:mongo-backend - name: Stop Cloudserver if: always() - run: docker compose -f .github/docker-compose.cloudserver-mongo.yml down + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile mongo down test-metadata-backend: name: Test with Scality metadata backend @@ -74,21 +69,45 @@ jobs: password: ${{ github.token }} - name: Start Cloudserver with Scality metadata backend - run: docker compose -f .github/docker-compose.cloudserver-metadata.yml up -d + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile metadata up -d --wait - name: Wait for metadata to be ready run: | set -o pipefail bash .github/scripts/wait_for_local_port.bash 9000 40 - - name: Wait for Cloudserver to be ready - run: | - set -o pipefail - bash .github/scripts/wait_for_local_port.bash 8000 60 - - name: Run metadata backend tests run: yarn test:metadata-backend - name: Stop Cloudserver if: always() - run: docker compose -f .github/docker-compose.cloudserver-metadata.yml down + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile metadata down + + test-backbeat-apis: + name: Test backbeat apis + runs-on: ubuntu-24.04 + needs: lint + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup and Build + uses: ./.github/actions/setup-and-build + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Start Backbeat with CloudServer + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile backbeat up -d --wait + + - name: Run backbeat apis tests + run: yarn test:backbeat-apis + + - name: Stop Backbeat and Cloudserver + if: always() + run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile backbeat down \ No newline at end of file diff --git a/models/proxyBackbeatApis/checkConnection.smithy b/models/proxyBackbeatApis/checkConnection.smithy new file mode 100644 index 00000000..0dda8173 --- /dev/null +++ b/models/proxyBackbeatApis/checkConnection.smithy @@ -0,0 +1,13 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/healthcheck") +operation CheckConnection { + input: CheckConnectionInput, + output: CheckConnectionOutput +} + +structure CheckConnectionInput {} + +structure CheckConnectionOutput {} diff --git a/models/proxyBackbeatApis/commonStructures.smithy b/models/proxyBackbeatApis/commonStructures.smithy new file mode 100644 index 00000000..2a9aeef5 --- /dev/null +++ b/models/proxyBackbeatApis/commonStructures.smithy @@ -0,0 +1,21 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +map LocationStatusMap { + key: String, + value: Document +} + +structure FailedObjectVersion { + Bucket: String, + + Key: String, + + VersionId: String, + + StorageClass: String, + + Size: Integer, + + LastModified: Timestamp +} \ No newline at end of file diff --git a/models/proxyBackbeatApis/getFailedObject.smithy b/models/proxyBackbeatApis/getFailedObject.smithy new file mode 100644 index 00000000..a39491ad --- /dev/null +++ b/models/proxyBackbeatApis/getFailedObject.smithy @@ -0,0 +1,29 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/failed/{Bucket}/{Key+}") +operation GetFailedObject { + input: GetFailedObjectInput, + output: GetFailedObjectOutput +} + +structure GetFailedObjectInput { + @required + @httpLabel + Bucket: String, + + @required + @httpLabel + Key: String, + + @required + @httpQuery("versionId") + VersionId: String +} + +structure GetFailedObjectOutput { + IsTruncated: Boolean, + + Versions: FailedObjectVersions +} diff --git a/models/proxyBackbeatApis/getLocationsIngestionStatus.smithy b/models/proxyBackbeatApis/getLocationsIngestionStatus.smithy new file mode 100644 index 00000000..31ad0db5 --- /dev/null +++ b/models/proxyBackbeatApis/getLocationsIngestionStatus.smithy @@ -0,0 +1,16 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/ingestion/status") +operation GetLocationsIngestionStatus { + input: GetLocationsIngestionStatusInput, + output: GetLocationsIngestionStatusOutput +} + +structure GetLocationsIngestionStatusInput {} + +structure GetLocationsIngestionStatusOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/getLocationsStatus.smithy b/models/proxyBackbeatApis/getLocationsStatus.smithy new file mode 100644 index 00000000..b2585950 --- /dev/null +++ b/models/proxyBackbeatApis/getLocationsStatus.smithy @@ -0,0 +1,16 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/status") +operation GetLocationsStatus { + input: GetLocationsStatusInput, + output: GetLocationsStatusOutput +} + +structure GetLocationsStatusInput {} + +structure GetLocationsStatusOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/listFailed.smithy b/models/proxyBackbeatApis/listFailed.smithy new file mode 100644 index 00000000..eeb4b627 --- /dev/null +++ b/models/proxyBackbeatApis/listFailed.smithy @@ -0,0 +1,29 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@readonly +@http(method: "GET", uri: "/_/backbeat/api/crr/failed") +operation ListFailed { + input: ListFailedInput, + output: ListFailedOutput +} + +structure ListFailedInput { + @httpQuery("marker") + Marker: String, + + @httpQuery("sitename") + Sitename: String +} + +structure ListFailedOutput { + IsTruncated: Boolean, + + NextMarker: String, + + Versions: FailedObjectVersions +} + +list FailedObjectVersions { + member: FailedObjectVersion +} diff --git a/models/proxyBackbeatApis/pauseAllIngestionSites.smithy b/models/proxyBackbeatApis/pauseAllIngestionSites.smithy new file mode 100644 index 00000000..5ebc40d2 --- /dev/null +++ b/models/proxyBackbeatApis/pauseAllIngestionSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/pause") +operation PauseAllIngestionSites { + input: PauseAllIngestionSitesInput, + output: PauseAllIngestionSitesOutput +} + +structure PauseAllIngestionSitesInput { + @httpPayload + Body: Blob +} + +structure PauseAllIngestionSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/pauseAllSites.smithy b/models/proxyBackbeatApis/pauseAllSites.smithy new file mode 100644 index 00000000..ea1a809b --- /dev/null +++ b/models/proxyBackbeatApis/pauseAllSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/pause") +operation PauseAllSites { + input: PauseAllSitesInput, + output: PauseAllSitesOutput +} + +structure PauseAllSitesInput { + @httpPayload + Body: Blob +} + +structure PauseAllSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/pauseIngestionSite.smithy b/models/proxyBackbeatApis/pauseIngestionSite.smithy new file mode 100644 index 00000000..8201e469 --- /dev/null +++ b/models/proxyBackbeatApis/pauseIngestionSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/pause/{Site}") +operation PauseIngestionSite { + input: PauseIngestionSiteInput, + output: PauseIngestionSiteOutput +} + +structure PauseIngestionSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure PauseIngestionSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/pauseSite.smithy b/models/proxyBackbeatApis/pauseSite.smithy new file mode 100644 index 00000000..1e8d3bb2 --- /dev/null +++ b/models/proxyBackbeatApis/pauseSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/pause/{Site}") +operation PauseSite { + input: PauseSiteInput, + output: PauseSiteOutput +} + +structure PauseSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure PauseSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeAllIngestionSites.smithy b/models/proxyBackbeatApis/resumeAllIngestionSites.smithy new file mode 100644 index 00000000..ba1690ae --- /dev/null +++ b/models/proxyBackbeatApis/resumeAllIngestionSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume") +operation ResumeAllIngestionSites { + input: ResumeAllIngestionSitesInput, + output: ResumeAllIngestionSitesOutput +} + +structure ResumeAllIngestionSitesInput { + @httpPayload + Body: Blob +} + +structure ResumeAllIngestionSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeAllSites.smithy b/models/proxyBackbeatApis/resumeAllSites.smithy new file mode 100644 index 00000000..b84d0d16 --- /dev/null +++ b/models/proxyBackbeatApis/resumeAllSites.smithy @@ -0,0 +1,18 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/resume") +operation ResumeAllSites { + input: ResumeAllSitesInput, + output: ResumeAllSitesOutput +} + +structure ResumeAllSitesInput { + @httpPayload + Body: Blob +} + +structure ResumeAllSitesOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeIngestionSite.smithy b/models/proxyBackbeatApis/resumeIngestionSite.smithy new file mode 100644 index 00000000..e3fcc38b --- /dev/null +++ b/models/proxyBackbeatApis/resumeIngestionSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume/{Site}") +operation ResumeIngestionSite { + input: ResumeIngestionSiteInput, + output: ResumeIngestionSiteOutput +} + +structure ResumeIngestionSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ResumeIngestionSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/resumeSite.smithy b/models/proxyBackbeatApis/resumeSite.smithy new file mode 100644 index 00000000..6f2ee7e1 --- /dev/null +++ b/models/proxyBackbeatApis/resumeSite.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/resume/{Site}") +operation ResumeSite { + input: ResumeSiteInput, + output: ResumeSiteOutput +} + +structure ResumeSiteInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ResumeSiteOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/retryFailedObjects.smithy b/models/proxyBackbeatApis/retryFailedObjects.smithy new file mode 100644 index 00000000..98f52cb3 --- /dev/null +++ b/models/proxyBackbeatApis/retryFailedObjects.smithy @@ -0,0 +1,19 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/failed") +operation RetryFailedObjects { + input: RetryFailedObjectsInput, + output: RetryFailedObjectsOutput +} + +structure RetryFailedObjectsInput { + @required + @httpPayload + Body: Blob +} + +structure RetryFailedObjectsOutput { + @httpPayload + Results: Document +} diff --git a/models/proxyBackbeatApis/scheduleIngestionSiteResume.smithy b/models/proxyBackbeatApis/scheduleIngestionSiteResume.smithy new file mode 100644 index 00000000..a4878a24 --- /dev/null +++ b/models/proxyBackbeatApis/scheduleIngestionSiteResume.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/ingestion/resume/{Site}/schedule") +operation ScheduleIngestionSiteResume { + input: ScheduleIngestionSiteResumeInput, + output: ScheduleIngestionSiteResumeOutput +} + +structure ScheduleIngestionSiteResumeInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ScheduleIngestionSiteResumeOutput { + @httpPayload + status: Document +} diff --git a/models/proxyBackbeatApis/scheduleSiteResume.smithy b/models/proxyBackbeatApis/scheduleSiteResume.smithy new file mode 100644 index 00000000..ef37da42 --- /dev/null +++ b/models/proxyBackbeatApis/scheduleSiteResume.smithy @@ -0,0 +1,22 @@ +$version: "2.0" +namespace cloudserver.proxyBackbeatApis + +@http(method: "POST", uri: "/_/backbeat/api/crr/resume/{Site}/schedule") +operation ScheduleSiteResume { + input: ScheduleSiteResumeInput, + output: ScheduleSiteResumeOutput +} + +structure ScheduleSiteResumeInput { + @required + @httpLabel + Site: String, + + @httpPayload + Body: Blob +} + +structure ScheduleSiteResumeOutput { + @httpPayload + status: Document +} diff --git a/package.json b/package.json index ec82a98c..78486a65 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@scality/cloudserverclient", - "version": "1.0.1", + "version": "1.0.2", "engines": { "node": ">=20" }, @@ -30,11 +30,13 @@ "build:smithy": "smithy build", "build:generated:backbeatRoutes": "cd build/smithy/cloudserverBackbeatRoutes/typescript-codegen && yarn install && yarn build", "build:generated:bucketQuota": "cd build/smithy/cloudserverBucketQuota/typescript-codegen && yarn install && yarn build", + "build:generated:proxyBackbeatApis": "cd build/smithy/cloudserverProxyBackbeatApis/typescript-codegen && yarn install && yarn build", "build:wrapper": "tsc", - "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated:backbeatRoutes && yarn build:generated:bucketQuota && yarn build:wrapper", + "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated:backbeatRoutes && yarn build:generated:bucketQuota && yarn build:generated:proxyBackbeatApis && yarn build:wrapper", "test": "jest", "test:mongo-backend": "BACKEND_TYPE=mongo jest", "test:metadata-backend": "BACKEND_TYPE=metadata jest", + "test:backbeat-apis": "BACKBEAT_SETUP=true yarn jest tests/testBackbeatProxyApis.test.ts", "lint": "eslint src tests", "typecheck": "tsc --noEmit" }, diff --git a/service/cloudserverProxyBackbeatApis.smithy b/service/cloudserverProxyBackbeatApis.smithy new file mode 100644 index 00000000..31b6425c --- /dev/null +++ b/service/cloudserverProxyBackbeatApis.smithy @@ -0,0 +1,32 @@ +$version: "2.0" + +namespace cloudserver.proxyBackbeatApis + +use aws.protocols#restJson1 +use aws.auth#sigv4 +use aws.api#service + +@restJson1 +@sigv4(name: "s3") +@service(sdkId: "CloudserverProxyBackbeatApis") +service CloudserverProxyBackbeatApis { + version: "2017-07-01", + operations: [ + CheckConnection, + GetFailedObject, + GetLocationsIngestionStatus, + GetLocationsStatus, + ListFailed, + PauseAllIngestionSites, + PauseAllSites, + PauseIngestionSite, + PauseSite, + ResumeAllIngestionSites, + ResumeAllSites, + ResumeIngestionSite, + ResumeSite, + RetryFailedObjects, + ScheduleIngestionSiteResume, + ScheduleSiteResume, + ] +} diff --git a/smithy-build.json b/smithy-build.json index 00589e82..78886c2f 100644 --- a/smithy-build.json +++ b/smithy-build.json @@ -25,6 +25,15 @@ "packageVersion": "1.0.0" } } + }, + "cloudserverProxyBackbeatApis": { + "plugins": { + "typescript-codegen": { + "service": "cloudserver.proxyBackbeatApis#CloudserverProxyBackbeatApis", + "package": "@scality/cloudserverclient-proxy-backbeat", + "packageVersion": "1.0.0" + } + } } } } diff --git a/src/clients/proxyBackbeatApis.ts b/src/clients/proxyBackbeatApis.ts new file mode 100644 index 00000000..2328baa8 --- /dev/null +++ b/src/clients/proxyBackbeatApis.ts @@ -0,0 +1,64 @@ +import { + CloudserverProxyBackbeatApisClient, + CloudserverProxyBackbeatApisClientConfig, +} from '../../build/smithy/cloudserverProxyBackbeatApis/typescript-codegen'; + +export { CloudserverProxyBackbeatApisClientConfig }; +export { + CheckConnectionCommand, + GetFailedObjectCommand, + GetLocationsIngestionStatusCommand, + GetLocationsStatusCommand, + ListFailedCommand, + PauseAllIngestionSitesCommand, + PauseAllSitesCommand, + PauseIngestionSiteCommand, + PauseSiteCommand, + ResumeAllIngestionSitesCommand, + ResumeAllSitesCommand, + ResumeIngestionSiteCommand, + ResumeSiteCommand, + RetryFailedObjectsCommand, + ScheduleIngestionSiteResumeCommand, + ScheduleSiteResumeCommand, +} from '../../build/smithy/cloudserverProxyBackbeatApis/typescript-codegen'; +export type { + CheckConnectionCommandInput, + CheckConnectionCommandOutput, + GetFailedObjectCommandInput, + GetFailedObjectCommandOutput, + GetLocationsIngestionStatusCommandInput, + GetLocationsIngestionStatusCommandOutput, + GetLocationsStatusCommandInput, + GetLocationsStatusCommandOutput, + ListFailedCommandInput, + ListFailedCommandOutput, + PauseAllIngestionSitesCommandInput, + PauseAllIngestionSitesCommandOutput, + PauseAllSitesCommandInput, + PauseAllSitesCommandOutput, + PauseIngestionSiteCommandInput, + PauseIngestionSiteCommandOutput, + PauseSiteCommandInput, + PauseSiteCommandOutput, + ResumeAllIngestionSitesCommandInput, + ResumeAllIngestionSitesCommandOutput, + ResumeAllSitesCommandInput, + ResumeAllSitesCommandOutput, + ResumeIngestionSiteCommandInput, + ResumeIngestionSiteCommandOutput, + ResumeSiteCommandInput, + ResumeSiteCommandOutput, + RetryFailedObjectsCommandInput, + RetryFailedObjectsCommandOutput, + ScheduleIngestionSiteResumeCommandInput, + ScheduleIngestionSiteResumeCommandOutput, + ScheduleSiteResumeCommandInput, + ScheduleSiteResumeCommandOutput, +} from '../../build/smithy/cloudserverProxyBackbeatApis/typescript-codegen'; + +export class ProxyBackbeatApisClient extends CloudserverProxyBackbeatApisClient { + constructor(config: CloudserverProxyBackbeatApisClientConfig) { + super(config); + } +} diff --git a/src/index.ts b/src/index.ts index e377ec7d..85656164 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ export * from './clients/backbeatRoutes'; export * from './clients/bucketQuota'; +export * from './clients/proxyBackbeatApis'; export * from './clients/s3Extended'; export { CloudserverClient, CloudserverClientConfig } from './clients/cloudserver'; export * from './utils'; diff --git a/tests/testBackbeatProxyApis.test.ts b/tests/testBackbeatProxyApis.test.ts new file mode 100644 index 00000000..cf4242e0 --- /dev/null +++ b/tests/testBackbeatProxyApis.test.ts @@ -0,0 +1,194 @@ +import { + ProxyBackbeatApisClient, + CheckConnectionCommandInput, + CheckConnectionCommand, + GetLocationsStatusCommandInput, + GetLocationsStatusCommand, + GetLocationsIngestionStatusCommandInput, + GetLocationsIngestionStatusCommand, + ListFailedCommandInput, + ListFailedCommand, + GetFailedObjectCommandInput, + GetFailedObjectCommand, + RetryFailedObjectsCommand, + PauseAllSitesCommandInput, + PauseAllSitesCommand, + PauseAllIngestionSitesCommandInput, + PauseAllIngestionSitesCommand, + PauseSiteCommandInput, + PauseSiteCommand, + PauseIngestionSiteCommandInput, + PauseIngestionSiteCommand, + ResumeAllSitesCommandInput, + ResumeAllSitesCommand, + ResumeAllIngestionSitesCommandInput, + ResumeAllIngestionSitesCommand, + ResumeSiteCommandInput, + ResumeSiteCommand, + ResumeIngestionSiteCommandInput, + ResumeIngestionSiteCommand, + ScheduleSiteResumeCommandInput, + ScheduleSiteResumeCommand, + ScheduleIngestionSiteResumeCommandInput, + ScheduleIngestionSiteResumeCommand, +} from '../src/clients/proxyBackbeatApis'; +import { createTestClient, testConfig } from './testSetup'; +import { describeForBackbeatSetup } from './testHelpers'; +import assert from 'assert'; + +// Note : these tests are relatively simple, as it's not straightforward to setup +// backbeat with locations, and real pause/resume tests scenarios. +// The tests in Zenko are also using these apis, and will represent an opportunity to +// test this client better : https://scality.atlassian.net/browse/ZENKO-5102 +describeForBackbeatSetup('CloudServer Check Status API Tests', () => { + let proxyBackbeatApisClient: ProxyBackbeatApisClient; + + beforeAll(() => { + ({proxyBackbeatApisClient} = createTestClient()); + }); + + it('should test checkConnection', async () => { + const input: CheckConnectionCommandInput = {}; + const cmd = new CheckConnectionCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test GetLocationsStatus', async () => { + const input: GetLocationsStatusCommandInput = {}; + const cmd = new GetLocationsStatusCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.strictEqual((result.status as Record)?.['wontwork-location'], 'enabled'); + }); + + it('should test GetLocationsIngestionStatus', async () => { + const input: GetLocationsIngestionStatusCommandInput = {}; + const cmd = new GetLocationsIngestionStatusCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test listFailed', async () => { + const input: ListFailedCommandInput = {}; + const cmd = new ListFailedCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test getFailedObject', async () => { + const input: GetFailedObjectCommandInput = { + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + VersionId: testConfig.versionID, + }; + const cmd = new GetFailedObjectCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test retryFailedObjects', async () => { + const bodyData = JSON.stringify([ + { + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + VersionId: testConfig.versionID, + StorageClass: 'STANDARD', + } + ]); + const cmd = new RetryFailedObjectsCommand({ + Body: new TextEncoder().encode(bodyData) + }); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseAllSites', async () => { + const input: PauseAllSitesCommandInput = {}; + const cmd = new PauseAllSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseAllIngestionSites', async () => { + const input: PauseAllIngestionSitesCommandInput = {}; + const cmd = new PauseAllIngestionSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test pauseSite', async () => { + const input: PauseSiteCommandInput = { + Site: 'zenko', // Value from backbeat default config.json, destination.bootstrapList + }; + const cmd = new PauseSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test pauseIngestionSite', async () => { + const input: PauseIngestionSiteCommandInput = { + Site: 'a-zenko-location', // Value from backbeat default config.json, extensions.ingestion.sources + }; + const cmd = new PauseIngestionSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeAllSites', async () => { + const input: ResumeAllSitesCommandInput = {}; + const cmd = new ResumeAllSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeAllIngestionSites', async () => { + const input: ResumeAllIngestionSitesCommandInput = {}; + const cmd = new ResumeAllIngestionSitesCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test resumeSite', async () => { + const input: ResumeSiteCommandInput = { + Site: 'zenko', + }; + const cmd = new ResumeSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test resumeIngestionSite', async () => { + const input: ResumeIngestionSiteCommandInput = { + Site: 'a-zenko-location', + }; + const cmd = new ResumeIngestionSiteCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('should test scheduleSiteResume', async () => { + const bodyData = JSON.stringify({ hours: 24 }); + const input: ScheduleSiteResumeCommandInput = { + Site: 'zenko', + Body: new TextEncoder().encode(bodyData) + }; + const cmd = new ScheduleSiteResumeCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + // No site available for ingestion on our test setup + it.skip('should test scheduleIngestionSiteResume', async () => { + const bodyData = JSON.stringify({ hours: 24 }); + const input: ScheduleIngestionSiteResumeCommandInput = { + Site: 'zenko', + Body: new TextEncoder().encode(bodyData) + }; + const cmd = new ScheduleIngestionSiteResumeCommand(input); + const result = await proxyBackbeatApisClient.send(cmd); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); +}); diff --git a/tests/testHelpers.ts b/tests/testHelpers.ts index 70be4765..23edf0a1 100644 --- a/tests/testHelpers.ts +++ b/tests/testHelpers.ts @@ -18,3 +18,11 @@ export function describeForMetadataBackend(name: string, fn: () => void): void { describe.skip(`${name} (tests skipped: metadata backend only)`, fn); } } + +export function describeForBackbeatSetup(name: string, fn: () => void): void { + if (process.env.BACKBEAT_SETUP === 'true') { + describe(name, fn); + } else { + describe.skip(`${name} (tests skipped: backbeat setup required)`, fn); + } +} diff --git a/tests/testSetup.ts b/tests/testSetup.ts index 383a76e2..d2d51dd6 100644 --- a/tests/testSetup.ts +++ b/tests/testSetup.ts @@ -4,6 +4,7 @@ import { BackbeatRoutesClient, CloudserverBackbeatRoutesClientConfig } from '../ import { S3Client, PutObjectCommand, CreateBucketCommand, PutBucketVersioningCommand } from '@aws-sdk/client-s3'; import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@aws-sdk/types'; import { BucketQuotaClient } from '../src/clients/bucketQuota'; +import { ProxyBackbeatApisClient } from '../src/clients/proxyBackbeatApis'; jest.setTimeout(30000); const credentialsProvider: AwsCredentialIdentityProvider = async (): Promise => ({ @@ -48,6 +49,7 @@ export const testConfig = { objectKey: `test-cloudserverclient-object-sla/sh${randomId()}`, objectData: 'iAmSomeData', canonicalID: '39383234313039353433383937313939393939395247303031202036353034352e30', + versionID: '' }; async function initBucketForTests() { @@ -70,7 +72,8 @@ async function initBucketForTests() { Key: testConfig.objectKey, Body: testConfig.objectData, }); - await s3client.send(putObjectCommand); + const result = await s3client.send(putObjectCommand); + testConfig.versionID = result.VersionId!; } catch (error: any) { assert.fail(`Failed to initialize bucket for tests: ${error}`); } @@ -79,11 +82,13 @@ async function initBucketForTests() { export function createTestClient(): { backbeatRoutesClient: BackbeatRoutesClient, bucketQuotaClient: BucketQuotaClient, + proxyBackbeatApisClient: ProxyBackbeatApisClient, s3client: S3Client } { return { backbeatRoutesClient: new BackbeatRoutesClient(config), bucketQuotaClient: new BucketQuotaClient(config), + proxyBackbeatApisClient: new ProxyBackbeatApisClient(config), s3client, }; }