From 4193237f06ac5b9ac18dbf40047e285ca0e3e7b1 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Wed, 17 Dec 2025 12:19:03 +0100 Subject: [PATCH 1/3] added s3 extended apis ISSUE: CLDSRVCLT-6 --- package.json | 6 +- src/clients/s3Extended.ts | 64 +++++++++++++++ src/index.ts | 1 + tests/testS3ExtendedApis.test.ts | 135 +++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 src/clients/s3Extended.ts create mode 100644 tests/testS3ExtendedApis.test.ts diff --git a/package.json b/package.json index c1b9fbce..0a668301 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,7 @@ }, "files": [ "dist", - "build/smithy/cloudserverBackbeatRoutes/typescript-codegen", - "build/smithy/cloudserverBucketQuota/typescript-codegen" + "build/smithy/*/typescript-codegen" ], "publishConfig": { "access": "public", @@ -42,7 +41,8 @@ "test:metadata": "jest tests/testMetadataApis.test.ts", "test:raft": "jest tests/testRaftApis.test.ts", "test:bucketQuotas": "jest tests/testQuotaApis.test.ts", - "test:mongo-backend": "yarn test:indexes && yarn test:error-handling && yarn test:multiple-backend && yarn test:bucketQuotas", + "test:s3extended": "jest tests/testS3ExtendedApis.test.ts", + "test:mongo-backend": "yarn test:indexes && yarn test:error-handling && yarn test:multiple-backend && yarn test:bucketQuotas && yarn test:s3extended", "test:metadata-backend": "yarn test:api && yarn test:lifecycle && yarn test:metadata && yarn test:raft", "lint": "eslint src tests", "typecheck": "tsc --noEmit" diff --git a/src/clients/s3Extended.ts b/src/clients/s3Extended.ts new file mode 100644 index 00000000..c3555a6e --- /dev/null +++ b/src/clients/s3Extended.ts @@ -0,0 +1,64 @@ +import { + ListObjectsCommand, + ListObjectsCommandInput, + ListObjectsV2Command, + ListObjectsV2CommandInput, + ListObjectVersionsCommand, + ListObjectVersionsCommandInput +} from '@aws-sdk/client-s3'; + +const extendCommandWithExtraParametersMiddleware = (query: string) => + (next: any) => async (args: any) => { + const request = args.request as any; + if (request.query) { + request.query.search = query; + } else { + request.query = { search: query }; + } + return next(args); + }; + +export interface ListObjectsExtendedInput extends ListObjectsCommandInput { + Query: string; +} + +export class ListObjectsExtendedCommand extends ListObjectsCommand { + constructor(input: ListObjectsExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + } +} + +export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput { + Query: string; +} + +export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command { + constructor(input: ListObjectsV2ExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + } +} + +export interface ListObjectVersionsExtendedInput extends ListObjectVersionsCommandInput { + Query: string; +} + +export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand { + constructor(input: ListObjectVersionsExtendedInput) { + super(input); + + this.middlewareStack.add( + extendCommandWithExtraParametersMiddleware(input.Query), + { step: 'build', name: 'extendCommandWithExtraParameters' } + ); + } +} diff --git a/src/index.ts b/src/index.ts index 5c274a04..e377ec7d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ export * from './clients/backbeatRoutes'; export * from './clients/bucketQuota'; +export * from './clients/s3Extended'; export { CloudserverClient, CloudserverClientConfig } from './clients/cloudserver'; export * from './utils'; diff --git a/tests/testS3ExtendedApis.test.ts b/tests/testS3ExtendedApis.test.ts new file mode 100644 index 00000000..632313c8 --- /dev/null +++ b/tests/testS3ExtendedApis.test.ts @@ -0,0 +1,135 @@ +import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from './testSetup'; +import { describeForMongoBackend } from './testHelpers'; +import assert from 'assert'; +import { + ListObjectsExtendedCommand, + ListObjectsV2ExtendedCommand, + ListObjectVersionsExtendedCommand, +} from '../src/clients/s3Extended'; + +describe('S3 Extended API Tests', () => { + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + s3client = testClients.s3client; + + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test ListObjectsExtended', async () => { + const getCommand1= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + + const maxKey = 1; + const getCommand2= new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: maxKey + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, maxKey); + assert.strictEqual(getData2.IsTruncated, true); + + const getCommand3 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length > ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData3 = await s3client.send(getCommand3); + assert.strictEqual(getData3.Contents?.length, 1); + assert.strictEqual(getData3.Contents[0].Key, key2ndObject); + + const getCommand4 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `key = ${key2ndObject}`, + MaxKeys: 5 + }); + const getData4 = await s3client.send(getCommand4); + assert.strictEqual(getData4.Contents?.length, 1); + assert.strictEqual(getData4.Contents[0].Key, key2ndObject); + + const getCommand5 = new ListObjectsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `key = iDontExists`, + MaxKeys: 5 + }); + const getData5 = await s3client.send(getCommand5); + assert.strictEqual(getData5.Contents, undefined); + }); + + it('should test ListObjectsV2Extended', async () => { + const getCommand1= new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5, + FetchOwner: true, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + assert.strictEqual(getData1.KeyCount, 2); + assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); + + const getCommand2 = new ListObjectsV2ExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + MaxKeys: 5, + StartAfter: testConfig.objectKey, // Skip first object + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, 1); + assert.strictEqual(getData2.Contents[0].Key, key2ndObject); + }); + + it('should test ListObjectVersionsExtended', async () => { + const getCommand1 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + MaxKeys: 100, + }); + const getData1 = await s3client.send(getCommand1); + assert.strictEqual(getData1.Versions?.length, 2); + assert.strictEqual(getData1.Versions[0].IsLatest, true); + + // Delete one object to create a DeleteMarker + const deleteCommand = new DeleteObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + }); + await s3client.send(deleteCommand); + + const getCommand2 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + MaxKeys: 100, + }); + const getData2 = await s3client.send(getCommand2); + assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject); + assert.ok(getData2.DeleteMarkers?.[0].VersionId); + + assert.ok(getData2.Versions); + const firstVersion = getData2.Versions[0]; + const getCommand3 = new ListObjectVersionsExtendedCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + KeyMarker: firstVersion.Key, + VersionIdMarker: firstVersion.VersionId, + MaxKeys: 100, + }); + const getData3 = await s3client.send(getCommand3); + assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key); + }); +}); From 29b0e8f295f22897f50f975e47187c2f9f8e051d Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Tue, 13 Jan 2026 22:30:46 +0100 Subject: [PATCH 2/3] add env variable on test to run conditionnaly ISSUE: CLDSRVCLT-6 --- package.json | 13 ++----------- tests/testApis.test.ts | 3 ++- tests/testErrorHandling.test.ts | 3 ++- tests/testHelpers.ts | 20 ++++++++++++++++++++ tests/testIndexesApis.test.ts | 3 ++- tests/testLifecycleApis.test.ts | 3 ++- tests/testMetadataApis.test.ts | 3 ++- tests/testMultipleBackendApis.test.ts | 3 ++- tests/testQuotaApis.test.ts | 3 ++- tests/testRaftApis.test.ts | 3 ++- tests/testS3ExtendedApis.test.ts | 2 +- 11 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 tests/testHelpers.ts diff --git a/package.json b/package.json index 0a668301..ec82a98c 100644 --- a/package.json +++ b/package.json @@ -33,17 +33,8 @@ "build:wrapper": "tsc", "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated:backbeatRoutes && yarn build:generated:bucketQuota && yarn build:wrapper", "test": "jest", - "test:indexes": "jest tests/testIndexesApis.test.ts", - "test:error-handling": "jest tests/testErrorHandling.test.ts", - "test:multiple-backend": "jest tests/testMultipleBackendApis.test.ts", - "test:api": "jest tests/testApis.test.ts", - "test:lifecycle": "jest tests/testLifecycleApis.test.ts", - "test:metadata": "jest tests/testMetadataApis.test.ts", - "test:raft": "jest tests/testRaftApis.test.ts", - "test:bucketQuotas": "jest tests/testQuotaApis.test.ts", - "test:s3extended": "jest tests/testS3ExtendedApis.test.ts", - "test:mongo-backend": "yarn test:indexes && yarn test:error-handling && yarn test:multiple-backend && yarn test:bucketQuotas && yarn test:s3extended", - "test:metadata-backend": "yarn test:api && yarn test:lifecycle && yarn test:metadata && yarn test:raft", + "test:mongo-backend": "BACKEND_TYPE=mongo jest", + "test:metadata-backend": "BACKEND_TYPE=metadata jest", "lint": "eslint src tests", "typecheck": "tsc --noEmit" }, diff --git a/tests/testApis.test.ts b/tests/testApis.test.ts index dd566083..947729f3 100644 --- a/tests/testApis.test.ts +++ b/tests/testApis.test.ts @@ -13,9 +13,10 @@ import { } from '../src/index'; import { S3Client, GetObjectCommand as S3getCommand } from '@aws-sdk/client-s3'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMetadataBackend } from './testHelpers'; import assert from 'assert'; -describe('CloudServer API Tests', () => { +describeForMetadataBackend('CloudServer Backbeat Routes API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; let s3client: S3Client; diff --git a/tests/testErrorHandling.test.ts b/tests/testErrorHandling.test.ts index c78cd031..0abe0d82 100644 --- a/tests/testErrorHandling.test.ts +++ b/tests/testErrorHandling.test.ts @@ -7,8 +7,9 @@ import { } from '../src/index'; import assert from 'assert'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMongoBackend } from './testHelpers'; -describe('CloudServer test error handling', () => { +describeForMongoBackend('CloudServer test error handling', () => { let backbeatRoutesClient: BackbeatRoutesClient; beforeAll(() => { diff --git a/tests/testHelpers.ts b/tests/testHelpers.ts new file mode 100644 index 00000000..70be4765 --- /dev/null +++ b/tests/testHelpers.ts @@ -0,0 +1,20 @@ +enum BackendType { + MONGO = 'mongo', + METADATA = 'metadata', +} + +export function describeForMongoBackend(name: string, fn: () => void): void { + if (process.env.BACKEND_TYPE === BackendType.METADATA) { + describe.skip(`${name} (tests skipped: mongo backend only)`, fn); + } else { + describe(name, fn); + } +} + +export function describeForMetadataBackend(name: string, fn: () => void): void { + if (process.env.BACKEND_TYPE === BackendType.METADATA) { + describe(name, fn); + } else { + describe.skip(`${name} (tests skipped: metadata backend only)`, fn); + } +} diff --git a/tests/testIndexesApis.test.ts b/tests/testIndexesApis.test.ts index 253b05e8..6f8c757c 100644 --- a/tests/testIndexesApis.test.ts +++ b/tests/testIndexesApis.test.ts @@ -9,8 +9,9 @@ import { } from '../src/index'; import assert from 'assert'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMongoBackend } from './testHelpers'; -describe('CloudServer Indexes API Tests', () => { +describeForMongoBackend('CloudServer Indexes API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; beforeAll(() => { diff --git a/tests/testLifecycleApis.test.ts b/tests/testLifecycleApis.test.ts index c4052e78..fc7c3cfb 100644 --- a/tests/testLifecycleApis.test.ts +++ b/tests/testLifecycleApis.test.ts @@ -11,8 +11,9 @@ import { } from '../src/index'; import assert from 'assert'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMetadataBackend } from './testHelpers'; -describe('CloudServer Lifecycle API Tests', () => { +describeForMetadataBackend('CloudServer Lifecycle API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; beforeAll(() => { diff --git a/tests/testMetadataApis.test.ts b/tests/testMetadataApis.test.ts index 2d5ef60d..b9a52f6d 100644 --- a/tests/testMetadataApis.test.ts +++ b/tests/testMetadataApis.test.ts @@ -9,8 +9,9 @@ import { } from '../src/index'; import assert from 'assert'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMetadataBackend } from './testHelpers'; -describe('CloudServer Metadata API Tests', () => { +describeForMetadataBackend('CloudServer Metadata API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; beforeAll(() => { diff --git a/tests/testMultipleBackendApis.test.ts b/tests/testMultipleBackendApis.test.ts index 6947f528..b498ebd8 100644 --- a/tests/testMultipleBackendApis.test.ts +++ b/tests/testMultipleBackendApis.test.ts @@ -21,10 +21,11 @@ import { addContentLengthMiddleware, } from '../src/index'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMongoBackend } from './testHelpers'; import assert from 'assert'; import crypto from 'crypto'; -describe('CloudServer Multiple Backend API Tests', () => { +describeForMongoBackend('CloudServer Multiple Backend API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; beforeAll(() => { diff --git a/tests/testQuotaApis.test.ts b/tests/testQuotaApis.test.ts index 90530715..bc476a43 100644 --- a/tests/testQuotaApis.test.ts +++ b/tests/testQuotaApis.test.ts @@ -5,9 +5,10 @@ import { DeleteBucketQuotaCommand, } from '../src/clients/bucketQuota'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMongoBackend } from './testHelpers'; import assert from 'assert'; -describe('Quota API Tests', () => { +describeForMongoBackend('Quota API Tests', () => { let bucketQuotaClient: BucketQuotaClient; const quotaValue = 12321; diff --git a/tests/testRaftApis.test.ts b/tests/testRaftApis.test.ts index 0b23455b..3f1c0e4d 100644 --- a/tests/testRaftApis.test.ts +++ b/tests/testRaftApis.test.ts @@ -11,10 +11,11 @@ import { } from '../src/index'; import assert from 'assert'; import { createTestClient, testConfig } from './testSetup'; +import { describeForMetadataBackend } from './testHelpers'; import stream from 'stream'; import JSONStream from 'JSONStream'; -describe('CloudServer Raft API Tests', () => { +describeForMetadataBackend('CloudServer Raft API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; beforeAll(() => { diff --git a/tests/testS3ExtendedApis.test.ts b/tests/testS3ExtendedApis.test.ts index 632313c8..0e43b52e 100644 --- a/tests/testS3ExtendedApis.test.ts +++ b/tests/testS3ExtendedApis.test.ts @@ -8,7 +8,7 @@ import { ListObjectVersionsExtendedCommand, } from '../src/clients/s3Extended'; -describe('S3 Extended API Tests', () => { +describeForMongoBackend('S3 Extended API Tests', () => { let s3client: S3Client; const key2ndObject = `${testConfig.objectKey}2nd`; const body2ndObject = `${testConfig.objectData}2nd`; From 3840086271c59dd64a5480c65b65649313d96ec6 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Wed, 14 Jan 2026 11:36:32 +0100 Subject: [PATCH 3/3] added s3extended example ISSUE: CLDSRVCLT-6 --- examples/s3extended.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 examples/s3extended.ts diff --git a/examples/s3extended.ts b/examples/s3extended.ts new file mode 100644 index 00000000..4303e2cd --- /dev/null +++ b/examples/s3extended.ts @@ -0,0 +1,21 @@ + +import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3'; +import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient/clients/s3Extended'; + +const config: S3ClientConfig = { + endpoint: 'http://localhost:8000', + credentials: { + accessKeyId: 'accessKey1', + secretAccessKey: 'verySecretKey1', + }, + region: 'us-east-1', +}; + +const client = new S3Client(config); + +const response = await client.send( + new ListObjectsV2ExtendedCommand({ + Bucket: 'aBucketName', + Query: 'content-length > 0', + }), +);