From 0794af6066892b46d3e16faa0caca0a74006d22e Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Mon, 18 May 2026 17:30:26 +0200 Subject: [PATCH 1/9] feat: add zstd message compression codec for SNS and SQS --- packages/core/lib/codec/messageCodec.ts | 43 +++ packages/core/lib/index.ts | 8 + .../core/lib/queues/AbstractQueueService.ts | 3 + packages/core/lib/types/queueOptionsTypes.ts | 9 + packages/core/package.json | 1 + packages/sns/lib/sns/AbstractSnsPublisher.ts | 9 +- .../sns/lib/sns/AbstractSnsSqsConsumer.ts | 10 +- packages/sns/package.json | 8 +- .../SnsSqsPermissionConsumer.codec.spec.ts | 104 ++++++++ .../consumers/SnsSqsPermissionConsumer.ts | 2 + .../test/publishers/SnsPermissionPublisher.ts | 2 + packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 11 + packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 9 +- packages/sqs/package.json | 8 +- .../SqsPermissionConsumer.codec.spec.ts | 123 +++++++++ .../test/consumers/SqsPermissionConsumer.ts | 2 + .../test/publishers/SqsPermissionPublisher.ts | 2 + pnpm-lock.yaml | 252 ++++++++++++++---- pnpm-workspace.yaml | 1 + 19 files changed, 542 insertions(+), 65 deletions(-) create mode 100644 packages/core/lib/codec/messageCodec.ts create mode 100644 packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts create mode 100644 packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts new file mode 100644 index 00000000..4709a8de --- /dev/null +++ b/packages/core/lib/codec/messageCodec.ts @@ -0,0 +1,43 @@ +import { compress, decompress } from '@mongodb-js/zstd' + +export const SUPPORTED_CODECS = ['zstd'] as const +export type MessageCodec = (typeof SUPPORTED_CODECS)[number] + +const CODEC_FIELD = '__codec' +const DATA_FIELD = '__data' + +export type CodecEnvelope = { + [CODEC_FIELD]: MessageCodec + [DATA_FIELD]: string +} + +export function isCodecEnvelope(value: unknown): value is CodecEnvelope { + return ( + typeof value === 'object' && + value !== null && + CODEC_FIELD in value && + DATA_FIELD in value && + SUPPORTED_CODECS.includes((value as Record)[CODEC_FIELD] as MessageCodec) + ) +} + +export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { + if (codec === 'zstd') { + const compressed = await compress(Buffer.from(jsonBody, 'utf8')) + const envelope: CodecEnvelope = { + [CODEC_FIELD]: codec, + [DATA_FIELD]: compressed.toString('base64'), + } + return JSON.stringify(envelope) + } + throw new Error(`Unsupported codec: ${codec}`) +} + +export async function decompressMessageBody(envelope: CodecEnvelope): Promise { + if (envelope[CODEC_FIELD] === 'zstd') { + const compressed = Buffer.from(envelope[DATA_FIELD], 'base64') + const decompressed = await decompress(compressed) + return JSON.parse(decompressed.toString('utf8')) + } + throw new Error(`Unsupported codec: ${envelope[CODEC_FIELD]}`) +} diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index bb97ea60..97855fb7 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,3 +1,11 @@ +export { + type CodecEnvelope, + compressMessageBody, + decompressMessageBody, + isCodecEnvelope, + type MessageCodec, + SUPPORTED_CODECS, +} from './codec/messageCodec.ts' export { DoNotProcessMessageError } from './errors/DoNotProcessError.ts' export { isMessageError, diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 40fa618b..ca305d0c 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -14,6 +14,7 @@ import { } from '@message-queue-toolkit/schemas' import { getProperty, setProperty } from 'dot-prop' import type { ZodSchema, ZodType } from 'zod/v4' +import type { MessageCodec } from '../codec/messageCodec.ts' import type { MessageInvalidFormatError, MessageValidationError } from '../errors/Errors.ts' import { type AcquireLockTimeoutError, @@ -135,6 +136,7 @@ export abstract class AbstractQueueService< protected readonly messageDeduplicationConfig?: MessageDeduplicationConfig protected readonly messageMetricsManager?: MessageMetricsManager protected readonly _handlerSpy?: HandlerSpy + protected readonly codec?: MessageCodec protected isInitted: boolean @@ -172,6 +174,7 @@ export abstract class AbstractQueueService< } : undefined this.messageDeduplicationConfig = options.messageDeduplicationConfig + this.codec = options.codec this.logMessages = options.logMessages ?? false this._handlerSpy = resolveHandlerSpy(options) diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index d8f26445..3310ca5b 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -1,5 +1,6 @@ import type { CommonLogger, ErrorReporter, ErrorResolver } from '@lokalise/node-core' import type { ZodSchema } from 'zod/v4' +import type { MessageCodec } from '../codec/messageCodec.ts' import type { MessageDeduplicationConfig } from '../message-deduplication/messageDeduplicationTypes.ts' import type { PayloadStoreConfig } from '../payload-store/payloadStoreTypes.ts' import type { MessageHandlerConfig } from '../queues/HandlerContainer.ts' @@ -139,6 +140,14 @@ export type CommonQueueOptions = { deletionConfig?: DeletionConfig payloadStoreConfig?: PayloadStoreConfig messageDeduplicationConfig?: MessageDeduplicationConfig + /** + * Codec to use for compressing outgoing messages and decompressing incoming messages. + * When set on a publisher, messages are compressed before sending. + * When set on a consumer, it signals that incoming messages may be compressed. + * Compressed messages are self-describing (the codec is embedded in the message envelope), + * so consumers can decompress even without this option explicitly set. + */ + codec?: MessageCodec } export type CommonCreationConfigType = { diff --git a/packages/core/package.json b/packages/core/package.json index 041db97b..12b5e3be 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -28,6 +28,7 @@ "@lokalise/node-core": "^14.2.0", "@lokalise/universal-ts-utils": "^4.5.1", "@message-queue-toolkit/schemas": "^7.0.0", + "@mongodb-js/zstd": "^7.0.0", "dot-prop": "^10.1.0", "fast-equals": "^6.0.0", "json-stream-stringify": "^3.1.6", diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index f6c14ad4..6e9f95f7 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -5,7 +5,9 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, + compressMessageBody, DeduplicationRequesterEnum, + isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -211,8 +213,13 @@ export abstract class AbstractSnsPublisher options: SNSMessageOptions, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) + const jsonBody = JSON.stringify(payload) + const body = + this.codec && !isOffloadedPayloadPointerPayload(payload) + ? await compressMessageBody(jsonBody, this.codec) + : jsonBody const command = new PublishCommand({ - Message: JSON.stringify(payload), + Message: body, MessageAttributes: attributes, TopicArn: this.topicArn, ...options, diff --git a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts index 9e1113e1..37f2d857 100644 --- a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts +++ b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts @@ -1,5 +1,11 @@ import type { SNSClient } from '@aws-sdk/client-sns' import type { STSClient } from '@aws-sdk/client-sts' +import type { Either } from '@lokalise/node-core' +import type { + MessageInvalidFormatError, + MessageValidationError, + ResolvedMessage, +} from '@message-queue-toolkit/core' import type { SQSConsumerDependencies, SQSConsumerOptions, @@ -201,7 +207,9 @@ export abstract class AbstractSnsSqsConsumer< await this.startConsumers() } - protected override resolveMessage(message: SQSMessage) { + protected override resolveMessage( + message: SQSMessage, + ): Either { const result = readSnsMessage(message, this.errorResolver) if (result.result) { return result diff --git a/packages/sns/package.json b/packages/sns/package.json index 06a5030e..f47833f4 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -47,10 +47,10 @@ "@biomejs/biome": "^2.3.6", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "*", - "@message-queue-toolkit/redis-message-deduplication-store": "*", - "@message-queue-toolkit/s3-payload-store": "*", - "@message-queue-toolkit/sqs": "*", + "@message-queue-toolkit/core": "workspace:*", + "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", + "@message-queue-toolkit/s3-payload-store": "workspace:*", + "@message-queue-toolkit/sqs": "workspace:*", "@types/node": "^25.0.2", "@vitest/coverage-v8": "^4.0.15", "awilix": "^13.0.3", diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts new file mode 100644 index 00000000..72611d4a --- /dev/null +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts @@ -0,0 +1,104 @@ +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { SnsPermissionPublisher } from '../publishers/SnsPermissionPublisher.ts' +import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { SnsSqsPermissionConsumer } from './SnsSqsPermissionConsumer.ts' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' + +describe('SnsSqsPermissionConsumer - zstd codec', () => { + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + let publisher: SnsPermissionPublisher + let consumer: SnsSqsPermissionConsumer + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + beforeEach(async () => { + await testAdmin.deleteQueues(SnsSqsPermissionConsumer.CONSUMED_QUEUE_NAME) + await testAdmin.deleteTopics(SnsSqsPermissionConsumer.SUBSCRIBED_TOPIC_NAME) + + consumer = new SnsSqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + }) + publisher = new SnsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + }) + + await consumer.start() + await publisher.init() + }) + + afterEach(async () => { + await publisher.close() + await consumer.close() + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('publishes a compressed SNS message and consumer decompresses it correctly', async () => { + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'sns-codec-test-1', + messageType: 'add', + metadata: { info: 'hello sns zstd' }, + } + + await publisher.publish(message) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + }, 15000) + + it('consumer correctly handles multiple compressed SNS messages in sequence', async () => { + const messages: PERMISSIONS_ADD_MESSAGE_TYPE[] = [ + { id: 'sns-codec-seq-1', messageType: 'add' }, + { id: 'sns-codec-seq-2', messageType: 'add' }, + { id: 'sns-codec-seq-3', messageType: 'add' }, + ] + + for (const msg of messages) { + await publisher.publish(msg) + } + + for (const msg of messages) { + const result = await consumer.handlerSpy.waitForMessageWithId(msg.id, 'consumed') + expect(result.message).toMatchObject(msg) + } + }, 15000) + + it('consumer without codec option auto-detects and decompresses zstd messages from SNS', async () => { + // Consumer without explicit codec — decompression is auto-detected from envelope __codec field + const autoConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { + locatorConfig: { + queueUrl: consumer.subscriptionProps.queueUrl, + topicArn: consumer.subscriptionProps.topicArn, + subscriptionArn: consumer.subscriptionProps.subscriptionArn, + }, + }) + await autoConsumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'sns-codec-auto-1', + messageType: 'add', + } + await publisher.publish(message) + + const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await autoConsumer.close() + }, 15000) +}) diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts index 3155b3fd..c4f6ab03 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts @@ -40,6 +40,7 @@ type SnsSqsPermissionConsumerOptions = Pick< | 'maxRetryDuration' | 'payloadStoreConfig' | 'concurrentConsumersAmount' + | 'codec' > & { addPreHandlerBarrier?: ( message: SupportedMessages, @@ -147,6 +148,7 @@ export class SnsSqsPermissionConsumer extends AbstractSnsSqsConsumer< deleteIfExists: false, }, payloadStoreConfig: options.payloadStoreConfig, + codec: options.codec, consumerOverrides: options.consumerOverrides ?? { terminateVisibilityTimeout: true, // this allows to retry failed messages immediately }, diff --git a/packages/sns/test/publishers/SnsPermissionPublisher.ts b/packages/sns/test/publishers/SnsPermissionPublisher.ts index e5c2dda7..659f8d87 100644 --- a/packages/sns/test/publishers/SnsPermissionPublisher.ts +++ b/packages/sns/test/publishers/SnsPermissionPublisher.ts @@ -26,6 +26,7 @@ export class SnsPermissionPublisher extends AbstractSnsPublisher | 'payloadStoreConfig' | 'messageDeduplicationConfig' | 'enablePublisherDeduplication' + | 'codec' >, ) { super(dependencies, { @@ -40,6 +41,7 @@ export class SnsPermissionPublisher extends AbstractSnsPublisher deleteIfExists: false, }, payloadStoreConfig: options?.payloadStoreConfig, + codec: options?.codec, messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], handlerSpy: true, messageTypeResolver: { messageTypePath: 'messageType' }, diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 81a5c1b3..7ce99a76 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -10,7 +10,9 @@ import { type BarrierResult, type DeadLetterQueueOptions, DeduplicationRequesterEnum, + decompressMessageBody, HandlerContainer, + isCodecEnvelope, isMessageError, type MessageSchemaContainer, noopReleasableLock, @@ -891,6 +893,15 @@ export abstract class AbstractSqsConsumer< return ABORT_EARLY_EITHER } resolveMessageResult.result.body = retrieveOffloadedMessagePayloadResult.result + } else if (isCodecEnvelope(resolveMessageResult.result.body)) { + try { + resolveMessageResult.result.body = await decompressMessageBody( + resolveMessageResult.result.body, + ) + } catch (err) { + this.handleError(err as Error) + return ABORT_EARLY_EITHER + } } return resolveMessageResult diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index f4ed02d4..54a2e43e 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -5,7 +5,9 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, + compressMessageBody, DeduplicationRequesterEnum, + isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -204,11 +206,16 @@ export abstract class AbstractSqsPublisher options: SQSMessageOptions, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) + const jsonBody = JSON.stringify(payload) + const body = + this.codec && !isOffloadedPayloadPointerPayload(payload) + ? await compressMessageBody(jsonBody, this.codec) + : jsonBody // Options are already resolved in publish() before offloading const command = new SendMessageCommand({ QueueUrl: this.queueUrl, - MessageBody: JSON.stringify(payload), + MessageBody: body, MessageAttributes: attributes, ...options, }) diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 0e19409c..b6c36306 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -42,10 +42,10 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "*", - "@message-queue-toolkit/redis-message-deduplication-store": "*", - "@message-queue-toolkit/s3-payload-store": "*", - "@message-queue-toolkit/schemas": "*", + "@message-queue-toolkit/core": "workspace:*", + "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", + "@message-queue-toolkit/s3-payload-store": "workspace:*", + "@message-queue-toolkit/schemas": "workspace:*", "@types/node": "^25.0.2", "@vitest/coverage-v8": "^4.0.15", "awilix": "^13.0.3", diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts new file mode 100644 index 00000000..042050e2 --- /dev/null +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -0,0 +1,123 @@ +import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { compressMessageBody } from '@message-queue-toolkit/core' +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' +import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { SqsPermissionConsumer } from './SqsPermissionConsumer.ts' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' + +describe('SqsPermissionConsumer - zstd codec', () => { + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + let publisher: SqsPermissionPublisher + let consumer: SqsPermissionConsumer + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + beforeEach(async () => { + await testAdmin.deleteQueues(SqsPermissionConsumer.QUEUE_NAME) + + consumer = new SqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + deletionConfig: { deleteIfExists: false }, + }) + publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + }) + + await consumer.start() + await publisher.init() + }) + + afterEach(async () => { + await publisher.close() + await consumer.close(true) + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('publishes a compressed message and consumer decompresses it correctly', async () => { + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-test-1', + messageType: 'add', + metadata: { info: 'hello zstd' }, + } + + await publisher.publish(message) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + }) + + it('consumer correctly handles multiple compressed messages in sequence', async () => { + const messages: PERMISSIONS_ADD_MESSAGE_TYPE[] = [ + { id: 'codec-seq-1', messageType: 'add' }, + { id: 'codec-seq-2', messageType: 'add' }, + { id: 'codec-seq-3', messageType: 'add' }, + ] + + for (const msg of messages) { + await publisher.publish(msg) + } + + for (const msg of messages) { + const result = await consumer.handlerSpy.waitForMessageWithId(msg.id, 'consumed') + expect(result.message).toMatchObject(msg) + } + }) + + it('consumer decompresses a message compressed externally with zstd', async () => { + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-external-1', + messageType: 'add', + metadata: { source: 'external-compressor' }, + } + + // Simulate a publisher that compressed the message itself + const compressedBody = await compressMessageBody(JSON.stringify(message), 'zstd') + await diContainer.cradle.sqsClient.send( + new SendMessageCommand({ + QueueUrl: consumer.queueProps.url, + MessageBody: compressedBody, + }), + ) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + }) + + it('consumer without codec option still decompresses zstd messages (auto-detection)', async () => { + // Consumer without codec — auto-detects from envelope __codec field + const autoConsumer = new SqsPermissionConsumer(diContainer.cradle, { + locatorConfig: { queueUrl: consumer.queueProps.url }, + deletionConfig: { deleteIfExists: false }, + }) + await autoConsumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-auto-detect-1', + messageType: 'add', + } + await publisher.publish(message) + + const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await autoConsumer.close(true) + }) +}) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.ts index 3155d507..cfadac40 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.ts @@ -31,6 +31,7 @@ type SqsPermissionConsumerOptions = Pick< | 'payloadStoreConfig' | 'messageDeduplicationConfig' | 'enableConsumerDeduplication' + | 'codec' > & { addPreHandlerBarrier?: ( message: SupportedMessages, @@ -126,6 +127,7 @@ export class SqsPermissionConsumer extends AbstractSqsConsumer< payloadStoreConfig: options.payloadStoreConfig, messageDeduplicationConfig: options.messageDeduplicationConfig, enableConsumerDeduplication: options.enableConsumerDeduplication, + codec: options.codec, messageDeduplicationIdField: 'deduplicationId', messageDeduplicationOptionsField: 'deduplicationOptions', handlers: new MessageHandlerConfigBuilder< diff --git a/packages/sqs/test/publishers/SqsPermissionPublisher.ts b/packages/sqs/test/publishers/SqsPermissionPublisher.ts index 33ab0c15..5aa73c20 100644 --- a/packages/sqs/test/publishers/SqsPermissionPublisher.ts +++ b/packages/sqs/test/publishers/SqsPermissionPublisher.ts @@ -31,6 +31,7 @@ export class SqsPermissionPublisher extends AbstractSqsPublisher, ) { super(dependencies, { @@ -53,6 +54,7 @@ export class SqsPermissionPublisher extends AbstractSqsPublisher=23.1.0' ioredis: ^5.3.2 - '@message-queue-toolkit/s3-payload-store@3.0.0': - resolution: {integrity: sha512-AX2PI74CN9CBQWHT/nJBhUPR8E6beGodTsuSSlZ/zQvy6ViDcI4gEKxFViqKR2xai7PeLsqw+HWdkXhawwEqYA==} - peerDependencies: - '@aws-sdk/client-s3': ^3.596.0 - '@message-queue-toolkit/core': '>=24.0.0' - '@message-queue-toolkit/schemas@7.1.0': resolution: {integrity: sha512-JAzSQAHouympK/cEDBxsfEuS2Ifu1pv0a/NRvhNWfFlgW0TmsWT7SkYNERA7x89OK7PGk9PyDN88cV9l0gZ22Q==} peerDependencies: zod: '>=3.25.76 <5.0.0' - '@message-queue-toolkit/sqs@24.2.1': - resolution: {integrity: sha512-R3+XSzvBUStsjbjKWtTIb64PD/4+cUM0CrqiATuPj4XUK0WTzCYhHF42okk+6k1Nv5H3U7P1gGXgeotBGRKadg==} - peerDependencies: - '@aws-sdk/client-sqs': ^3.632.0 - '@message-queue-toolkit/core': '>=25.0.0' - zod: '>=3.25.76 <5.0.0' + '@mongodb-js/zstd@7.0.0': + resolution: {integrity: sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA==} + engines: {node: '>= 20.19.0'} '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} @@ -1567,6 +1561,9 @@ packages: bintrees@1.0.2: resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + bowser@2.14.1: resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} @@ -1584,6 +1581,9 @@ packages: buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bullmq@5.76.8: resolution: {integrity: sha512-v3WTwA8diFtsADaJ8eK2ozyi2CYK9rDZCeoKF+dIPF/MUL8HxAOa3H72Gmu1lC4yKlho6t1PwNr/QpDVqaNEZQ==} engines: {node: '>=12.22.0'} @@ -1596,6 +1596,9 @@ packages: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -1653,6 +1656,14 @@ packages: supports-color: optional: true + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -1729,6 +1740,10 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -1823,6 +1838,9 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1859,6 +1877,9 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1945,9 +1966,15 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + ioredis@5.10.1: resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==} engines: {node: '>=12.22.0'} @@ -2168,6 +2195,10 @@ packages: engines: {node: '>=10.0.0'} hasBin: true + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -2183,6 +2214,9 @@ packages: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + mnemonist@0.40.4: resolution: {integrity: sha512-ZAv+KNavneRVzu4tUeOgzkScI3W5BGwZ3rkxIpKtzzVgfTtWQFN1CgX0U72cyvyh3iTuHL3SiSmrQxTlryEIcw==} @@ -2201,9 +2235,20 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + + node-abi@3.92.0: + resolution: {integrity: sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==} + engines: {node: '>=10'} + node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-addon-api@8.7.0: + resolution: {integrity: sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==} + engines: {node: ^18 || ^20 || >= 21} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2302,6 +2347,12 @@ packages: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. + hasBin: true + process-warning@4.0.1: resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} @@ -2333,6 +2384,10 @@ packages: quick-format-unescaped@4.0.4: resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} @@ -2449,6 +2504,12 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} @@ -2460,12 +2521,6 @@ packages: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} - sqs-consumer@14.2.8: - resolution: {integrity: sha512-Iq3AdASfsxY2s3KL88aPRGgvST3gviZphL5teTKxUqDZzXRIB8s6SWATft+v7YaGMspCs8BwrOKo9vD6ltcJmw==} - engines: {node: '>=20.0.0'} - peerDependencies: - '@aws-sdk/client-sqs': ^3.1018.0 - sqs-consumer@15.0.1: resolution: {integrity: sha512-cDcSkekXxhLMn3u+FSnAG3Ryh6jwHMPna1/oqs2dUL0gvUXOLLMa2Yzu0K4QqwVyaH0MDZw1IU7+W+W618/xcw==} engines: {node: '>=22.0.0'} @@ -2506,6 +2561,10 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + strip-json-comments@5.0.3: resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} engines: {node: '>=14.16'} @@ -2524,6 +2583,13 @@ packages: resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} engines: {node: '>=20'} + tar-fs@2.1.4: + resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} @@ -2579,6 +2645,9 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + turbo@2.9.14: resolution: {integrity: sha512-BQqXRr4UoWI3UPFrtznCLykYHxwxWh53iCB57x092jPMjIlW1wnm3N895g5irpiXmnxUhREBB0n6+y8BHhs4nw==} hasBin: true @@ -3378,24 +3447,14 @@ snapshots: - supports-color - zod - '@message-queue-toolkit/s3-payload-store@3.0.0(@aws-sdk/client-s3@3.1048.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3))': - dependencies: - '@aws-sdk/client-s3': 3.1048.0 - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) - '@message-queue-toolkit/schemas@7.1.0(zod@4.4.3)': dependencies: zod: 4.4.3 - '@message-queue-toolkit/sqs@24.2.1(@aws-sdk/client-sqs@3.1048.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3))(zod@4.4.3)': + '@mongodb-js/zstd@7.0.0': dependencies: - '@aws-sdk/client-sqs': 3.1048.0 - '@lokalise/node-core': 14.8.1(zod@4.4.3) - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) - sqs-consumer: 14.2.8(@aws-sdk/client-sqs@3.1048.0) - zod: 4.4.3 - transitivePeerDependencies: - - supports-color + node-addon-api: 8.7.0 + prebuild-install: 7.1.3 '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -3861,6 +3920,12 @@ snapshots: bintrees@1.0.2: {} + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + bowser@2.14.1: {} brace-expansion@2.1.0: @@ -3877,6 +3942,11 @@ snapshots: buffer-equal-constant-time@1.0.1: {} + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + bullmq@5.76.8: dependencies: cron-parser: 4.9.0 @@ -3895,6 +3965,8 @@ snapshots: chai@6.2.2: {} + chownr@1.1.4: {} + cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -3939,6 +4011,12 @@ snapshots: dependencies: ms: 2.1.3 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + + deep-extend@0.6.0: {} + delayed-stream@1.0.0: {} denque@2.1.0: {} @@ -4007,6 +4085,8 @@ snapshots: event-target-shim@5.0.1: {} + expand-template@2.0.3: {} + expect-type@1.3.0: {} extend@3.0.2: {} @@ -4138,6 +4218,8 @@ snapshots: dependencies: fetch-blob: 3.2.0 + fs-constants@1.0.0: {} + fsevents@2.3.3: optional: true @@ -4199,6 +4281,8 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + github-from-package@0.0.0: {} + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -4320,8 +4404,12 @@ snapshots: transitivePeerDependencies: - supports-color + ieee754@1.2.1: {} + inherits@2.0.4: {} + ini@1.3.8: {} + ioredis@5.10.1: dependencies: '@ioredis/commands': 1.5.1 @@ -4506,6 +4594,8 @@ snapshots: mime@3.0.0: {} + mimic-response@3.1.0: {} + minimatch@10.2.5: dependencies: brace-expansion: 5.0.6 @@ -4518,6 +4608,8 @@ snapshots: minipass@7.1.3: {} + mkdirp-classic@0.5.3: {} + mnemonist@0.40.4: dependencies: obliterator: 2.0.5 @@ -4542,8 +4634,16 @@ snapshots: nanoid@3.3.12: {} + napi-build-utils@2.0.0: {} + + node-abi@3.92.0: + dependencies: + semver: 7.8.0 + node-abort-controller@3.1.1: {} + node-addon-api@8.7.0: {} + node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -4645,6 +4745,21 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.92.0 + pump: 3.0.4 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.4 + tunnel-agent: 0.6.0 + process-warning@4.0.1: {} process-warning@5.0.0: {} @@ -4687,6 +4802,13 @@ snapshots: quick-format-unescaped@4.0.4: {} + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + readable-stream@3.6.2: dependencies: inherits: 2.0.4 @@ -4799,6 +4921,14 @@ snapshots: signal-exit@4.1.0: {} + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -4807,13 +4937,6 @@ snapshots: split2@4.2.0: {} - sqs-consumer@14.2.8(@aws-sdk/client-sqs@3.1048.0): - dependencies: - '@aws-sdk/client-sqs': 3.1048.0 - debug: 4.4.3 - transitivePeerDependencies: - - supports-color - sqs-consumer@15.0.1(@aws-sdk/client-sqs@3.1048.0): dependencies: '@aws-sdk/client-sqs': 3.1048.0 @@ -4857,6 +4980,8 @@ snapshots: dependencies: ansi-regex: 6.2.2 + strip-json-comments@2.0.1: {} + strip-json-comments@5.0.3: {} strnum@2.3.0: {} @@ -4869,6 +4994,21 @@ snapshots: tagged-tag@1.0.0: {} + tar-fs@2.1.4: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.4 + tar-stream: 2.2.0 + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + tdigest@0.1.2: dependencies: bintrees: 1.0.2 @@ -4926,6 +5066,10 @@ snapshots: tslib@2.8.1: {} + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + turbo@2.9.14: optionalDependencies: '@turbo/darwin-64': 2.9.14 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5918a4e3..35668904 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ packages: # pnpm 11 auto-appends new entries here whenever a dep has scripts, prompting # an explicit decision per package. allowBuilds: + '@mongodb-js/zstd': true msgpackr-extract: false protobufjs: false From b3b0fcfd66c23d0884b61984796a364ea0c70590 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Mon, 18 May 2026 18:28:35 +0200 Subject: [PATCH 2/9] refactor: move zstd implementation out of core, harden codec guardrails, fix test races - Move @mongodb-js/zstd out of core into sqs; core now only defines MessageCodecHandler interface + pure envelope types (CodecEnvelope, isCodecEnvelope, MessageCodec) with no native dependencies - Add packages/sqs/lib/codec/sqsCodecHandler.ts: ZstdCodecHandler, resolveCodecHandler, and the concrete compressMessageBody / decompressMessageBody helpers - AbstractSqsPublisher and AbstractSqsConsumer import from local codec; AbstractSnsPublisher imports compressMessageBody from @message-queue-toolkit/sqs - Strengthen isCodecEnvelope to assert typeof __data === 'string' so Buffer.from downstream is guaranteed a string - Fix race condition in SQS codec auto-detection test: use a dedicated queue (user_permissions_multi-auto-detect) instead of sharing the beforeEach consumer's queue, eliminating both the steal-race and the localstack long-poll timing issue - Fix race condition in SNS codec auto-detection test: stop the original consumer before starting autoConsumer, reassign consumer = autoConsumer so afterEach handles cleanup without a double-close Co-Authored-By: Claude Sonnet 4.6 --- packages/core/lib/codec/messageCodec.ts | 32 +++++------------ packages/core/lib/index.ts | 3 +- packages/core/package.json | 1 - packages/sns/lib/sns/AbstractSnsPublisher.ts | 3 +- .../SnsSqsPermissionConsumer.codec.spec.ts | 7 ++-- packages/sqs/lib/codec/sqsCodecHandler.ts | 36 +++++++++++++++++++ packages/sqs/lib/index.ts | 6 ++++ packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 6 ++-- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 2 +- packages/sqs/package.json | 1 + .../SqsPermissionConsumer.codec.spec.ts | 20 ++++++++--- pnpm-lock.yaml | 6 ++-- 12 files changed, 81 insertions(+), 42 deletions(-) create mode 100644 packages/sqs/lib/codec/sqsCodecHandler.ts diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 4709a8de..8b6f0253 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -1,5 +1,3 @@ -import { compress, decompress } from '@mongodb-js/zstd' - export const SUPPORTED_CODECS = ['zstd'] as const export type MessageCodec = (typeof SUPPORTED_CODECS)[number] @@ -11,33 +9,19 @@ export type CodecEnvelope = { [DATA_FIELD]: string } +export interface MessageCodecHandler { + compress(data: Buffer): Promise + decompress(data: Buffer): Promise +} + export function isCodecEnvelope(value: unknown): value is CodecEnvelope { + const record = value as Record return ( typeof value === 'object' && value !== null && CODEC_FIELD in value && DATA_FIELD in value && - SUPPORTED_CODECS.includes((value as Record)[CODEC_FIELD] as MessageCodec) + SUPPORTED_CODECS.includes(record[CODEC_FIELD] as MessageCodec) && + typeof record[DATA_FIELD] === 'string' ) } - -export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { - if (codec === 'zstd') { - const compressed = await compress(Buffer.from(jsonBody, 'utf8')) - const envelope: CodecEnvelope = { - [CODEC_FIELD]: codec, - [DATA_FIELD]: compressed.toString('base64'), - } - return JSON.stringify(envelope) - } - throw new Error(`Unsupported codec: ${codec}`) -} - -export async function decompressMessageBody(envelope: CodecEnvelope): Promise { - if (envelope[CODEC_FIELD] === 'zstd') { - const compressed = Buffer.from(envelope[DATA_FIELD], 'base64') - const decompressed = await decompress(compressed) - return JSON.parse(decompressed.toString('utf8')) - } - throw new Error(`Unsupported codec: ${envelope[CODEC_FIELD]}`) -} diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index 97855fb7..c3334659 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,9 +1,8 @@ export { type CodecEnvelope, - compressMessageBody, - decompressMessageBody, isCodecEnvelope, type MessageCodec, + type MessageCodecHandler, SUPPORTED_CODECS, } from './codec/messageCodec.ts' export { DoNotProcessMessageError } from './errors/DoNotProcessError.ts' diff --git a/packages/core/package.json b/packages/core/package.json index 12b5e3be..041db97b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -28,7 +28,6 @@ "@lokalise/node-core": "^14.2.0", "@lokalise/universal-ts-utils": "^4.5.1", "@message-queue-toolkit/schemas": "^7.0.0", - "@mongodb-js/zstd": "^7.0.0", "dot-prop": "^10.1.0", "fast-equals": "^6.0.0", "json-stream-stringify": "^3.1.6", diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 6e9f95f7..81c98269 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -5,7 +5,6 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, - compressMessageBody, DeduplicationRequesterEnum, isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, @@ -15,7 +14,7 @@ import { type QueuePublisherOptions, type ResolvedMessage, } from '@message-queue-toolkit/core' -import { resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' +import { compressMessageBody, resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' import { calculateOutgoingMessageSize, validateFifoTopicName } from '../utils/snsUtils.ts' diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts index 72611d4a..e859c317 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts @@ -80,6 +80,9 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { }, 15000) it('consumer without codec option auto-detects and decompresses zstd messages from SNS', async () => { + // Stop the beforeEach consumer so it cannot steal messages from the shared queue + await consumer.close() + // Consumer without explicit codec — decompression is auto-detected from envelope __codec field const autoConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { locatorConfig: { @@ -89,6 +92,8 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { }, }) await autoConsumer.start() + // Reassign so afterEach closes autoConsumer instead of the already-closed consumer + consumer = autoConsumer const message: PERMISSIONS_ADD_MESSAGE_TYPE = { id: 'sns-codec-auto-1', @@ -98,7 +103,5 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') expect(result.message).toMatchObject(message) - - await autoConsumer.close() }, 15000) }) diff --git a/packages/sqs/lib/codec/sqsCodecHandler.ts b/packages/sqs/lib/codec/sqsCodecHandler.ts new file mode 100644 index 00000000..4a095e21 --- /dev/null +++ b/packages/sqs/lib/codec/sqsCodecHandler.ts @@ -0,0 +1,36 @@ +import type { CodecEnvelope, MessageCodec, MessageCodecHandler } from '@message-queue-toolkit/core' +import { compress, decompress } from '@mongodb-js/zstd' + +export class ZstdCodecHandler implements MessageCodecHandler { + compress(data: Buffer): Promise { + return compress(data) + } + + decompress(data: Buffer): Promise { + return decompress(data) + } +} + +const ZSTD_HANDLER = new ZstdCodecHandler() + +export function resolveCodecHandler(codec: MessageCodec): MessageCodecHandler { + if (codec === 'zstd') return ZSTD_HANDLER + throw new Error(`Unsupported codec: ${codec}`) +} + +export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { + const handler = resolveCodecHandler(codec) + const compressed = await handler.compress(Buffer.from(jsonBody, 'utf8')) + const envelope: CodecEnvelope = { + __codec: codec, + __data: compressed.toString('base64'), + } + return JSON.stringify(envelope) +} + +export async function decompressMessageBody(envelope: CodecEnvelope): Promise { + const handler = resolveCodecHandler(envelope.__codec) + const compressed = Buffer.from(envelope.__data, 'base64') + const decompressed = await handler.decompress(compressed) + return JSON.parse(decompressed.toString('utf8')) +} diff --git a/packages/sqs/lib/index.ts b/packages/sqs/lib/index.ts index 49d3803c..7705888f 100644 --- a/packages/sqs/lib/index.ts +++ b/packages/sqs/lib/index.ts @@ -1,3 +1,9 @@ +export { + compressMessageBody, + decompressMessageBody, + resolveCodecHandler, + ZstdCodecHandler, +} from './codec/sqsCodecHandler.ts' export { SqsConsumerErrorResolver } from './errors/SqsConsumerErrorResolver.ts' export { FakeConsumerErrorResolver } from './fakes/FakeConsumerErrorResolver.ts' export { TestSqsPublisher, type TestSqsPublishOptions } from './fakes/TestSqsPublisher.ts' diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 7ce99a76..e45e94f9 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -10,7 +10,6 @@ import { type BarrierResult, type DeadLetterQueueOptions, DeduplicationRequesterEnum, - decompressMessageBody, HandlerContainer, isCodecEnvelope, isMessageError, @@ -28,6 +27,7 @@ import { import type { ConsumerOptions } from 'sqs-consumer' import { Consumer } from 'sqs-consumer' import type { ZodSchema } from 'zod/v4' +import { decompressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { hasOffloadedPayload } from '../utils/messageUtils.ts' import { deleteSqs, initSqs } from '../utils/sqsInitter.ts' @@ -880,7 +880,7 @@ export abstract class AbstractSqsConsumer< } // Empty content for whatever reason - if (!resolveMessageResult.result || !resolveMessageResult.result.body) { + if (!resolveMessageResult.result?.body) { return ABORT_EARLY_EITHER } @@ -916,7 +916,7 @@ export abstract class AbstractSqsConsumer< const resolvedMessage = resolveMessageResult.result // Empty content for whatever reason - if (!resolvedMessage || !resolvedMessage.body) return ABORT_EARLY_EITHER + if (!resolvedMessage?.body) return ABORT_EARLY_EITHER // @ts-expect-error if (this.messageIdField in resolvedMessage.body) { diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index 54a2e43e..f6cc3f7f 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -5,7 +5,6 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, - compressMessageBody, DeduplicationRequesterEnum, isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, @@ -16,6 +15,7 @@ import { type ResolvedMessage, } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod/v4' +import { compressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { resolveOutgoingMessageAttributes } from '../utils/messageUtils.ts' diff --git a/packages/sqs/package.json b/packages/sqs/package.json index b6c36306..823da8db 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -29,6 +29,7 @@ }, "dependencies": { "@lokalise/node-core": "^14.6.1", + "@mongodb-js/zstd": "^7.0.0", "sqs-consumer": "^15.0.1" }, "peerDependencies": { diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 042050e2..b7229361 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,8 +1,8 @@ import { SendMessageCommand } from '@aws-sdk/client-sqs' -import { compressMessageBody } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { compressMessageBody } from '../../lib/codec/sqsCodecHandler.ts' import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' @@ -102,9 +102,20 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) it('consumer without codec option still decompresses zstd messages (auto-detection)', async () => { + // Use a dedicated queue so only autoConsumer polls it — avoids both the race + // condition (shared queue) and localstack long-poll timing issues (abort + restart) + const autoQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-auto-detect` + await testAdmin.deleteQueues(autoQueueName) + + const autoPublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: autoQueueName } }, + }) + await autoPublisher.init() + // Consumer without codec — auto-detects from envelope __codec field const autoConsumer = new SqsPermissionConsumer(diContainer.cradle, { - locatorConfig: { queueUrl: consumer.queueProps.url }, + creationConfig: { queue: { QueueName: autoQueueName } }, deletionConfig: { deleteIfExists: false }, }) await autoConsumer.start() @@ -113,11 +124,12 @@ describe('SqsPermissionConsumer - zstd codec', () => { id: 'codec-auto-detect-1', messageType: 'add', } - await publisher.publish(message) + await autoPublisher.publish(message) const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') expect(result.message).toMatchObject(message) + await autoPublisher.close() await autoConsumer.close(true) - }) + }, 15000) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01681c80..53770fcc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -75,9 +75,6 @@ importers: '@message-queue-toolkit/schemas': specifier: ^7.0.0 version: 7.1.0(zod@4.4.3) - '@mongodb-js/zstd': - specifier: ^7.0.0 - version: 7.0.0 dot-prop: specifier: ^10.1.0 version: 10.1.0 @@ -548,6 +545,9 @@ importers: '@lokalise/node-core': specifier: ^14.6.1 version: 14.8.1(zod@4.4.3) + '@mongodb-js/zstd': + specifier: ^7.0.0 + version: 7.0.0 sqs-consumer: specifier: ^15.0.1 version: 15.0.1(@aws-sdk/client-sqs@3.1048.0) From 9950db2fba16696d31c516cb4cb5681727899962 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 09:16:40 +0200 Subject: [PATCH 3/9] ci: rebuild @mongodb-js/zstd native binary after --ignore-scripts install The native addon requires node-gyp compilation. pnpm install runs with --ignore-scripts in CI, so the binary is never built. pnpm rebuild explicitly compiles it regardless of that flag. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.common.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.common.yml b/.github/workflows/ci.common.yml index 4875f728..94f3d65c 100644 --- a/.github/workflows/ci.common.yml +++ b/.github/workflows/ci.common.yml @@ -36,6 +36,9 @@ jobs: - name: Install run: pnpm install --frozen-lockfile --ignore-scripts + - name: Build native dependencies + run: pnpm rebuild @mongodb-js/zstd + - name: Build TS env: PACKAGE_NAME: ${{ inputs.package_name }} From 2dfdb6e334511fd1da3c0a4690fe0fb14cf7db82 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 09:35:30 +0200 Subject: [PATCH 4/9] feat(sqs): add codec benchmarks for publish and consume throughput Measures wall-clock time and msg/s for 50 messages with and without zstd compression across small (~80 B) and large (~6 KB) payloads. Each run deletes its queues before and after so no resources are left behind. Run with: pnpm --filter @message-queue-toolkit/sqs bench Co-Authored-By: Claude Sonnet 4.6 --- packages/sqs/bench/codec.bench.ts | 206 ++++++++++++++++++++++++++++ packages/sqs/package.json | 1 + packages/sqs/vitest.bench.config.ts | 14 ++ 3 files changed, 221 insertions(+) create mode 100644 packages/sqs/bench/codec.bench.ts create mode 100644 packages/sqs/vitest.bench.config.ts diff --git a/packages/sqs/bench/codec.bench.ts b/packages/sqs/bench/codec.bench.ts new file mode 100644 index 00000000..12735c67 --- /dev/null +++ b/packages/sqs/bench/codec.bench.ts @@ -0,0 +1,206 @@ +/** + * Codec benchmarks — publish and consume throughput with vs without zstd compression. + * + * Run: pnpm --filter @message-queue-toolkit/sqs bench + * + * Each benchmark pre-fills queues (consume) or sends N messages (publish) and + * measures wall-clock time, reporting msg/s and the overhead percentage. + * All queues are deleted before and after each case. + */ +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, beforeAll, describe, it } from 'vitest' + +import { SqsPermissionConsumer } from '../test/consumers/SqsPermissionConsumer.ts' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from '../test/consumers/userConsumerSchemas.ts' +import { SqsPermissionPublisher } from '../test/publishers/SqsPermissionPublisher.ts' +import type { TestAwsResourceAdmin } from '../test/utils/testAdmin.ts' +import type { Dependencies } from '../test/utils/testContext.ts' +import { registerDependencies } from '../test/utils/testContext.ts' + +// ─── Configuration ──────────────────────────────────────────────────────────── + +const N = 50 + +/** Small message with minimal payload (~80 B serialised). */ +const SMALL_META: undefined = undefined + +/** + * Large message with repetitive text (~6 KB serialised). + * Repetitive content compresses very well, showing the realistic best case. + */ +const LARGE_META: Record = { + description: 'The quick brown fox jumps over the lazy dog. '.repeat(60), + items: Array.from({ length: 80 }, (_, i) => ({ + id: `item-${i}`, + value: `value-number-${i}`, + enabled: i % 2 === 0, + })), +} + +const CASES = [ + { label: 'small payload (~80 B) ', suffix: 'sm', meta: SMALL_META }, + { label: 'large payload (~6 KB) ', suffix: 'lg', meta: LARGE_META }, +] as const + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function makeMessages( + prefix: string, + count: number, + meta: Record | undefined, +): PERMISSIONS_ADD_MESSAGE_TYPE[] { + return Array.from({ length: count }, (_, i) => ({ + id: `${prefix}-${i}`, + messageType: 'add' as const, + ...(meta !== undefined ? { metadata: meta } : {}), + })) +} + +function printRow(label: string, count: number, plainMs: number, codecMs: number): void { + const tps = (ms: number) => ((count / ms) * 1000).toFixed(0).padStart(6) + const diff = codecMs - plainMs + const pct = ((diff / plainMs) * 100).toFixed(1) + const sign = diff >= 0 ? '+' : '' + console.log( + ` ${label}` + + ` plain: ${String(plainMs.toFixed(0)).padStart(5)} ms (${tps(plainMs)} msg/s)` + + ` zstd: ${String(codecMs.toFixed(0)).padStart(5)} ms (${tps(codecMs)} msg/s)` + + ` overhead: ${sign}${pct}%`, + ) +} + +// ─── Suite ──────────────────────────────────────────────────────────────────── + +describe('SQS codec benchmarks', () => { + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + // ── Publish ──────────────────────────────────────────────────────────────── + + it( + `publish: with vs without zstd (${N} messages)`, + async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` PUBLISH BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-pub-plain-${suffix}` + const codecQ = `bench-pub-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpp', N, meta) + const codecMsgs = makeMessages('bcp', N, meta) + + // ── Plain publish ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + const t0 = performance.now() + for (const msg of plainMsgs) await plainPub.publish(msg) + const plainMs = performance.now() - t0 + await plainPub.close() + + // ── Codec publish ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + const t1 = performance.now() + for (const msg of codecMsgs) await codecPub.publish(msg) + const codecMs = performance.now() - t1 + await codecPub.close() + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, + 120_000, + ) + + // ── Consume ──────────────────────────────────────────────────────────────── + + it( + `consume: with vs without zstd (${N} messages)`, + async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` CONSUME BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-con-plain-${suffix}` + const codecQ = `bench-con-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpc', N, meta) + const codecMsgs = makeMessages('bcc', N, meta) + + // ── Pre-fill plain queue ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + for (const msg of plainMsgs) await plainPub.publish(msg) + await plainPub.close() + + // ── Pre-fill codec queue ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + for (const msg of codecMsgs) await codecPub.publish(msg) + await codecPub.close() + + // ── Measure plain consume ── + // deletionConfig: { deleteIfExists: false } preserves the pre-filled queue + const plainCon = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await plainCon.start() + const t2 = performance.now() + await Promise.all( + plainMsgs.map((m) => plainCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const plainMs = performance.now() - t2 + await plainCon.close(true) + + // ── Measure codec consume ── + const codecCon = new SqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await codecCon.start() + const t3 = performance.now() + await Promise.all( + codecMsgs.map((m) => codecCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const codecMs = performance.now() - t3 + await codecCon.close(true) + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, + 120_000, + ) +}) diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 823da8db..acf138f3 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -19,6 +19,7 @@ "scripts": { "build": "pnpm run clean && tsc --project tsconfig.build.json", "clean": "rimraf dist", + "bench": "vitest run --config vitest.bench.config.ts --reporter=verbose", "test": "vitest", "test:coverage": "pnpm run test --coverage", "lint": "biome check && tsc", diff --git a/packages/sqs/vitest.bench.config.ts b/packages/sqs/vitest.bench.config.ts new file mode 100644 index 00000000..fd4e580e --- /dev/null +++ b/packages/sqs/vitest.bench.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vitest/config' + +// biome-ignore lint/style/noDefaultExport: vite expects default export +export default defineConfig({ + test: { + globals: true, + watch: false, + mockReset: true, + pool: 'threads', + maxWorkers: 1, + setupFiles: ['test/utils/vitest.setup.ts'], + include: ['bench/**/*.ts'], + }, +}) From c1d83e7b6e47fadb8eba448c504a0843ebaeecd1 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 10:12:02 +0200 Subject: [PATCH 5/9] feat(sqs/core): replace @mongodb-js/zstd with Node.js built-in zlib and add codec documentation Switch from @mongodb-js/zstd (native node-gyp addon requiring Python and a C++ toolchain) to zlib.zstdCompress/zstdDecompress built into Node.js 22+. This removes 24 transitive packages, drops the pnpm rebuild CI step, and eliminates native build requirements for end users of the package. Refactor MessageCodec to use MessageCodecEnum object pattern, enabling MessageCodecEnum.ZSTD usage alongside the plain string literal. Add JSDoc to MessageCodecEnum, MessageCodecHandler, and the codec option in queueOptionsTypes. Add a Message Compression section to the SQS README with publisher/consumer examples and auto-detection behaviour, and reference it from the SNS README. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.common.yml | 3 - biome.json | 14 +- packages/core/lib/codec/messageCodec.ts | 28 ++- packages/core/lib/index.ts | 2 +- packages/core/lib/types/queueOptionsTypes.ts | 23 +- packages/sns/README.md | 5 + packages/sqs/README.md | 58 ++++++ packages/sqs/bench/codec.bench.ts | 208 +++++++++---------- packages/sqs/lib/codec/sqsCodecHandler.ts | 13 +- packages/sqs/package.json | 1 - pnpm-lock.yaml | 185 ----------------- 11 files changed, 229 insertions(+), 311 deletions(-) diff --git a/.github/workflows/ci.common.yml b/.github/workflows/ci.common.yml index 94f3d65c..4875f728 100644 --- a/.github/workflows/ci.common.yml +++ b/.github/workflows/ci.common.yml @@ -36,9 +36,6 @@ jobs: - name: Install run: pnpm install --frozen-lockfile --ignore-scripts - - name: Build native dependencies - run: pnpm rebuild @mongodb-js/zstd - - name: Build TS env: PACKAGE_NAME: ${{ inputs.package_name }} diff --git a/biome.json b/biome.json index e55fa69d..b0390747 100644 --- a/biome.json +++ b/biome.json @@ -15,5 +15,17 @@ "noUnusedPrivateClassMembers": "off" } } - } + }, + "overrides": [ + { + "includes": ["**/bench/**"], + "linter": { + "rules": { + "suspicious": { + "noConsole": "off" + } + } + } + } + ] } diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 8b6f0253..cc7fab0f 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -1,5 +1,20 @@ -export const SUPPORTED_CODECS = ['zstd'] as const -export type MessageCodec = (typeof SUPPORTED_CODECS)[number] +type ObjectValues = T[keyof T] + +/** + * Supported message compression codecs. + * + * Use the enum values instead of raw strings so that adding a new codec in + * the future is a single-place change and consumers benefit from IDE + * auto-complete. + * + * @example + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) + */ +export const MessageCodecEnum = { + /** zstd compression via Node.js built-in `zlib` (requires Node.js 22+). */ + ZSTD: 'zstd', +} as const +export type MessageCodec = ObjectValues const CODEC_FIELD = '__codec' const DATA_FIELD = '__data' @@ -9,6 +24,13 @@ export type CodecEnvelope = { [DATA_FIELD]: string } +/** + * Low-level interface for a compression codec. + * + * Implement this interface to plug in a custom compression algorithm. + * The built-in implementation (`ZstdCodecHandler` in `@message-queue-toolkit/sqs`) + * uses Node.js built-in `zlib` zstd support. + */ export interface MessageCodecHandler { compress(data: Buffer): Promise decompress(data: Buffer): Promise @@ -21,7 +43,7 @@ export function isCodecEnvelope(value: unknown): value is CodecEnvelope { value !== null && CODEC_FIELD in value && DATA_FIELD in value && - SUPPORTED_CODECS.includes(record[CODEC_FIELD] as MessageCodec) && + (Object.values(MessageCodecEnum) as string[]).includes(record[CODEC_FIELD] as string) && typeof record[DATA_FIELD] === 'string' ) } diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index c3334659..6d23f0ca 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -2,8 +2,8 @@ export { type CodecEnvelope, isCodecEnvelope, type MessageCodec, + MessageCodecEnum, type MessageCodecHandler, - SUPPORTED_CODECS, } from './codec/messageCodec.ts' export { DoNotProcessMessageError } from './errors/DoNotProcessError.ts' export { diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index 3310ca5b..c8ae6d7c 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -141,11 +141,24 @@ export type CommonQueueOptions = { payloadStoreConfig?: PayloadStoreConfig messageDeduplicationConfig?: MessageDeduplicationConfig /** - * Codec to use for compressing outgoing messages and decompressing incoming messages. - * When set on a publisher, messages are compressed before sending. - * When set on a consumer, it signals that incoming messages may be compressed. - * Compressed messages are self-describing (the codec is embedded in the message envelope), - * so consumers can decompress even without this option explicitly set. + * Compression codec applied to message bodies. + * + * - **Publisher**: every outgoing message body is compressed and wrapped in a + * self-describing envelope `{ __codec: 'zstd', __data: '' }`. + * - **Consumer**: when set, the consumer expects compressed messages. + * Even without this option, consumers auto-detect and decompress any message + * that carries a codec envelope, so mixed queues work transparently. + * + * Uses Node.js built-in `zlib` zstd support — **requires Node.js 22+**. + * + * @example + * import { MessageCodecEnum } from '@message-queue-toolkit/core' + * + * // Publisher + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) + * + * // Consumer (optional — auto-detection handles it even without this) + * new MyConsumer(deps, { codec: MessageCodecEnum.ZSTD }) */ codec?: MessageCodec } diff --git a/packages/sns/README.md b/packages/sns/README.md index ed6b7fb6..34112836 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -58,6 +58,7 @@ npm install @message-queue-toolkit/sns @message-queue-toolkit/sqs @message-queue - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Cross-account and cross-region publishing** +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) ## Core Concepts @@ -683,6 +684,9 @@ await consumer.start() // Optional - Payload Offloading (same as SQS) payloadStoreConfig: { /* ... */ }, + // Optional - Compression (Node.js 22+ required) + codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd + // Optional - Deletion deletionConfig: { /* ... */ }, } @@ -1000,6 +1004,7 @@ SNS consumers inherit all advanced features from SQS consumers. See the SQS READ - **[Message Retry Logic](../sqs/README.md#message-retry-logic)** - Exponential backoff and retry limits - **[Message Deduplication](../sqs/README.md#message-deduplication)** - Publisher and consumer-level deduplication - **[Payload Offloading](../sqs/README.md#payload-offloading)** - S3 storage for large messages +- **[Message Compression](../sqs/README.md#message-compression)** - zstd compression via Node.js built-in `zlib` - **[Message Handlers](../sqs/README.md#message-handlers)** - Type-safe handler configuration - **[Pre-handlers and Barriers](../sqs/README.md#pre-handlers-and-barriers)** - Middleware and message dependencies - **[Handler Spies](../sqs/README.md#handler-spies)** - Testing async message flows diff --git a/packages/sqs/README.md b/packages/sqs/README.md index 94bdaf57..f37edd79 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -24,6 +24,7 @@ for publishing and consuming messages from both standard and FIFO SQS queues. - [Message Retry Logic](#message-retry-logic) - [Message Deduplication](#message-deduplication) - [Payload Offloading](#payload-offloading) + - [Message Compression](#message-compression) - [Message Handlers](#message-handlers) - [Pre-handlers and Barriers](#pre-handlers-and-barriers) - [Handler Spies](#handler-spies) @@ -62,6 +63,7 @@ npm install @message-queue-toolkit/sqs @message-queue-toolkit/core - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Automatic queue creation** with validation +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) ## Core Concepts @@ -460,6 +462,9 @@ When using `locatorConfig`, you connect to an existing queue without creating it maxPayloadSize: 1024 * 1024, // 1 MiB }, + // Optional - Compression (Node.js 22+ required) + codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd + // Optional - Deletion deletionConfig: { deleteIfExists: false, // Delete queue on init @@ -531,6 +536,11 @@ When using `locatorConfig`, you connect to an existing queue without creating it payloadStore: s3Store, }, + // Optional - Compression (Node.js 22+ required) + // Auto-detection is always active: consumers decompress codec envelopes + // even without this option set. + codec: MessageCodecEnum.ZSTD, + // Optional - Other logMessages: false, handlerSpy: true, @@ -791,6 +801,54 @@ await publisher.publish({ **Note:** Payload cleanup is the responsibility of the store (e.g., S3 lifecycle policies). +### Message Compression + +Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js 22+**. + +Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __codec: 'zstd', __data: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. + +#### Publisher + +```typescript +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +class MyPublisher extends AbstractSqsPublisher { + constructor(deps: SQSDependencies) { + super(deps, { + codec: MessageCodecEnum.ZSTD, // compress every outgoing message + creationConfig: { queue: { QueueName: 'my-queue' } }, + // ... + }) + } +} +``` + +#### Consumer + +```typescript +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +class MyConsumer extends AbstractSqsConsumer { + constructor(deps: SQSConsumerDependencies) { + super(deps, { + // Optional: explicitly declare that messages are compressed. + // Without this, consumers still auto-detect and decompress codec envelopes. + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: 'my-queue' } }, + handlers: new MessageHandlerConfigBuilder() + .addConfig(MySchema, myHandler) + .build(), + }, executionContext) + } +} +``` + +#### Notes + +- Compression is applied **after** schema validation and **before** the SQS `SendMessage` call. +- Compressed payloads are still subject to the SQS 256 KB message size limit. For large messages that remain oversized after compression, combine with [Payload Offloading](#payload-offloading). +- Uses `MessageCodecEnum.ZSTD` (value `'zstd'`). You can use the string literal or the enum — both satisfy the `MessageCodec` type. + ### Message Handlers Handlers process messages based on their type. Messages are routed to the appropriate handler using the discriminator field (configurable via `messageTypeResolver`): diff --git a/packages/sqs/bench/codec.bench.ts b/packages/sqs/bench/codec.bench.ts index 12735c67..600a7ca9 100644 --- a/packages/sqs/bench/codec.bench.ts +++ b/packages/sqs/bench/codec.bench.ts @@ -92,115 +92,107 @@ describe('SQS codec benchmarks', () => { // ── Publish ──────────────────────────────────────────────────────────────── - it( - `publish: with vs without zstd (${N} messages)`, - async () => { - console.log(`\n${'─'.repeat(72)}`) - console.log(` PUBLISH BENCHMARK — ${N} messages per run`) - console.log('─'.repeat(72)) - - for (const { label, suffix, meta } of CASES) { - const plainQ = `bench-pub-plain-${suffix}` - const codecQ = `bench-pub-codec-${suffix}` - await testAdmin.deleteQueues(plainQ, codecQ) - - const plainMsgs = makeMessages('bpp', N, meta) - const codecMsgs = makeMessages('bcp', N, meta) - - // ── Plain publish ── - const plainPub = new SqsPermissionPublisher(diContainer.cradle, { - creationConfig: { queue: { QueueName: plainQ } }, - }) - await plainPub.init() - const t0 = performance.now() - for (const msg of plainMsgs) await plainPub.publish(msg) - const plainMs = performance.now() - t0 - await plainPub.close() - - // ── Codec publish ── - const codecPub = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', - creationConfig: { queue: { QueueName: codecQ } }, - }) - await codecPub.init() - const t1 = performance.now() - for (const msg of codecMsgs) await codecPub.publish(msg) - const codecMs = performance.now() - t1 - await codecPub.close() - - await testAdmin.deleteQueues(plainQ, codecQ) - printRow(label, N, plainMs, codecMs) - } - }, - 120_000, - ) + it(`publish: with vs without zstd (${N} messages)`, async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` PUBLISH BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-pub-plain-${suffix}` + const codecQ = `bench-pub-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpp', N, meta) + const codecMsgs = makeMessages('bcp', N, meta) + + // ── Plain publish ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + const t0 = performance.now() + for (const msg of plainMsgs) await plainPub.publish(msg) + const plainMs = performance.now() - t0 + await plainPub.close() + + // ── Codec publish ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + const t1 = performance.now() + for (const msg of codecMsgs) await codecPub.publish(msg) + const codecMs = performance.now() - t1 + await codecPub.close() + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, 120_000) // ── Consume ──────────────────────────────────────────────────────────────── - it( - `consume: with vs without zstd (${N} messages)`, - async () => { - console.log(`\n${'─'.repeat(72)}`) - console.log(` CONSUME BENCHMARK — ${N} messages per run`) - console.log('─'.repeat(72)) - - for (const { label, suffix, meta } of CASES) { - const plainQ = `bench-con-plain-${suffix}` - const codecQ = `bench-con-codec-${suffix}` - await testAdmin.deleteQueues(plainQ, codecQ) - - const plainMsgs = makeMessages('bpc', N, meta) - const codecMsgs = makeMessages('bcc', N, meta) - - // ── Pre-fill plain queue ── - const plainPub = new SqsPermissionPublisher(diContainer.cradle, { - creationConfig: { queue: { QueueName: plainQ } }, - }) - await plainPub.init() - for (const msg of plainMsgs) await plainPub.publish(msg) - await plainPub.close() - - // ── Pre-fill codec queue ── - const codecPub = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', - creationConfig: { queue: { QueueName: codecQ } }, - }) - await codecPub.init() - for (const msg of codecMsgs) await codecPub.publish(msg) - await codecPub.close() - - // ── Measure plain consume ── - // deletionConfig: { deleteIfExists: false } preserves the pre-filled queue - const plainCon = new SqsPermissionConsumer(diContainer.cradle, { - creationConfig: { queue: { QueueName: plainQ } }, - deletionConfig: { deleteIfExists: false }, - }) - await plainCon.start() - const t2 = performance.now() - await Promise.all( - plainMsgs.map((m) => plainCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), - ) - const plainMs = performance.now() - t2 - await plainCon.close(true) - - // ── Measure codec consume ── - const codecCon = new SqsPermissionConsumer(diContainer.cradle, { - codec: 'zstd', - creationConfig: { queue: { QueueName: codecQ } }, - deletionConfig: { deleteIfExists: false }, - }) - await codecCon.start() - const t3 = performance.now() - await Promise.all( - codecMsgs.map((m) => codecCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), - ) - const codecMs = performance.now() - t3 - await codecCon.close(true) - - await testAdmin.deleteQueues(plainQ, codecQ) - printRow(label, N, plainMs, codecMs) - } - }, - 120_000, - ) + it(`consume: with vs without zstd (${N} messages)`, async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` CONSUME BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-con-plain-${suffix}` + const codecQ = `bench-con-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpc', N, meta) + const codecMsgs = makeMessages('bcc', N, meta) + + // ── Pre-fill plain queue ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + for (const msg of plainMsgs) await plainPub.publish(msg) + await plainPub.close() + + // ── Pre-fill codec queue ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + for (const msg of codecMsgs) await codecPub.publish(msg) + await codecPub.close() + + // ── Measure plain consume ── + // deletionConfig: { deleteIfExists: false } preserves the pre-filled queue + const plainCon = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await plainCon.start() + const t2 = performance.now() + await Promise.all( + plainMsgs.map((m) => plainCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const plainMs = performance.now() - t2 + await plainCon.close(true) + + // ── Measure codec consume ── + const codecCon = new SqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await codecCon.start() + const t3 = performance.now() + await Promise.all( + codecMsgs.map((m) => codecCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const codecMs = performance.now() - t3 + await codecCon.close(true) + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, 120_000) }) diff --git a/packages/sqs/lib/codec/sqsCodecHandler.ts b/packages/sqs/lib/codec/sqsCodecHandler.ts index 4a095e21..b00c18ba 100644 --- a/packages/sqs/lib/codec/sqsCodecHandler.ts +++ b/packages/sqs/lib/codec/sqsCodecHandler.ts @@ -1,20 +1,25 @@ +import { promisify } from 'node:util' +import zlib from 'node:zlib' import type { CodecEnvelope, MessageCodec, MessageCodecHandler } from '@message-queue-toolkit/core' -import { compress, decompress } from '@mongodb-js/zstd' +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +const zstdCompress = promisify(zlib.zstdCompress) +const zstdDecompress = promisify(zlib.zstdDecompress) export class ZstdCodecHandler implements MessageCodecHandler { compress(data: Buffer): Promise { - return compress(data) + return zstdCompress(data) } decompress(data: Buffer): Promise { - return decompress(data) + return zstdDecompress(data) } } const ZSTD_HANDLER = new ZstdCodecHandler() export function resolveCodecHandler(codec: MessageCodec): MessageCodecHandler { - if (codec === 'zstd') return ZSTD_HANDLER + if (codec === MessageCodecEnum.ZSTD) return ZSTD_HANDLER throw new Error(`Unsupported codec: ${codec}`) } diff --git a/packages/sqs/package.json b/packages/sqs/package.json index acf138f3..de529971 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -30,7 +30,6 @@ }, "dependencies": { "@lokalise/node-core": "^14.6.1", - "@mongodb-js/zstd": "^7.0.0", "sqs-consumer": "^15.0.1" }, "peerDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 53770fcc..572f62f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -545,9 +545,6 @@ importers: '@lokalise/node-core': specifier: ^14.6.1 version: 14.8.1(zod@4.4.3) - '@mongodb-js/zstd': - specifier: ^7.0.0 - version: 7.0.0 sqs-consumer: specifier: ^15.0.1 version: 15.0.1(@aws-sdk/client-sqs@3.1048.0) @@ -999,10 +996,6 @@ packages: peerDependencies: zod: '>=3.25.76 <5.0.0' - '@mongodb-js/zstd@7.0.0': - resolution: {integrity: sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA==} - engines: {node: '>= 20.19.0'} - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} cpu: [arm64] @@ -1561,9 +1554,6 @@ packages: bintrees@1.0.2: resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} - bl@4.1.0: - resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} - bowser@2.14.1: resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} @@ -1581,9 +1571,6 @@ packages: buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - buffer@5.7.1: - resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} - bullmq@5.76.8: resolution: {integrity: sha512-v3WTwA8diFtsADaJ8eK2ozyi2CYK9rDZCeoKF+dIPF/MUL8HxAOa3H72Gmu1lC4yKlho6t1PwNr/QpDVqaNEZQ==} engines: {node: '>=12.22.0'} @@ -1596,9 +1583,6 @@ packages: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} - chownr@1.1.4: - resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -1656,14 +1640,6 @@ packages: supports-color: optional: true - decompress-response@6.0.0: - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} - engines: {node: '>=10'} - - deep-extend@0.6.0: - resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} - engines: {node: '>=4.0.0'} - delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -1740,10 +1716,6 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} - expand-template@2.0.3: - resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} - engines: {node: '>=6'} - expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -1838,9 +1810,6 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} - fs-constants@1.0.0: - resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} - fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1877,9 +1846,6 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} - github-from-package@0.0.0: - resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1966,15 +1932,9 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} - ieee754@1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - ini@1.3.8: - resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - ioredis@5.10.1: resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==} engines: {node: '>=12.22.0'} @@ -2195,10 +2155,6 @@ packages: engines: {node: '>=10.0.0'} hasBin: true - mimic-response@3.1.0: - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} - engines: {node: '>=10'} - minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -2214,9 +2170,6 @@ packages: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} - mkdirp-classic@0.5.3: - resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - mnemonist@0.40.4: resolution: {integrity: sha512-ZAv+KNavneRVzu4tUeOgzkScI3W5BGwZ3rkxIpKtzzVgfTtWQFN1CgX0U72cyvyh3iTuHL3SiSmrQxTlryEIcw==} @@ -2235,20 +2188,9 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - napi-build-utils@2.0.0: - resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} - - node-abi@3.92.0: - resolution: {integrity: sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==} - engines: {node: '>=10'} - node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} - node-addon-api@8.7.0: - resolution: {integrity: sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==} - engines: {node: ^18 || ^20 || >= 21} - node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2347,12 +2289,6 @@ packages: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} - prebuild-install@7.1.3: - resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} - engines: {node: '>=10'} - deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. - hasBin: true - process-warning@4.0.1: resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} @@ -2384,10 +2320,6 @@ packages: quick-format-unescaped@4.0.4: resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} - rc@1.2.8: - resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} - hasBin: true - readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} @@ -2504,12 +2436,6 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} - simple-concat@1.0.1: - resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - - simple-get@4.0.1: - resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} - sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} @@ -2561,10 +2487,6 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} - strip-json-comments@2.0.1: - resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} - engines: {node: '>=0.10.0'} - strip-json-comments@5.0.3: resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} engines: {node: '>=14.16'} @@ -2583,13 +2505,6 @@ packages: resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} engines: {node: '>=20'} - tar-fs@2.1.4: - resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} - - tar-stream@2.2.0: - resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} - engines: {node: '>=6'} - tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} @@ -2645,9 +2560,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - tunnel-agent@0.6.0: - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - turbo@2.9.14: resolution: {integrity: sha512-BQqXRr4UoWI3UPFrtznCLykYHxwxWh53iCB57x092jPMjIlW1wnm3N895g5irpiXmnxUhREBB0n6+y8BHhs4nw==} hasBin: true @@ -3451,11 +3363,6 @@ snapshots: dependencies: zod: 4.4.3 - '@mongodb-js/zstd@7.0.0': - dependencies: - node-addon-api: 8.7.0 - prebuild-install: 7.1.3 - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -3920,12 +3827,6 @@ snapshots: bintrees@1.0.2: {} - bl@4.1.0: - dependencies: - buffer: 5.7.1 - inherits: 2.0.4 - readable-stream: 3.6.2 - bowser@2.14.1: {} brace-expansion@2.1.0: @@ -3942,11 +3843,6 @@ snapshots: buffer-equal-constant-time@1.0.1: {} - buffer@5.7.1: - dependencies: - base64-js: 1.5.1 - ieee754: 1.2.1 - bullmq@5.76.8: dependencies: cron-parser: 4.9.0 @@ -3965,8 +3861,6 @@ snapshots: chai@6.2.2: {} - chownr@1.1.4: {} - cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -4011,12 +3905,6 @@ snapshots: dependencies: ms: 2.1.3 - decompress-response@6.0.0: - dependencies: - mimic-response: 3.1.0 - - deep-extend@0.6.0: {} - delayed-stream@1.0.0: {} denque@2.1.0: {} @@ -4085,8 +3973,6 @@ snapshots: event-target-shim@5.0.1: {} - expand-template@2.0.3: {} - expect-type@1.3.0: {} extend@3.0.2: {} @@ -4218,8 +4104,6 @@ snapshots: dependencies: fetch-blob: 3.2.0 - fs-constants@1.0.0: {} - fsevents@2.3.3: optional: true @@ -4281,8 +4165,6 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 - github-from-package@0.0.0: {} - glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -4404,12 +4286,8 @@ snapshots: transitivePeerDependencies: - supports-color - ieee754@1.2.1: {} - inherits@2.0.4: {} - ini@1.3.8: {} - ioredis@5.10.1: dependencies: '@ioredis/commands': 1.5.1 @@ -4594,8 +4472,6 @@ snapshots: mime@3.0.0: {} - mimic-response@3.1.0: {} - minimatch@10.2.5: dependencies: brace-expansion: 5.0.6 @@ -4608,8 +4484,6 @@ snapshots: minipass@7.1.3: {} - mkdirp-classic@0.5.3: {} - mnemonist@0.40.4: dependencies: obliterator: 2.0.5 @@ -4634,16 +4508,8 @@ snapshots: nanoid@3.3.12: {} - napi-build-utils@2.0.0: {} - - node-abi@3.92.0: - dependencies: - semver: 7.8.0 - node-abort-controller@3.1.1: {} - node-addon-api@8.7.0: {} - node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -4745,21 +4611,6 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - prebuild-install@7.1.3: - dependencies: - detect-libc: 2.1.2 - expand-template: 2.0.3 - github-from-package: 0.0.0 - minimist: 1.2.8 - mkdirp-classic: 0.5.3 - napi-build-utils: 2.0.0 - node-abi: 3.92.0 - pump: 3.0.4 - rc: 1.2.8 - simple-get: 4.0.1 - tar-fs: 2.1.4 - tunnel-agent: 0.6.0 - process-warning@4.0.1: {} process-warning@5.0.0: {} @@ -4802,13 +4653,6 @@ snapshots: quick-format-unescaped@4.0.4: {} - rc@1.2.8: - dependencies: - deep-extend: 0.6.0 - ini: 1.3.8 - minimist: 1.2.8 - strip-json-comments: 2.0.1 - readable-stream@3.6.2: dependencies: inherits: 2.0.4 @@ -4921,14 +4765,6 @@ snapshots: signal-exit@4.1.0: {} - simple-concat@1.0.1: {} - - simple-get@4.0.1: - dependencies: - decompress-response: 6.0.0 - once: 1.4.0 - simple-concat: 1.0.1 - sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -4980,8 +4816,6 @@ snapshots: dependencies: ansi-regex: 6.2.2 - strip-json-comments@2.0.1: {} - strip-json-comments@5.0.3: {} strnum@2.3.0: {} @@ -4994,21 +4828,6 @@ snapshots: tagged-tag@1.0.0: {} - tar-fs@2.1.4: - dependencies: - chownr: 1.1.4 - mkdirp-classic: 0.5.3 - pump: 3.0.4 - tar-stream: 2.2.0 - - tar-stream@2.2.0: - dependencies: - bl: 4.1.0 - end-of-stream: 1.4.5 - fs-constants: 1.0.0 - inherits: 2.0.4 - readable-stream: 3.6.2 - tdigest@0.1.2: dependencies: bintrees: 1.0.2 @@ -5066,10 +4885,6 @@ snapshots: tslib@2.8.1: {} - tunnel-agent@0.6.0: - dependencies: - safe-buffer: 5.2.1 - turbo@2.9.14: optionalDependencies: '@turbo/darwin-64': 2.9.14 From 12897ca6037e1edb4852835d3111322589288b39 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 10:49:59 +0200 Subject: [PATCH 6/9] feat: extract codec into standalone @message-queue-toolkit/codec package Move the zstd codec implementation (ZstdCodecHandler, compressMessageBody, decompressMessageBody, resolveCodecHandler) from packages/sqs into a new dedicated packages/codec package so any adapter can use compression without depending on @message-queue-toolkit/sqs. - Create packages/codec with package.json, tsconfigs, and lib/codec/codecHandler.ts - Delete packages/sqs/lib/codec/sqsCodecHandler.ts - Update sqs and sns to import from @message-queue-toolkit/codec - Re-export codec functions from @message-queue-toolkit/sqs for backwards compatibility - Add @message-queue-toolkit/codec as peer dependency in sqs and sns packages - Remove @mongodb-js/zstd from pnpm-workspace.yaml allowBuilds (no longer used) - Register packages/codec in CI PATH_TO_NAME map - Update SQS and SNS READMEs to document codec as a separate peer dependency Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 1 + packages/codec/README.md | 58 +++++++++++++++++++ .../lib/codec/codecHandler.ts} | 0 packages/codec/lib/index.ts | 6 ++ packages/codec/package.json | 55 ++++++++++++++++++ packages/codec/tsconfig.build.json | 4 ++ packages/codec/tsconfig.json | 4 ++ packages/sns/README.md | 1 + packages/sns/lib/sns/AbstractSnsPublisher.ts | 3 +- packages/sns/package.json | 2 + packages/sqs/README.md | 7 +++ packages/sqs/lib/index.ts | 2 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 2 +- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 2 +- packages/sqs/package.json | 2 + .../SqsPermissionConsumer.codec.spec.ts | 2 +- pnpm-lock.yaml | 30 ++++++++++ pnpm-workspace.yaml | 1 - 18 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 packages/codec/README.md rename packages/{sqs/lib/codec/sqsCodecHandler.ts => codec/lib/codec/codecHandler.ts} (100%) create mode 100644 packages/codec/lib/index.ts create mode 100644 packages/codec/package.json create mode 100644 packages/codec/tsconfig.build.json create mode 100644 packages/codec/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1796fc95..9c6cc4bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: run: | declare -A PATH_TO_NAME=( ["packages/amqp"]="@message-queue-toolkit/amqp" + ["packages/codec"]="@message-queue-toolkit/codec" ["packages/core"]="@message-queue-toolkit/core" ["packages/gcp-pubsub"]="@message-queue-toolkit/gcp-pubsub" ["packages/gcs-payload-store"]="@message-queue-toolkit/gcs-payload-store" diff --git a/packages/codec/README.md b/packages/codec/README.md new file mode 100644 index 00000000..0b34c725 --- /dev/null +++ b/packages/codec/README.md @@ -0,0 +1,58 @@ +# @message-queue-toolkit/codec + +Message compression codec implementations for [message-queue-toolkit](https://github.com/kibertoad/message-queue-toolkit). + +This package provides the concrete codec implementations (e.g. zstd) used by the SQS and SNS adapters. The codec interfaces and types (`MessageCodecEnum`, `MessageCodecHandler`, `CodecEnvelope`) live in `@message-queue-toolkit/core`. + +## Installation + +```sh +npm install @message-queue-toolkit/codec @message-queue-toolkit/core +``` + +> **Requirements:** Node.js 22+ (uses the built-in `zlib` zstd support). + +## Usage + +Codec options are typically set on the publisher/consumer constructor in the SQS or SNS adapter packages. You do not need to interact with this package directly unless you are building a custom adapter. + +### Compress / decompress a message body + +```typescript +import { compressMessageBody, decompressMessageBody } from '@message-queue-toolkit/codec' +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +// Compress (returns a JSON string containing the codec envelope) +const compressed = await compressMessageBody(JSON.stringify(payload), MessageCodecEnum.ZSTD) + +// Decompress (parses the envelope and returns the original object) +const original = await decompressMessageBody(JSON.parse(compressed)) +``` + +### Custom codec handler + +```typescript +import type { MessageCodecHandler } from '@message-queue-toolkit/core' + +class MyCodecHandler implements MessageCodecHandler { + compress(data: Buffer): Promise { /* ... */ } + decompress(data: Buffer): Promise { /* ... */ } +} +``` + +## Codec envelope format + +Compressed messages are wrapped in a self-describing JSON envelope: + +```json +{ + "__codec": "zstd", + "__data": "" +} +``` + +Consumers auto-detect this envelope and decompress transparently, even if the `codec` option is not set on the consumer. + +## License + +MIT diff --git a/packages/sqs/lib/codec/sqsCodecHandler.ts b/packages/codec/lib/codec/codecHandler.ts similarity index 100% rename from packages/sqs/lib/codec/sqsCodecHandler.ts rename to packages/codec/lib/codec/codecHandler.ts diff --git a/packages/codec/lib/index.ts b/packages/codec/lib/index.ts new file mode 100644 index 00000000..97ca1042 --- /dev/null +++ b/packages/codec/lib/index.ts @@ -0,0 +1,6 @@ +export { + compressMessageBody, + decompressMessageBody, + resolveCodecHandler, + ZstdCodecHandler, +} from './codec/codecHandler.ts' diff --git a/packages/codec/package.json b/packages/codec/package.json new file mode 100644 index 00000000..811a191d --- /dev/null +++ b/packages/codec/package.json @@ -0,0 +1,55 @@ +{ + "name": "@message-queue-toolkit/codec", + "version": "1.0.0", + "private": false, + "license": "MIT", + "description": "Message compression codec implementations for message-queue-toolkit", + "maintainers": [ + { + "name": "Igor Savin", + "email": "kibertoad@gmail.com" + } + ], + "type": "module", + "main": "./dist/index.js", + "exports": { + ".": "./dist/index.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "pnpm run clean && tsc --project tsconfig.build.json", + "clean": "rimraf dist", + "lint": "biome check . && tsc", + "lint:fix": "biome check --write .", + "prepublishOnly": "pnpm run lint && pnpm run build" + }, + "peerDependencies": { + "@message-queue-toolkit/core": ">=25.0.0" + }, + "devDependencies": { + "@biomejs/biome": "^2.3.8", + "@lokalise/biome-config": "^3.1.0", + "@lokalise/tsconfig": "^3.0.0", + "@message-queue-toolkit/core": "workspace:*", + "@types/node": "^25.0.2", + "rimraf": "^6.0.1", + "typescript": "^5.9.3" + }, + "homepage": "https://github.com/kibertoad/message-queue-toolkit", + "repository": { + "type": "git", + "url": "git://github.com/kibertoad/message-queue-toolkit.git" + }, + "keywords": [ + "message", + "queue", + "codec", + "compression", + "zstd" + ], + "files": [ + "README.md", + "LICENSE", + "dist/*" + ] +} diff --git a/packages/codec/tsconfig.build.json b/packages/codec/tsconfig.build.json new file mode 100644 index 00000000..198dcfd5 --- /dev/null +++ b/packages/codec/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": ["./tsconfig.json", "@lokalise/tsconfig/build-public-lib"], + "include": ["lib/**/*"] +} diff --git a/packages/codec/tsconfig.json b/packages/codec/tsconfig.json new file mode 100644 index 00000000..8dca583f --- /dev/null +++ b/packages/codec/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@lokalise/tsconfig/tsc", + "include": ["lib/**/*"] +} diff --git a/packages/sns/README.md b/packages/sns/README.md index 34112836..4e38a4e2 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -43,6 +43,7 @@ npm install @message-queue-toolkit/sns @message-queue-toolkit/sqs @message-queue - `@aws-sdk/client-sqs` - AWS SDK for SQS (required for consumers) - `@aws-sdk/client-sts` - AWS SDK for STS (for ARN resolution) - `zod` - Schema validation +- `@message-queue-toolkit/codec` - Required when using message compression ## Features diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 81c98269..34713ced 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -2,6 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sns' import { PublishCommand } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' +import { compressMessageBody } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -14,7 +15,7 @@ import { type QueuePublisherOptions, type ResolvedMessage, } from '@message-queue-toolkit/core' -import { compressMessageBody, resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' +import { resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' import { calculateOutgoingMessageSize, validateFifoTopicName } from '../utils/snsUtils.ts' diff --git a/packages/sns/package.json b/packages/sns/package.json index f47833f4..f6dfe5e1 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -35,6 +35,7 @@ "@aws-sdk/client-sns": "^3.632.0", "@aws-sdk/client-sqs": "^3.632.0", "@aws-sdk/client-sts": "^3.632.0", + "@message-queue-toolkit/codec": ">=1.0.0", "@message-queue-toolkit/core": ">=24.0.0", "@message-queue-toolkit/schemas": ">=7.0.0", "@message-queue-toolkit/sqs": ">=23.0.0", @@ -47,6 +48,7 @@ "@biomejs/biome": "^2.3.6", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", + "@message-queue-toolkit/codec": "workspace:*", "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", "@message-queue-toolkit/s3-payload-store": "workspace:*", diff --git a/packages/sqs/README.md b/packages/sqs/README.md index f37edd79..e06234eb 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -49,6 +49,7 @@ npm install @message-queue-toolkit/sqs @message-queue-toolkit/core **Peer Dependencies:** - `@aws-sdk/client-sqs` - AWS SDK for SQS - `zod` - Schema validation +- `@message-queue-toolkit/codec` - Required when using message compression ## Features @@ -805,6 +806,12 @@ await publisher.publish({ Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js 22+**. +The codec implementation lives in the separate [`@message-queue-toolkit/codec`](../codec/README.md) package, which must be installed alongside this package when using compression. + +```bash +npm install @message-queue-toolkit/codec +``` + Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __codec: 'zstd', __data: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. #### Publisher diff --git a/packages/sqs/lib/index.ts b/packages/sqs/lib/index.ts index 7705888f..40828db2 100644 --- a/packages/sqs/lib/index.ts +++ b/packages/sqs/lib/index.ts @@ -3,7 +3,7 @@ export { decompressMessageBody, resolveCodecHandler, ZstdCodecHandler, -} from './codec/sqsCodecHandler.ts' +} from '@message-queue-toolkit/codec' export { SqsConsumerErrorResolver } from './errors/SqsConsumerErrorResolver.ts' export { FakeConsumerErrorResolver } from './fakes/FakeConsumerErrorResolver.ts' export { TestSqsPublisher, type TestSqsPublishOptions } from './fakes/TestSqsPublisher.ts' diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index e45e94f9..9bd8cf7f 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -5,6 +5,7 @@ import { SetQueueAttributesCommand, } from '@aws-sdk/client-sqs' import type { Either, ErrorResolver } from '@lokalise/node-core' +import { decompressMessageBody } from '@message-queue-toolkit/codec' import type { ProcessedMessageMetadata } from '@message-queue-toolkit/core' import { type BarrierResult, @@ -27,7 +28,6 @@ import { import type { ConsumerOptions } from 'sqs-consumer' import { Consumer } from 'sqs-consumer' import type { ZodSchema } from 'zod/v4' -import { decompressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { hasOffloadedPayload } from '../utils/messageUtils.ts' import { deleteSqs, initSqs } from '../utils/sqsInitter.ts' diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index f6cc3f7f..9c02ef52 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -2,6 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sqs' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' +import { compressMessageBody } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -15,7 +16,6 @@ import { type ResolvedMessage, } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod/v4' -import { compressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { resolveOutgoingMessageAttributes } from '../utils/messageUtils.ts' diff --git a/packages/sqs/package.json b/packages/sqs/package.json index de529971..5eb68739 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -34,6 +34,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.632.0", + "@message-queue-toolkit/codec": ">=1.0.0", "@message-queue-toolkit/core": ">=25.0.0", "zod": ">=3.25.76 <5.0.0" }, @@ -43,6 +44,7 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", + "@message-queue-toolkit/codec": "workspace:*", "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", "@message-queue-toolkit/s3-payload-store": "workspace:*", diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index b7229361..de4cf018 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,8 +1,8 @@ import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { compressMessageBody } from '@message-queue-toolkit/codec' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' -import { compressMessageBody } from '../../lib/codec/sqsCodecHandler.ts' import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 572f62f2..895502f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -64,6 +64,30 @@ importers: specifier: ^4.1.13 version: 4.4.3 + packages/codec: + devDependencies: + '@biomejs/biome': + specifier: ^2.3.8 + version: 2.4.15 + '@lokalise/biome-config': + specifier: ^3.1.0 + version: 3.1.1 + '@lokalise/tsconfig': + specifier: ^3.0.0 + version: 3.1.0 + '@message-queue-toolkit/core': + specifier: workspace:* + version: link:../core + '@types/node': + specifier: ^25.0.2 + version: 25.8.0 + rimraf: + specifier: ^6.0.1 + version: 6.1.3 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + packages/core: dependencies: '@lokalise/node-core': @@ -497,6 +521,9 @@ importers: '@lokalise/tsconfig': specifier: ^3.0.0 version: 3.1.0 + '@message-queue-toolkit/codec': + specifier: workspace:* + version: link:../codec '@message-queue-toolkit/core': specifier: workspace:* version: link:../core @@ -564,6 +591,9 @@ importers: '@lokalise/tsconfig': specifier: ^3.0.0 version: 3.1.0 + '@message-queue-toolkit/codec': + specifier: workspace:* + version: link:../codec '@message-queue-toolkit/core': specifier: workspace:* version: link:../core diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 35668904..5918a4e3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,7 +6,6 @@ packages: # pnpm 11 auto-appends new entries here whenever a dep has scripts, prompting # an explicit decision per package. allowBuilds: - '@mongodb-js/zstd': true msgpackr-extract: false protobufjs: false From 094dc5df0de39aac3fe07595bda13100bfd92c72 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 10:57:27 +0200 Subject: [PATCH 7/9] fix(codec): tighten Node.js version requirement to >=22.15.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit zlib.zstdCompress/zstdDecompress were added in Node.js v22.15.0 and v23.8.0, not v22.0.0. The previous "Node.js 22+" claim was incorrect and would cause a cryptic TypeError at import time on v22.0.0-v22.14.x. - Add runtime guard in codecHandler.ts that throws a clear error if zstd functions are missing, before promisify() is called - Add engines: { node: ">=22.15.0" } to packages/codec/package.json - Update all JSDoc and README references from "Node.js 22+" to ">=22.15.0" CI matrix (22.x, 24.x) resolves to latest patches which are >=22.15.0 — no change needed. Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 2 +- packages/codec/lib/codec/codecHandler.ts | 7 +++++++ packages/codec/package.json | 3 +++ packages/core/lib/codec/messageCodec.ts | 2 +- packages/core/lib/types/queueOptionsTypes.ts | 2 +- packages/sns/README.md | 4 ++-- packages/sqs/README.md | 8 ++++---- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/codec/README.md b/packages/codec/README.md index 0b34c725..6460bc51 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -10,7 +10,7 @@ This package provides the concrete codec implementations (e.g. zstd) used by the npm install @message-queue-toolkit/codec @message-queue-toolkit/core ``` -> **Requirements:** Node.js 22+ (uses the built-in `zlib` zstd support). +> **Requirements:** Node.js >=22.15.0 (uses the built-in `zlib` zstd support). ## Usage diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index b00c18ba..bcdffb3e 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -3,6 +3,13 @@ import zlib from 'node:zlib' import type { CodecEnvelope, MessageCodec, MessageCodecHandler } from '@message-queue-toolkit/core' import { MessageCodecEnum } from '@message-queue-toolkit/core' +if (typeof zlib.zstdCompress !== 'function' || typeof zlib.zstdDecompress !== 'function') { + throw new Error( + 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + + '@message-queue-toolkit/codec requires Node.js >=22.15.0 or >=23.8.0.', + ) +} + const zstdCompress = promisify(zlib.zstdCompress) const zstdDecompress = promisify(zlib.zstdDecompress) diff --git a/packages/codec/package.json b/packages/codec/package.json index 811a191d..aca4b8cf 100644 --- a/packages/codec/package.json +++ b/packages/codec/package.json @@ -23,6 +23,9 @@ "lint:fix": "biome check --write .", "prepublishOnly": "pnpm run lint && pnpm run build" }, + "engines": { + "node": ">=22.15.0" + }, "peerDependencies": { "@message-queue-toolkit/core": ">=25.0.0" }, diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index cc7fab0f..6ecc46dd 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -11,7 +11,7 @@ type ObjectValues = T[keyof T] * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) */ export const MessageCodecEnum = { - /** zstd compression via Node.js built-in `zlib` (requires Node.js 22+). */ + /** zstd compression via Node.js built-in `zlib` (requires Node.js >=22.15.0). */ ZSTD: 'zstd', } as const export type MessageCodec = ObjectValues diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index c8ae6d7c..204f37a8 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -149,7 +149,7 @@ export type CommonQueueOptions = { * Even without this option, consumers auto-detect and decompress any message * that carries a codec envelope, so mixed queues work transparently. * - * Uses Node.js built-in `zlib` zstd support — **requires Node.js 22+**. + * Uses Node.js built-in `zlib` zstd support — **requires Node.js >=22.15.0** (or >=23.8.0). * * @example * import { MessageCodecEnum } from '@message-queue-toolkit/core' diff --git a/packages/sns/README.md b/packages/sns/README.md index 4e38a4e2..1b58d404 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -59,7 +59,7 @@ npm install @message-queue-toolkit/sns @message-queue-toolkit/sqs @message-queue - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Cross-account and cross-region publishing** -- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js >=22.15.0 required) ## Core Concepts @@ -685,7 +685,7 @@ await consumer.start() // Optional - Payload Offloading (same as SQS) payloadStoreConfig: { /* ... */ }, - // Optional - Compression (Node.js 22+ required) + // Optional - Compression (Node.js >=22.15.0 required) codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd // Optional - Deletion diff --git a/packages/sqs/README.md b/packages/sqs/README.md index e06234eb..1bafa729 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -64,7 +64,7 @@ npm install @message-queue-toolkit/sqs @message-queue-toolkit/core - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Automatic queue creation** with validation -- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js >=22.15.0 required) ## Core Concepts @@ -463,7 +463,7 @@ When using `locatorConfig`, you connect to an existing queue without creating it maxPayloadSize: 1024 * 1024, // 1 MiB }, - // Optional - Compression (Node.js 22+ required) + // Optional - Compression (Node.js >=22.15.0 required) codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd // Optional - Deletion @@ -537,7 +537,7 @@ When using `locatorConfig`, you connect to an existing queue without creating it payloadStore: s3Store, }, - // Optional - Compression (Node.js 22+ required) + // Optional - Compression (Node.js >=22.15.0 required) // Auto-detection is always active: consumers decompress codec envelopes // even without this option set. codec: MessageCodecEnum.ZSTD, @@ -804,7 +804,7 @@ await publisher.publish({ ### Message Compression -Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js 22+**. +Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js >=22.15.0**. The codec implementation lives in the separate [`@message-queue-toolkit/codec`](../codec/README.md) package, which must be installed alongside this package when using compression. From ff779b5076cd8298b8d195600f69e1baeec1aa50 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 14:49:50 +0200 Subject: [PATCH 8/9] refactor: compress-once in publish, split offload into focused helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes double compression: previously publish() delegated to offloadMessagePayloadIfNeeded which compressed the payload to check the size threshold, then returned the original message, and sendMessage() compressed again. Now when a codec is set, the message is compressed exactly once at publish() entry point, regardless of whether offloading is also configured. The same compressed Buffer is then either: - stored in S3 and replaced with a pointer (if compressed size exceeds messageSizeThreshold), or - wrapped in a codec envelope and sent inline (if it fits). The payload is never compressed twice. Key changes: - codec: add buildCodecEnvelope(compressed, codec) to wrap pre-compressed bytes without re-compressing - core: replace offloadMessagePayloadIfNeeded with three focused methods: - private buildPointer() — shared pointer construction logic - protected offloadPayload() — no-codec path, returns null if fits - protected offloadCompressedPayload() — codec path, always stores - sqs/sns: restructure publish() via private prepareOutgoingPayload() that compresses once and branches; sendMessage() accepts preBuiltBody to skip re-serialization - gcp-pubsub: migrate to offloadPayload(), pin core to workspace:* - docs: update SQS, SNS, core, and codec READMEs to explain the single compression pass and how codec interacts with payload offloading Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 26 ++++ packages/codec/lib/codec/codecHandler.ts | 8 + packages/codec/lib/index.ts | 1 + packages/core/README.md | 11 ++ .../offloadedPayloadMessageSchemas.ts | 6 + .../core/lib/queues/AbstractQueueService.ts | 146 ++++++++++++------ packages/core/lib/utils/streamUtils.ts | 9 +- .../AbstractQueueService.offload.spec.ts | 6 +- .../lib/pubsub/AbstractPubSubPublisher.ts | 11 +- packages/gcp-pubsub/package.json | 2 +- packages/sns/lib/sns/AbstractSnsPublisher.ts | 53 +++++-- packages/sqs/README.md | 23 ++- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 6 +- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 54 +++++-- .../SqsPermissionConsumer.codec.spec.ts | 9 +- ...rmissionConsumer.payloadOffloading.spec.ts | 109 ++++++++++++- pnpm-lock.yaml | 16 +- 17 files changed, 396 insertions(+), 100 deletions(-) diff --git a/packages/codec/README.md b/packages/codec/README.md index 6460bc51..22b082dc 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -16,6 +16,16 @@ npm install @message-queue-toolkit/codec @message-queue-toolkit/core Codec options are typically set on the publisher/consumer constructor in the SQS or SNS adapter packages. You do not need to interact with this package directly unless you are building a custom adapter. +### How compression works during publish + +When `codec` is set on a publisher, compression happens **exactly once** at the start of `publish()`, before any other processing: + +1. The message JSON is compressed to a raw `Buffer`. +2. If a payload store is configured **and** the compressed size exceeds `messageSizeThreshold`, the compressed bytes are stored in S3 and only a lightweight pointer is sent. The codec name is recorded in `payloadRef.codec` so the consumer can decompress after retrieval. +3. If the compressed size fits within the threshold (or no store is configured), the message is sent inline as a self-describing codec envelope. + +The payload is never compressed twice. The same compressed `Buffer` from step 1 is either uploaded to S3 or wrapped in the envelope — whichever path is taken. + ### Compress / decompress a message body ```typescript @@ -29,6 +39,22 @@ const compressed = await compressMessageBody(JSON.stringify(payload), MessageCod const original = await decompressMessageBody(JSON.parse(compressed)) ``` +### Build a codec envelope from already-compressed bytes + +When you have pre-compressed bytes (e.g., from `resolveCodecHandler(codec).compress(...)`) and want to produce the envelope string without compressing again: + +```typescript +import { buildCodecEnvelope } from '@message-queue-toolkit/codec' +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +const handler = resolveCodecHandler(MessageCodecEnum.ZSTD) +const compressed: Buffer = await handler.compress(Buffer.from(JSON.stringify(payload), 'utf8')) + +// Build envelope without a second compression pass +const envelopeString = buildCodecEnvelope(compressed, MessageCodecEnum.ZSTD) +// → '{"__codec":"zstd","__data":""}' +``` + ### Custom codec handler ```typescript diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index bcdffb3e..e39eebc0 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -33,6 +33,14 @@ export function resolveCodecHandler(codec: MessageCodec): MessageCodecHandler { export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { const handler = resolveCodecHandler(codec) const compressed = await handler.compress(Buffer.from(jsonBody, 'utf8')) + return buildCodecEnvelope(compressed, codec) +} + +/** + * Wraps an already-compressed buffer in a codec envelope string. + * Use this when you have pre-compressed bytes and want to avoid compressing twice. + */ +export function buildCodecEnvelope(compressed: Buffer, codec: MessageCodec): string { const envelope: CodecEnvelope = { __codec: codec, __data: compressed.toString('base64'), diff --git a/packages/codec/lib/index.ts b/packages/codec/lib/index.ts index 97ca1042..ed36e669 100644 --- a/packages/codec/lib/index.ts +++ b/packages/codec/lib/index.ts @@ -1,4 +1,5 @@ export { + buildCodecEnvelope, compressMessageBody, decompressMessageBody, resolveCodecHandler, diff --git a/packages/core/README.md b/packages/core/README.md index 125a1285..ce1139be 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -641,6 +641,17 @@ class MyPayloadStore implements PayloadStore { } ``` +#### Interaction with codec (compression) + +When both `codec` and `payloadStoreConfig` are set on a publisher, compression and offloading work together with a single compression pass: + +1. The message is compressed **once** at publish time. +2. The **compressed** size is compared against `messageSizeThreshold`. +3. If the compressed size exceeds the threshold, the raw compressed bytes are stored in the payload store. The codec name is written to `payloadRef.codec` so the consumer knows how to decompress after retrieval. +4. If the compressed size fits within the threshold, the message is sent inline as a self-describing codec envelope — S3 is never touched. + +This means compression can prevent offloading entirely for messages that are large before compression but small after. + ## API Reference ### Types diff --git a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts index fa62f183..29bc770f 100644 --- a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts +++ b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts @@ -11,6 +11,12 @@ export const PAYLOAD_REF_SCHEMA = z.object({ store: z.string().min(1), /** Size of the payload in bytes */ size: z.number().int().positive(), + /** + * Codec used to compress the stored payload. + * When set, the stored bytes are raw compressed binary (not base64 JSON). + * The consumer must decompress using this codec before parsing. + */ + codec: z.string().optional(), }) export type PayloadRef = z.output diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index ca305d0c..c42c926a 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -1,3 +1,4 @@ +import { Readable } from 'node:stream' import { types } from 'node:util' import { type CommonLogger, @@ -49,7 +50,7 @@ import type { QueueOptions, } from '../types/queueOptionsTypes.ts' import { isRetryDateExceeded } from '../utils/dateUtils.ts' -import { streamWithKnownSizeToString } from '../utils/streamUtils.ts' +import { streamWithKnownSizeToBuffer, streamWithKnownSizeToString } from '../utils/streamUtils.ts' import { toDatePreprocessor } from '../utils/toDateProcessor.ts' import type { BarrierCallback, @@ -660,49 +661,30 @@ export abstract class AbstractQueueService< } /** - * Offload message payload to an external store if it exceeds the threshold. - * Returns a special type that contains a pointer to the offloaded payload or the original payload if it was not offloaded. - * Requires message size as only the implementation knows how to calculate it. + * Builds an OffloadedPayloadPointerPayload from the given message and storage metadata. + * Copies identity fields and preserves the message type field through offloading. * - * For multi-store configuration, uses the configured outgoingStore. - * For single-store configuration, uses the single store. - * - * The returned payload includes both the new payloadRef format and legacy fields for backward compatibility. + * We default to the conventional top-level `type` path so that routing/identity fields are + * handled consistently with `messageIdField`/`messageTimestampField`/etc. Without this + * fallback, `messageTypeResolver` modes that don't specify a body path silently strip `type` + * from the offloaded body, breaking downstream SNS subscription FilterPolicy filters. */ - protected async offloadMessagePayloadIfNeeded( + private buildPointer( message: MessagePayloadSchemas, - messageSizeFn: () => number, - ): Promise { - if ( - !this.payloadStoreConfig || - messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold - ) { - return message - } - - const { store, storeName } = this.resolveOutgoingStore() - const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message) - - let payloadId: string - try { - payloadId = await store.storePayload(serializedPayload) - } finally { - if (isDestroyable(serializedPayload)) { - await serializedPayload.destroy() - } - } - - // Return message with both new and legacy formats for backward compatibility + payloadId: string, + storeName: string, + size: number, + codec?: MessageCodec, + ): OffloadedPayloadPointerPayload { const result: OffloadedPayloadPointerPayload = { - // Extended payload reference format payloadRef: { id: payloadId, store: storeName, - size: serializedPayload.size, + size, + ...(codec ? { codec } : {}), }, - // Legacy format for backward compatibility offloadedPayloadPointer: payloadId, - offloadedPayloadSize: serializedPayload.size, + offloadedPayloadSize: size, // @ts-expect-error [this.messageIdField]: message[this.messageIdField], // @ts-expect-error @@ -713,14 +695,6 @@ export abstract class AbstractQueueService< [this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField], } - // Preserve the message type field through offloading. We default to the conventional - // top-level `type` path so that routing/identity fields are handled consistently with - // `messageIdField`/`messageTimestampField`/etc., which have defaulted names ('id', - // 'timestamp', ...) and are always copied across when present. Without this fallback, - // `messageTypeResolver` modes that don't specify a body path (no resolver, `literal`, - // or `resolver`) silently strip `type` from the offloaded SNS body, which then breaks - // any downstream subscription whose FilterPolicy filters on `type` - // (FilterPolicyScope: 'MessageBody'). const typePath = this.messageTypeResolver && isMessageTypePathConfig(this.messageTypeResolver) ? this.messageTypeResolver.messageTypePath @@ -733,14 +707,82 @@ export abstract class AbstractQueueService< return result } + /** + * Offloads the message payload to the configured store if it exceeds the size threshold. + * Returns null if no offloading is needed (store not configured or message fits within threshold). + * + * For multi-store configuration, uses the configured outgoingStore. + * For single-store configuration, uses the single store. + * + * The returned pointer includes both the new payloadRef format and legacy fields for backward + * compatibility. The message type field is always preserved through offloading. + */ + protected async offloadPayload( + message: MessagePayloadSchemas, + messageSizeFn: () => number, + ): Promise { + if (!this.payloadStoreConfig) { + return null + } + + if (messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold) { + return null + } + + const { store, storeName } = this.resolveOutgoingStore() + const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message) + + let payloadId: string + try { + payloadId = await store.storePayload(serializedPayload) + } finally { + if (isDestroyable(serializedPayload)) { + await serializedPayload.destroy() + } + } + + return this.buildPointer(message, payloadId, storeName, serializedPayload.size) + } + + /** + * Stores an already-compressed payload in the configured store. + * The `codec` name is recorded in payloadRef so the consumer can decompress after retrieval. + * + * The threshold check is NOT performed here — callers must decide whether to offload. + * Use this when compression has already been done and the compressed size exceeds the threshold. + * + * @throws Error if payload store is not configured + */ + protected async offloadCompressedPayload( + message: MessagePayloadSchemas, + compressed: Buffer, + codec: MessageCodec, + ): Promise { + if (!this.payloadStoreConfig) { + throw new Error('Payload store is not configured') + } + + const { store, storeName } = this.resolveOutgoingStore() + const payloadId = await store.storePayload({ + value: Readable.from(compressed), + size: compressed.byteLength, + }) + + return this.buildPointer(message, payloadId, storeName, compressed.byteLength, codec) + } + /** * Retrieve previously offloaded message payload using provided pointer payload. * Returns the original payload or an error if the payload was not found or could not be parsed. * * Supports both new multi-store format (payloadRef) and legacy format (offloadedPayloadPointer). + * + * When `decompress` is provided and the pointer's `payloadRef.codec` matches, the fetched bytes + * are treated as raw compressed binary and decompressed before JSON parsing. */ protected async retrieveOffloadedMessagePayload( maybeOffloadedPayloadPointerPayload: unknown, + decompress?: (codec: string, data: Buffer) => Promise, ): Promise> { if (!this.payloadStoreConfig) { return { @@ -790,6 +832,24 @@ export abstract class AbstractQueueService< } } + const codec = parsedPayload.payloadRef?.codec + if (codec && decompress) { + try { + const compressedBuffer = await streamWithKnownSizeToBuffer( + serializedOffloadedPayloadReadable, + payloadSize, + ) + const decompressed = await decompress(codec, compressedBuffer) + return { result: JSON.parse(decompressed.toString('utf8')) } + } catch (e) { + return { + error: new Error(`Failed to decompress offloaded payload with codec "${codec}"`, { + cause: e, + }), + } + } + } + const serializedOffloadedPayloadString = await streamWithKnownSizeToString( serializedOffloadedPayloadReadable, payloadSize, diff --git a/packages/core/lib/utils/streamUtils.ts b/packages/core/lib/utils/streamUtils.ts index 7ea2155f..7e69617b 100644 --- a/packages/core/lib/utils/streamUtils.ts +++ b/packages/core/lib/utils/streamUtils.ts @@ -1,6 +1,6 @@ import type { Readable } from 'node:stream' -export async function streamWithKnownSizeToString(stream: Readable, size: number): Promise { +export async function streamWithKnownSizeToBuffer(stream: Readable, size: number): Promise { const buffer = Buffer.alloc(size) let offset = 0 @@ -14,5 +14,10 @@ export async function streamWithKnownSizeToString(stream: Readable, size: number offset += chunkBuffer.length } - return buffer.toString('utf8', 0, offset) + return buffer.subarray(0, offset) +} + +export async function streamWithKnownSizeToString(stream: Readable, size: number): Promise { + const buffer = await streamWithKnownSizeToBuffer(stream, size) + return buffer.toString('utf8') } diff --git a/packages/core/test/queues/AbstractQueueService.offload.spec.ts b/packages/core/test/queues/AbstractQueueService.offload.spec.ts index 2d6142ac..32167f30 100644 --- a/packages/core/test/queues/AbstractQueueService.offload.spec.ts +++ b/packages/core/test/queues/AbstractQueueService.offload.spec.ts @@ -1,5 +1,5 @@ /** - * Regression tests for `AbstractQueueService.offloadMessagePayloadIfNeeded`. + * Regression tests for `AbstractQueueService.offloadPayload`. * * Identity fields (`messageIdField`, `messageTimestampField`, `messageDeduplicationIdField`, * `messageDeduplicationOptionsField`) all have defaulted names ('id', 'timestamp', ...) and @@ -58,7 +58,7 @@ class TestQueueService extends AbstractQueueService< // Expose protected method for direct testing. public callOffload(message: TestMessage, sizeFn: () => number) { - return this.offloadMessagePayloadIfNeeded(message, sizeFn) + return this.offloadPayload(message, sizeFn) } } @@ -102,7 +102,7 @@ const baseMessage: TestMessage = { payload: { large: 'data' }, } -describe('AbstractQueueService.offloadMessagePayloadIfNeeded — `type` preservation', () => { +describe('AbstractQueueService.offloadPayload — `type` preservation', () => { it('preserves `type` when no messageTypeResolver is configured', async () => { const svc = buildService(undefined) const result = (await svc.callOffload( diff --git a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts index 5927afa0..067fdc58 100644 --- a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts +++ b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts @@ -69,11 +69,12 @@ export abstract class AbstractPubSubPublisher const parsedMessage = messageSchemaResult.result.parse(message) message = this.updateInternalProperties(message) - const maybeOffloadedPayloadMessage = await this.offloadMessagePayloadIfNeeded(message, () => { - // Calculate message size for PubSub - const messageData = Buffer.from(JSON.stringify(message)) - return messageData.length - }) + const maybeOffloadedPayloadMessage = + (await this.offloadPayload(message, () => { + // Calculate message size for PubSub + const messageData = Buffer.from(JSON.stringify(message)) + return messageData.length + })) ?? message if ( this.isDeduplicationEnabledForMessage(parsedMessage) && diff --git a/packages/gcp-pubsub/package.json b/packages/gcp-pubsub/package.json index e731daef..b43d284d 100644 --- a/packages/gcp-pubsub/package.json +++ b/packages/gcp-pubsub/package.json @@ -40,7 +40,7 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "*", + "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/gcs-payload-store": "*", "@message-queue-toolkit/redis-message-deduplication-store": "*", "@message-queue-toolkit/schemas": "*", diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 34713ced..090f27b1 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -2,12 +2,11 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sns' import { PublishCommand } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { compressMessageBody } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, DeduplicationRequesterEnum, - isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -133,10 +132,7 @@ export abstract class AbstractSnsPublisher // (offloaded payload won't have user fields needed for messageGroupIdField) const resolvedOptions = this.resolveFifoOptions(updatedMessage, options) - const maybeOffloadedPayloadMessage = await this.offloadMessagePayloadIfNeeded( - updatedMessage, - () => calculateOutgoingMessageSize(updatedMessage), - ) + const { payload, preBuiltBody } = await this.prepareOutgoingPayload(updatedMessage) if ( this.isDeduplicationEnabledForMessage(parsedMessage) && @@ -152,7 +148,7 @@ export abstract class AbstractSnsPublisher return } - await this.sendMessage(maybeOffloadedPayloadMessage, resolvedOptions) + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ message: parsedMessage, @@ -208,16 +204,49 @@ export abstract class AbstractSnsPublisher return this.isDeduplicationEnabled && super.isDeduplicationEnabledForMessage(message) } + /** + * Compresses (when codec is set) or offloads (when store is configured) the message. + * Returns the payload to send and an optional pre-built body string. + * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. + */ + private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ + payload: MessagePayloadType | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { + const codec = this.codec + + if (codec) { + // Compress once up-front, then decide: offload the compressed bytes or send inline. + const compressed = await resolveCodecHandler(codec).compress( + Buffer.from(JSON.stringify(message), 'utf8'), + ) + + if ( + this.payloadStoreConfig && + compressed.byteLength > this.payloadStoreConfig.messageSizeThreshold + ) { + return { payload: await this.offloadCompressedPayload(message, compressed, codec) } + } + + return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } + } + + return { + payload: + (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? + message, + } + } + protected async sendMessage( payload: MessagePayloadType | OffloadedPayloadPointerPayload, options: SNSMessageOptions, + preBuiltBody?: string, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) - const jsonBody = JSON.stringify(payload) - const body = - this.codec && !isOffloadedPayloadPointerPayload(payload) - ? await compressMessageBody(jsonBody, this.codec) - : jsonBody + // preBuiltBody is set when codec is active and the payload was not offloaded — + // it contains the already-compressed codec envelope, so we skip re-serialization. + const body = preBuiltBody ?? JSON.stringify(payload) const command = new PublishCommand({ Message: body, MessageAttributes: attributes, diff --git a/packages/sqs/README.md b/packages/sqs/README.md index bac33c86..9eebaa7b 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -799,13 +799,20 @@ await publisher.publish({ }) ``` -**How it works:** +**How it works (without codec):** 1. Publisher checks message size before sending -2. If size exceeds `maxPayloadSize`, stores payload in S3 -3. Replaces payload with pointer: `{ _offloadedPayload: { bucketName, key, size } }` -4. Sends pointer message to SQS -5. Consumer detects pointer, fetches payload from S3 -6. Processes message with full payload +2. If size exceeds `messageSizeThreshold`, serializes and stores payload in S3 +3. Sends a lightweight pointer message to SQS instead +4. Consumer detects the pointer, fetches payload from S3 +5. Processes message with full payload + +**How it works (with codec — compress + offload):** +1. Publisher compresses the serialized message with zstd **once**, up-front +2. If the **compressed** size exceeds `messageSizeThreshold`, stores the compressed bytes in S3 and sends a pointer +3. If the compressed size fits within the threshold, sends the message inline as a codec envelope +4. Consumer fetches the pointer payload as raw bytes, decompresses, then processes as normal + +The codec embedded in `payloadRef.codec` tells the consumer which algorithm to use — no `codec` option is needed on the consumer. **Note:** Payload cleanup is the responsibility of the store (e.g., S3 lifecycle policies). @@ -860,7 +867,9 @@ class MyConsumer extends AbstractSqsConsumer { + const handler = resolveCodecHandler(codec as Parameters[0]) + return handler.decompress(data) + }, ) if (retrieveOffloadedMessagePayloadResult.error) { this.handleError(retrieveOffloadedMessagePayloadResult.error) diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index 9c02ef52..29d8b9c8 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -2,12 +2,11 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sqs' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { compressMessageBody } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, DeduplicationRequesterEnum, - isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -125,9 +124,7 @@ export abstract class AbstractSqsPublisher // (offloaded payload won't have user fields needed for messageGroupIdField) const resolvedOptions = this.resolveFifoOptions(message, options) - const maybeOffloadedPayloadMessage = await this.offloadMessagePayloadIfNeeded(message, () => - calculateOutgoingMessageSize(message), - ) + const { payload, preBuiltBody } = await this.prepareOutgoingPayload(message) if ( this.isDeduplicationEnabledForMessage(parsedMessage) && @@ -143,7 +140,7 @@ export abstract class AbstractSqsPublisher return } - await this.sendMessage(maybeOffloadedPayloadMessage, resolvedOptions) + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ message: parsedMessage, processingResult: { status: 'published' }, @@ -201,18 +198,49 @@ export abstract class AbstractSqsPublisher return this.messageSchemaContainer.resolveSchema(message) } + /** + * Compresses (when codec is set) or offloads (when store is configured) the message. + * Returns the payload to send and an optional pre-built body string. + * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. + */ + private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ + payload: MessagePayloadType | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { + const codec = this.codec + + if (codec) { + // Compress once up-front, then decide: offload the compressed bytes or send inline. + const compressed = await resolveCodecHandler(codec).compress( + Buffer.from(JSON.stringify(message), 'utf8'), + ) + + if ( + this.payloadStoreConfig && + compressed.byteLength > this.payloadStoreConfig.messageSizeThreshold + ) { + return { payload: await this.offloadCompressedPayload(message, compressed, codec) } + } + + return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } + } + + return { + payload: + (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? + message, + } + } + protected async sendMessage( payload: MessagePayloadType | OffloadedPayloadPointerPayload, options: SQSMessageOptions, + preBuiltBody?: string, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) - const jsonBody = JSON.stringify(payload) - const body = - this.codec && !isOffloadedPayloadPointerPayload(payload) - ? await compressMessageBody(jsonBody, this.codec) - : jsonBody - - // Options are already resolved in publish() before offloading + // preBuiltBody is set when codec is active and the payload was not offloaded — + // it contains the already-compressed codec envelope, so we skip re-serialization. + const body = preBuiltBody ?? JSON.stringify(payload) const command = new SendMessageCommand({ QueueUrl: this.queueUrl, MessageBody: body, diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index de4cf018..9ac1b3a8 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,5 +1,6 @@ import { SendMessageCommand } from '@aws-sdk/client-sqs' import { compressMessageBody } from '@message-queue-toolkit/codec' +import { MessageCodecEnum } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' @@ -29,11 +30,11 @@ describe('SqsPermissionConsumer - zstd codec', () => { await testAdmin.deleteQueues(SqsPermissionConsumer.QUEUE_NAME) consumer = new SqsPermissionConsumer(diContainer.cradle, { - codec: 'zstd', + codec: MessageCodecEnum.ZSTD, deletionConfig: { deleteIfExists: false }, }) publisher = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', + codec: MessageCodecEnum.ZSTD, }) await consumer.start() @@ -89,7 +90,7 @@ describe('SqsPermissionConsumer - zstd codec', () => { } // Simulate a publisher that compressed the message itself - const compressedBody = await compressMessageBody(JSON.stringify(message), 'zstd') + const compressedBody = await compressMessageBody(JSON.stringify(message), MessageCodecEnum.ZSTD) await diContainer.cradle.sqsClient.send( new SendMessageCommand({ QueueUrl: consumer.queueProps.url, @@ -108,7 +109,7 @@ describe('SqsPermissionConsumer - zstd codec', () => { await testAdmin.deleteQueues(autoQueueName) const autoPublisher = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', + codec: MessageCodecEnum.ZSTD, creationConfig: { queue: { QueueName: autoQueueName } }, }) await autoPublisher.init() diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts index c53ff481..370901a0 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts @@ -1,7 +1,7 @@ import type { S3 } from '@aws-sdk/client-s3' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { SinglePayloadStoreConfig } from '@message-queue-toolkit/core' -import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' +import { MessageCodecEnum, MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' import { S3PayloadStore } from '@message-queue-toolkit/s3-payload-store' import { OFFLOADED_PAYLOAD_SIZE_ATTRIBUTE } from '@message-queue-toolkit/sqs' import type { AwilixContainer } from 'awilix' @@ -401,3 +401,110 @@ describe('SqsPermissionConsumer - nested messageTypePath with payload offloading }) }) }) + +describe('SqsPermissionConsumer - codec + payload offloading', () => { + const s3BucketName = 'test-bucket-codec' + // Threshold low enough that even a small compressed payload triggers offloading + const smallThreshold = 10 + + let diContainer: AwilixContainer + let s3: S3 + let testAdmin: TestAwsResourceAdmin + let payloadStoreConfig: SinglePayloadStoreConfig + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + s3 = diContainer.cradle.s3 + testAdmin = diContainer.cradle.testAdmin + + await testAdmin.createBucket(s3BucketName) + payloadStoreConfig = { + messageSizeThreshold: smallThreshold, + store: new S3PayloadStore(diContainer.cradle, { bucketName: s3BucketName }), + storeName: 's3', + } + }) + + afterAll(async () => { + await testAdmin.emptyBuckets(s3BucketName) + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('compresses payload, offloads to S3 as raw binary, and consumer decompresses correctly', async () => { + const queueName = 'codec-offload-roundtrip' + await testAdmin.deleteQueues(queueName) + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-offload-1', + messageType: 'add', + metadata: { info: 'compressed and offloaded' }, + } + + const publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + }) + // No codec on consumer — codec is read from payloadRef.codec in the pointer + const consumer = new SqsPermissionConsumer(diContainer.cradle, { + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + }) + + await publisher.init() + await consumer.start() + + await publisher.publish(message) + + // Verify payload was offloaded to S3 + const s3Keys = await waitForS3Objects(s3, s3BucketName, 1, 5000) + expect(s3Keys.length).toBeGreaterThan(0) + + // Verify consumer receives the correct decompressed payload + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await publisher.close() + await consumer.close(true) + }, 30_000) + + it('consumer without explicit codec still decompresses codec-offloaded payload', async () => { + const queueName = 'codec-offload-auto-detect' + await testAdmin.deleteQueues(queueName) + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-offload-auto-1', + messageType: 'add', + metadata: { info: 'auto-detect codec from pointer' }, + } + + const publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + }) + // Consumer has no explicit codec — should still work because codec comes from payloadRef.codec + const consumer = new SqsPermissionConsumer(diContainer.cradle, { + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + }) + + await publisher.init() + await consumer.start() + + await publisher.publish(message) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await publisher.close() + await consumer.close(true) + }, 30_000) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eeda1f46..cf6ea3d5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,14 +174,14 @@ importers: specifier: ^3.0.0 version: 3.1.0 '@message-queue-toolkit/core': - specifier: '*' - version: 25.5.0(zod@4.4.3) + specifier: workspace:* + version: link:../core '@message-queue-toolkit/gcs-payload-store': specifier: '*' - version: 1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3)) + version: 1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@packages+core) '@message-queue-toolkit/redis-message-deduplication-store': specifier: '*' - version: 2.0.2(@message-queue-toolkit/core@25.5.0(zod@4.4.3))(ioredis@5.10.1)(zod@4.4.3) + version: 2.0.2(@message-queue-toolkit/core@packages+core)(ioredis@5.10.1)(zod@4.4.3) '@message-queue-toolkit/schemas': specifier: '*' version: 7.1.0(zod@4.4.3) @@ -3374,15 +3374,15 @@ snapshots: toad-cache: 3.7.0 zod: 4.4.3 - '@message-queue-toolkit/gcs-payload-store@1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3))': + '@message-queue-toolkit/gcs-payload-store@1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@packages+core)': dependencies: '@google-cloud/storage': 7.19.0 - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) + '@message-queue-toolkit/core': link:packages/core - '@message-queue-toolkit/redis-message-deduplication-store@2.0.2(@message-queue-toolkit/core@25.5.0(zod@4.4.3))(ioredis@5.10.1)(zod@4.4.3)': + '@message-queue-toolkit/redis-message-deduplication-store@2.0.2(@message-queue-toolkit/core@packages+core)(ioredis@5.10.1)(zod@4.4.3)': dependencies: '@lokalise/node-core': 14.8.1(zod@4.4.3) - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) + '@message-queue-toolkit/core': link:packages/core ioredis: 5.10.1 redis-semaphore: 5.7.0(ioredis@5.10.1) transitivePeerDependencies: From 989165f392d19468f3541192fed68f59c1c7e975 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 15:31:40 +0200 Subject: [PATCH 9/9] test: verify wire format of compressed messages in integration tests For the inline codec path, read the raw SQS message body from an isolated queue (no consumer) using ReceiveMessageCommand and assert: - body is a JSON codec envelope with __codec === 'zstd' - __data decodes from base64 to a valid zstd frame (magic bytes 28 B5 2F FD) For the codec + payload offloading path, assert: - SQS message body is a plain JSON pointer (no __codec field), with payloadRef.codec === 'zstd' confirming which algorithm was used - S3 object contains raw compressed binary, not a JSON envelope (first 4 bytes match the zstd magic number 0xFD2FB528) Also add getObjectBuffer() to s3Utils for reading S3 objects as raw Buffer without UTF-8 decoding, and fix missing resolveCodecHandler import in codec README example snippet. Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 2 +- .../SqsPermissionConsumer.codec.spec.ts | 42 ++++++++++++++- ...rmissionConsumer.payloadOffloading.spec.ts | 53 +++++++++++++++++-- packages/sqs/test/utils/s3Utils.ts | 7 +++ 4 files changed, 99 insertions(+), 5 deletions(-) diff --git a/packages/codec/README.md b/packages/codec/README.md index 22b082dc..d915544f 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -44,7 +44,7 @@ const original = await decompressMessageBody(JSON.parse(compressed)) When you have pre-compressed bytes (e.g., from `resolveCodecHandler(codec).compress(...)`) and want to produce the envelope string without compressing again: ```typescript -import { buildCodecEnvelope } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' import { MessageCodecEnum } from '@message-queue-toolkit/core' const handler = resolveCodecHandler(MessageCodecEnum.ZSTD) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 9ac1b3a8..43f44493 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,4 +1,4 @@ -import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' import { compressMessageBody } from '@message-queue-toolkit/codec' import { MessageCodecEnum } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' @@ -65,6 +65,46 @@ describe('SqsPermissionConsumer - zstd codec', () => { expect(result.message).toMatchObject(message) }) + it('published SQS message body is a codec envelope containing valid zstd bytes', async () => { + // Use an isolated queue with no consumer so we can read the raw message without a race + const wireQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-wire-check` + await testAdmin.deleteQueues(wireQueueName) + + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: wireQueueName } }, + }) + await wirePublisher.init() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-wire-1', + messageType: 'add', + } + await wirePublisher.publish(message) + + // Read the raw message directly from SQS — no consumer is running on this queue + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages, 'Expected a message to be in the queue').toBeDefined() + expect(Messages!.length).toBe(1) + + // Body must be a self-describing codec envelope, not raw message JSON + const envelope = JSON.parse(Messages![0]!.Body!) as Record + expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__data).toBe('string') + + // __data must decode to a valid zstd frame: magic number 0xFD2FB528 (LE → 28 B5 2F FD) + const compressed = Buffer.from(envelope.__data as string, 'base64') + expect(compressed.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) + + await wirePublisher.close() + }) + it('consumer correctly handles multiple compressed messages in sequence', async () => { const messages: PERMISSIONS_ADD_MESSAGE_TYPE[] = [ { id: 'codec-seq-1', messageType: 'add' }, diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts index 370901a0..d41bad0c 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts @@ -1,5 +1,5 @@ import type { S3 } from '@aws-sdk/client-s3' -import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' import type { SinglePayloadStoreConfig } from '@message-queue-toolkit/core' import { MessageCodecEnum, MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' import { S3PayloadStore } from '@message-queue-toolkit/s3-payload-store' @@ -13,7 +13,7 @@ import { AbstractSqsConsumer } from '../../lib/sqs/AbstractSqsConsumer.ts' import { AbstractSqsPublisher } from '../../lib/sqs/AbstractSqsPublisher.ts' import { SQS_MESSAGE_MAX_SIZE } from '../../lib/sqs/AbstractSqsService.ts' import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' -import { putObjectContent, waitForS3Objects } from '../utils/s3Utils.ts' +import { getObjectBuffer, putObjectContent, waitForS3Objects } from '../utils/s3Utils.ts' import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' import type { Dependencies } from '../utils/testContext.ts' import { registerDependencies } from '../utils/testContext.ts' @@ -435,6 +435,54 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { await diContainer.dispose() }) + it('S3 object is raw zstd binary and SQS message carries a plain pointer (not a codec envelope)', async () => { + // Use an isolated queue with no consumer so we can read the raw SQS message without a race + const wireQueueName = 'codec-offload-wire-check' + await testAdmin.deleteQueues(wireQueueName) + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-offload-wire-1', + messageType: 'add', + metadata: { info: 'wire format check' }, + } + + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + payloadStoreConfig, + creationConfig: { queue: { QueueName: wireQueueName } }, + }) + await wirePublisher.init() + await wirePublisher.publish(message) + + // Read the raw SQS message before any consumer touches it + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages, 'Expected a message to be in the queue').toBeDefined() + expect(Messages!.length).toBe(1) + + // SQS body must be a plain JSON pointer — not a codec envelope. + // Compressed bytes live in S3; only the pointer is sent inline. + const sqsBody = JSON.parse(Messages![0]!.Body!) as Record + expect(sqsBody.__codec, 'SQS body must not be a codec envelope when offloading').toBeUndefined() + expect(sqsBody.payloadRef, 'SQS body must contain a payloadRef pointer').toBeDefined() + const payloadRef = sqsBody.payloadRef as Record + expect(payloadRef.codec).toBe(MessageCodecEnum.ZSTD) + + // S3 object must be raw compressed binary, not a JSON codec envelope. + // zstd frames start with magic number 0xFD2FB528 (little-endian: 28 B5 2F FD). + const s3Keys = await waitForS3Objects(s3, s3BucketName, 1, 5000) + expect(s3Keys.length).toBeGreaterThan(0) + const s3Bytes = await getObjectBuffer(s3, s3BucketName, s3Keys[0]!) + expect(s3Bytes.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) + + await wirePublisher.close() + }, 30_000) + it('compresses payload, offloads to S3 as raw binary, and consumer decompresses correctly', async () => { const queueName = 'codec-offload-roundtrip' await testAdmin.deleteQueues(queueName) @@ -459,7 +507,6 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { await publisher.init() await consumer.start() - await publisher.publish(message) // Verify payload was offloaded to S3 diff --git a/packages/sqs/test/utils/s3Utils.ts b/packages/sqs/test/utils/s3Utils.ts index d453eae3..01f0e5af 100644 --- a/packages/sqs/test/utils/s3Utils.ts +++ b/packages/sqs/test/utils/s3Utils.ts @@ -34,6 +34,13 @@ export async function getObjectContent(s3: S3, bucket: string, key: string) { return result.Body?.transformToString() } +export async function getObjectBuffer(s3: S3, bucket: string, key: string): Promise { + const result = await s3.getObject({ Bucket: bucket, Key: key }) + const bytes = await result.Body?.transformToByteArray() + if (!bytes) throw new Error(`No body for S3 object ${key}`) + return Buffer.from(bytes) +} + export async function putObjectContent(s3: S3, bucket: string, key: string, content: string) { await s3.putObject({ Bucket: bucket, Key: key, Body: content }) }