From a176d75ee7267376dfdf1217cf6f73be3e977db4 Mon Sep 17 00:00:00 2001 From: CarlosGamero <101278162+CarlosGamero@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:53:44 +0200 Subject: [PATCH] Mono publisher and consumer removal (#112) * Removing sqs mon publisher/consumer * core preparation for multi publishers only * SQS Abstract consumer * SQS publisher rename * SQS test fixes * Unused code removed * Fix lint * sqs Index fix + lint fix * Removing SNS mono consumer and publisher * Minor fix on sqs * SNS Fixing types * SNS renaming * SNS test fixes * amqp service types fix * amqp single publisher * amqp single consumer * removing unused tests * AMQP tests fixes * amqp index fix * Minor changes * Solving TODO * Minor change to not allow changing internal properties * SQS improving coverage * Adding tests * Lint fix * Removing unused test class * SQS making props protected * SNS marking some props as protected * Minor changes * AMQP improving tests coverage * Improving AMQP test coverage * Fix test * Trying to fix test * readme updated * Major version * Minor change * upgrading md * Lint fixes + solving todo + build fix * CR comments --- README.md | 79 +--- UPGRADING.md | 146 ++++++++ packages/amqp/index.ts | 8 +- .../amqp/lib/AbstractAmqpBasePublisher.ts | 53 --- ...aseConsumer.ts => AbstractAmqpConsumer.ts} | 290 +++++++++------ .../lib/AbstractAmqpConsumerMonoSchema.ts | 95 ----- .../lib/AbstractAmqpConsumerMultiSchema.ts | 118 ------ packages/amqp/lib/AbstractAmqpPublisher.ts | 94 +++++ .../lib/AbstractAmqpPublisherMonoSchema.ts | 47 --- .../lib/AbstractAmqpPublisherMultiSchema.ts | 56 --- packages/amqp/lib/AbstractAmqpService.ts | 20 +- packages/amqp/lib/types/MessageTypes.ts | 3 - packages/amqp/lib/utils/amqpInitter.ts | 4 +- packages/amqp/package.json | 2 +- ...spec.ts => AmqpPermissionConsumer.spec.ts} | 93 +++-- .../test/consumers/AmqpPermissionConsumer.ts | 143 +++++--- .../AmqpPermissionConsumerMultiSchema.ts | 111 ------ .../consumers/AmqpPermissionsConsumer.spec.ts | 243 ------------ packages/amqp/test/fakes/FakeConsumer.ts | 46 +-- .../test/fakes/FakeConsumerErrorResolver.ts | 12 +- .../test/fakes/FakeConsumerMultiSchema.ts | 43 --- .../AmqpPermissionPublisher.spec.ts | 122 +++--- .../publishers/AmqpPermissionPublisher.ts | 45 ++- .../AmqpPermissionPublisherMultiSchema.ts | 42 --- .../test/repositories/PermissionRepository.ts | 1 - packages/amqp/test/utils/testContext.ts | 21 +- packages/core/index.ts | 19 +- .../core/lib/queues/AbstractQueueService.ts | 121 ++---- packages/core/lib/queues/HandlerContainer.ts | 6 +- packages/core/lib/queues/HandlerSpy.ts | 3 +- packages/core/lib/types/queueOptionsTypes.ts | 70 ++++ packages/core/package.json | 2 +- packages/sns/index.ts | 25 +- ...MultiSchema.ts => AbstractSnsPublisher.ts} | 59 ++- .../lib/sns/AbstractSnsPublisherMonoSchema.ts | 71 ---- packages/sns/lib/sns/AbstractSnsService.ts | 103 +----- ...ltiSchema.ts => AbstractSnsSqsConsumer.ts} | 65 ++-- .../sns/AbstractSnsSqsConsumerMonoSchema.ts | 186 ---------- packages/sns/lib/utils/snsInitter.ts | 2 +- packages/sns/package.json | 2 +- ...ec.ts => SnsSqsPermissionConsumer.spec.ts} | 58 +-- ...iSchema.ts => SnsSqsPermissionConsumer.ts} | 69 ++-- .../SnsSqsPermissionConsumerMonoSchema.ts | 99 ----- ...nsSqsPermissionsConsumerMonoSchema.spec.ts | 277 -------------- .../publishers/SnsPermissionPublisher.spec.ts | 29 +- .../test/publishers/SnsPermissionPublisher.ts | 41 +++ .../SnsPermissionPublisherMonoSchema.ts | 33 -- .../SnsPermissionPublisherMultiSchema.ts | 41 --- .../test/repositories/PermissionRepository.ts | 1 - packages/sns/test/utils/testContext.ts | 32 +- packages/sqs/index.ts | 17 +- .../lib/errors/SqsConsumerErrorResolver.ts | 2 + .../lib/fakes/FakeConsumerErrorResolver.ts | 16 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 347 +++++++++++------- .../lib/sqs/AbstractSqsConsumerMonoSchema.ts | 151 -------- .../lib/sqs/AbstractSqsConsumerMultiSchema.ts | 160 -------- ...MultiSchema.ts => AbstractSqsPublisher.ts} | 48 ++- .../lib/sqs/AbstractSqsPublisherMonoSchema.ts | 70 ---- packages/sqs/lib/sqs/AbstractSqsService.ts | 81 +--- packages/sqs/lib/utils/sqsInitter.ts | 12 +- packages/sqs/package.json | 2 +- ....spec.ts => SqsPermissionConsumer.spec.ts} | 152 ++++---- ...ultiSchema.ts => SqsPermissionConsumer.ts} | 70 ++-- .../SqsPermissionConsumerMonoSchema.ts | 89 ----- ...rmissionsConsumerMonoSchema.errors.spec.ts | 90 ----- .../SqsPermissionsConsumerMonoSchema.spec.ts | 208 ----------- .../sqs/test/consumers/userConsumerSchemas.ts | 5 - .../publishers/SqsPermissionPublisher.spec.ts | 206 +++++++++++ .../test/publishers/SqsPermissionPublisher.ts | 54 +++ .../SqsPermissionPublisherMonoSchema.spec.ts | 157 -------- .../SqsPermissionPublisherMonoSchema.ts | 25 -- .../SqsPermissionPublisherMultiSchema.ts | 31 -- .../test/repositories/PermissionRepository.ts | 1 - packages/sqs/test/utils/testContext.ts | 29 +- 74 files changed, 1756 insertions(+), 3618 deletions(-) create mode 100644 UPGRADING.md delete mode 100644 packages/amqp/lib/AbstractAmqpBasePublisher.ts rename packages/amqp/lib/{AbstractAmqpBaseConsumer.ts => AbstractAmqpConsumer.ts} (58%) delete mode 100644 packages/amqp/lib/AbstractAmqpConsumerMonoSchema.ts delete mode 100644 packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts create mode 100644 packages/amqp/lib/AbstractAmqpPublisher.ts delete mode 100644 packages/amqp/lib/AbstractAmqpPublisherMonoSchema.ts delete mode 100644 packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts delete mode 100644 packages/amqp/lib/types/MessageTypes.ts rename packages/amqp/test/consumers/{AmqpPermissionsConsumerMultiSchema.spec.ts => AmqpPermissionConsumer.spec.ts} (70%) delete mode 100644 packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts delete mode 100644 packages/amqp/test/consumers/AmqpPermissionsConsumer.spec.ts delete mode 100644 packages/amqp/test/fakes/FakeConsumerMultiSchema.ts delete mode 100644 packages/amqp/test/publishers/AmqpPermissionPublisherMultiSchema.ts delete mode 100644 packages/amqp/test/repositories/PermissionRepository.ts create mode 100644 packages/core/lib/types/queueOptionsTypes.ts rename packages/sns/lib/sns/{AbstractSnsPublisherMultiSchema.ts => AbstractSnsPublisher.ts} (53%) delete mode 100644 packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts rename packages/sns/lib/sns/{AbstractSnsSqsConsumerMultiSchema.ts => AbstractSnsSqsConsumer.ts} (63%) delete mode 100644 packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts rename packages/sns/test/consumers/{SnsSqsPermissionsConsumerMultiSchema.spec.ts => SnsSqsPermissionConsumer.spec.ts} (83%) rename packages/sns/test/consumers/{SnsSqsPermissionConsumerMultiSchema.ts => SnsSqsPermissionConsumer.ts} (75%) delete mode 100644 packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts delete mode 100644 packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts create mode 100644 packages/sns/test/publishers/SnsPermissionPublisher.ts delete mode 100644 packages/sns/test/publishers/SnsPermissionPublisherMonoSchema.ts delete mode 100644 packages/sns/test/publishers/SnsPermissionPublisherMultiSchema.ts delete mode 100644 packages/sns/test/repositories/PermissionRepository.ts delete mode 100644 packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts delete mode 100644 packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts rename packages/sqs/lib/sqs/{AbstractSqsPublisherMultiSchema.ts => AbstractSqsPublisher.ts} (61%) delete mode 100644 packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts rename packages/sqs/test/consumers/{SqsPermissionsConsumerMultiSchema.spec.ts => SqsPermissionConsumer.spec.ts} (72%) rename packages/sqs/test/consumers/{SqsPermissionConsumerMultiSchema.ts => SqsPermissionConsumer.ts} (71%) delete mode 100644 packages/sqs/test/consumers/SqsPermissionConsumerMonoSchema.ts delete mode 100644 packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.errors.spec.ts delete mode 100644 packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.spec.ts create mode 100644 packages/sqs/test/publishers/SqsPermissionPublisher.spec.ts create mode 100644 packages/sqs/test/publishers/SqsPermissionPublisher.ts delete mode 100644 packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts delete mode 100644 packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts delete mode 100644 packages/sqs/test/publishers/SqsPermissionPublisherMultiSchema.ts delete mode 100644 packages/sqs/test/repositories/PermissionRepository.ts diff --git a/README.md b/README.md index 96287183..8bba8c1a 100644 --- a/README.md +++ b/README.md @@ -22,31 +22,7 @@ It consists of the following submodules: ### Publishers `message-queue-toolkit` provides base classes for implementing publishers for each of the supported protocol. - -#### Mono-schema publishers - -Mono-schema publishers only support a single message type and are simpler to implement. They expose the following public methods: - -* `constructor()`, which accepts the following parameters: - * `dependencies` – a set of dependencies depending on the protocol; - * `options`, composed by - * `messageSchema` – the `zod` schema for the message; - * `messageTypeField` - which field in the message describes the type of a message. This field needs to be defined as `z.literal` in the schema; - * `locatorConfig` - configuration for resolving existing queue and/or topic. Should not be specified together with the `creationConfig`. - * `creationConfig` - configuration for queue and/or topic to create, if one does not exist. Should not be specified together with the `locatorConfig`. -* `init()`, prepare publisher for use (e. g. establish all necessary connections), it will be called automatically by `publish()` if not called before explicitly (lazy loading). -* `close()`, stop publisher use (e. g. disconnect); -* `publish()`, send a message to a queue or topic. It accepts the following parameters: - * `message` – a message following a `zod` schema; - * `options` – a protocol-dependent set of message parameters. For more information please check documentation for options for each protocol: [AMQP](https://amqp-node.github.io/amqplib/channel_api.html#channel_sendToQueue), [SQS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sqs/interfaces/sendmessagecommandinput.html) and [SNS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sns/interfaces/publishcommandinput.html). - -> **_NOTE:_** See [SqsPermissionPublisherMonoSchema.ts](./packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts) for a practical example. - -> **_NOTE:_** Lazy loading is not supported for AMQP publishers. - -#### Multi-schema publishers - -Multi-schema publishers support multiple messages types. They implement the following public methods: +They implement the following public methods: * `constructor()`, which accepts the following parameters: * `dependencies` – a set of dependencies depending on the protocol; @@ -61,35 +37,14 @@ Multi-schema publishers support multiple messages types. They implement the foll * `message` – a message following one of the `zod` schemas, supported by the publisher; * `options` – a protocol-dependent set of message parameters. For more information please check documentation for options for each protocol: [AMQP](https://amqp-node.github.io/amqplib/channel_api.html#channel_sendToQueue), [SQS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sqs/interfaces/sendmessagecommandinput.html) and [SNS](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-sns/interfaces/publishcommandinput.html). +> **_NOTE:_** See [SqsPermissionPublisher.ts](./packages/sqs/test/publishers/SqsPermissionPublisher.ts) for a practical example. +> **_NOTE:_** Lazy loading is not supported for AMQP publishers. + ### Consumers `message-queue-toolkit` provides base classes for implementing consumers for each of the supported protocol. - -#### Mono-schema consumers - -Mono-schema consumers only support a single message type and are simpler to implement. They expose the following public methods: - -* `constructor()`, which accepts the following parameters: - * `dependencies` – a set of dependencies depending on the protocol; - * `options`, composed by - * `messageSchema` – the `zod` schema for the message; - * `messageTypeField` - which field in the message is used for resolving the message type (for observability purposes); - * `queueName`; (for SNS publishers this is a misnomer which actually refers to a topic name) - * `locatorConfig` - configuration for resolving existing queue and/or topic. Should not be specified together with the `creationConfig`. - * `creationConfig` - configuration for queue and/or topic to create, if one does not exist. Should not be specified together with the `locatorConfig`. - * `subscriptionConfig` - SNS SQS consumer only - configuration for SNS -> SQS subscription to create, if one doesn't exist. - * `consumerOverrides` – available only for SQS consumers; - * `subscribedToTopic` – parameters for a topic to use during creation if it does not exist. Ignored if `queueLocator.subscriptionArn` is set. Available only for SNS consumers; -* `init()`, prepare consumer for use (e. g. establish all necessary connections); -* `close()`, stop listening for messages and disconnect; -* `processMessage()`, which accepts as parameter a `message` following a `zod` schema and should be overridden with logic on what to do with the message; -* `start()`, which invokes `init()` and `processMessage()` and handles errors. -* `preHandlerBarrier`, which accepts as a parameter a `message` following a `zod` schema and can be overridden to enable the barrier pattern (see [Barrier pattern](#barrier-pattern)) - -> **_NOTE:_** See [SqsPermissionConsumerMonoSchema.ts](./packages/sqs/test/consumers/SqsPermissionConsumerMonoSchema.ts) for a practical example. - -#### Multi-schema consumers +They expose the following public methods: Multi-schema consumers support multiple message types via handler configs. They expose the following public methods: @@ -106,17 +61,18 @@ Multi-schema consumers support multiple message types via handler configs. They * `subscribedToTopic` – parameters for a topic to use during creation if it does not exist. Ignored if `queueLocator.subscriptionArn` is set. Available only for SNS consumers; * `init()`, prepare consumer for use (e. g. establish all necessary connections); * `close()`, stop listening for messages and disconnect; +* `start()`, which invokes `init()`. -* `processMessage()`, which accepts as parameter a `message` following a `zod` schema and should be overridden with logic on what to do with the message; -* `start()`, which invokes `init()` and `processMessage()` and handles errors. +> **_NOTE:_** See [SqsPermissionConsumer.ts](./packages/sqs/test/consumers/SqsPermissionConsumer.ts) for a practical example. -##### Multi-schema handler definition + +##### How to define a handler You can define handlers for each of the supported messages in a type-safe way using the MessageHandlerConfigBuilder. Here is an example: -```ts +```typescript type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE type ExecutionContext = { userService: UserService @@ -176,7 +132,7 @@ export class TestConsumerMultiSchema extends AbstractSqsConsumerMultiSchema< #### Error Handling -When implementing message handler in consumer (by overriding the `processMessage()` method), you are expected to return an instance of `Either`, containing either an error `retryLater`, or result `success`. In case of `retryLater`, the abstract consumer is instructed to requeue the message. Otherwise, in case of success, the message is finally removed from the queue. If an error is thrown while processing the message, the abstract consumer will also requeue the message. When overriding the `processMessage()` method, you should leverage the possible types to process the message as you need. +When implementing a handler, you are expected to return an instance of `Either`, containing either an error `retryLater`, or result `success`. In case of `retryLater`, the abstract consumer is instructed to requeue the message. Otherwise, in case of success, the message is finally removed from the queue. If an error is thrown while processing the message, the abstract consumer will also requeue the message. #### Schema Validation and Deserialization @@ -189,23 +145,22 @@ If Then the message is automatically nacked without requeueing by the abstract consumer and processing fails. -> **_NOTE:_** See [userConsumerSchemas.ts](./packages/sqs/test/consumers/userConsumerSchemas.ts) and [SqsPermissionsConsumerMonoSchema.spec.ts](./packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.spec.ts) for a practical example. +> **_NOTE:_** See [userConsumerSchemas.ts](./packages/sqs/test/consumers/userConsumerSchemas.ts) and [SqsPermissionsConsumer.spec.ts](./packages/sqs/test/consumers/SqsPermissionsConsumer.spec.ts) for a practical example. ### Barrier pattern The barrier pattern facilitates the out-of-order message handling by retrying the message later if the system is not yet in the proper state to be able to process that message (e. g. some prerequisite messages have not yet arrived). -To enable this pattern you should implement `preHandlerBarrier` in order to define the conditions for starting to process the message. +To enable this pattern you should define `preHandlerBarrier` on your message handler in order to define the conditions for starting to process the message. If the barrier method returns `false`, message will be returned into the queue for the later processing. If the barrier method returns `true`, message will be processed. -> **_NOTE:_** See [SqsPermissionConsumerMonoSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts) for a practical example on mono consumers. -> **_NOTE:_** See [SqsPermissionConsumerMultiSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts) for a practical example on multi consumers. +> **_NOTE:_** See [SqsPermissionConsumer.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumer.ts) for a practical example. ## Fan-out to Multiple Consumers SQS queues are built in a way that every message is only consumed once, and then deleted. If you want to do fan-out to multiple consumers, you need SNS topic in the middle, which is then propagated to all the SQS queues that have subscribed. -> **_NOTE:_** See [SnsPermissionPublisher.ts](./packages/sns/test/publishers/SnsPermissionPublisherMonoSchema.ts) and [SnsSqsPermissionConsumerMonoSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts) for a practical example. +> **_NOTE:_** See [SnsPermissionPublisher.ts](./packages/sns/test/publishers/SnsPermissionPublisher.ts) and [SnsSqsPermissionConsumerMonoSchema.ts](./packages/sns/test/consumers/SnsSqsPermissionConsumer.ts) for a practical example. ## Automatic Queue and Topic Creation @@ -220,7 +175,7 @@ In certain cases you want to await until certain publisher publishes a message, In order to enable this functionality, configure spyHandler on the publisher or consumer: ```ts -export class TestConsumerMultiSchema extends AbstractSqsConsumerMultiSchema< +export class TestConsumerMultiSchema extends AbstractSqsConsumer< SupportedMessages, ExecutionContext > { @@ -236,7 +191,7 @@ export class TestConsumerMultiSchema extends AbstractSqsConsumerMultiSchema< bufferSize: 100, // how many processed messages should be retained in memory for spy lookup. Default is 100 messageIdField: 'id', // which field within a message payload uniquely identifies it. Default is `id` }, - } + }) } } ``` diff --git a/UPGRADING.md b/UPGRADING.md new file mode 100644 index 00000000..cd62cc1b --- /dev/null +++ b/UPGRADING.md @@ -0,0 +1,146 @@ +# Upgrading Guide + +We have introduced the following breaking changes on version `12.0.0`, please follow the steps below to update your code +from the previous version to the new one. + +## Breaking Changes + +### Description of Breaking Change +Multi consumers and publishers can accomplish the same tasks as mono ones, but they add extra layer of complexity by +requiring features to be implemented in both. +As a result, we have decided to remove the mono ones to enhance maintainability. + +### Migration Steps +#### Multi consumers and publishers +If you are using the multi consumer or consumer, you will only need to rename the class you are extending, and it should +work as before. +- `AbstractSqsMultiConsumer` -> `AbstractSqsConsumer` +- `AbstractSqsMultiPublisher` -> `AbstractSqsPublisher` + +#### Mono consumers and publishers +If you are using the mono consumer or publisher, they no longer exist, so you will need to adjust your code to use +the old named multi consumer or publisher (now called just consumer or publisher). Please check the guide below. + +##### Publisher +1. Rename the class you are extending from `AbstractSqsPublisherMonoSchema` to `AbstractSqsPublisherSchema`. +2. replace the `messageSchema` property with `messageSchemas`, it is an array of `zod` schemas. +```typescript +// Old code +export class MyPublisher extends AbstractSqsPublisherMonoSchema { + public static QUEUE_NAME = 'my-queue-name' + + constructor(dependencies: SQSDependencies) { + super(dependencies, { + creationConfig: { + queue: { + QueueName: SqsPermissionPublisherMonoSchema.QUEUE_NAME, + }, + }, + handlerSpy: true, + deletionConfig: { + deleteIfExists: false, + }, + logMessages: true, + messageSchema: MY_MESSAGE_SCHEMA, + messageTypeField: 'messageType', + }) + } +} + +// Updated code +export class MyPublisher extends AbstractSqsPublisher { + public static QUEUE_NAME = 'my-queue-name' + + constructor(dependencies: SQSDependencies) { + super(dependencies, { + creationConfig: { + queue: { + QueueName: SqsPermissionPublisherMonoSchema.QUEUE_NAME, + }, + }, + handlerSpy: true, + deletionConfig: { + deleteIfExists: false, + }, + logMessages: true, + messageSchemas: [MY_MESSAGE_SCHEMA], + messageTypeField: 'messageType', + }) + } +} +``` + +##### Consumer +1. Rename the class you are extending from `AbstractSqsConsumerMonoSchema` to `AbstractSqsConsumer`. +2. Remove the `messageSchema` property. +3. Define a handler (`handlers` property) for your message, specifying the `zod` schema (old `messageSchema`) and the + method to handle the message (old `processMessage` method) +```typescript +// Old code +export class MyConsumer extends AbstractAmqpConsumerMonoSchema { + public static QUEUE_NAME = 'my-queue-name' + + constructor(dependencies: AMQPConsumerDependencies) { + super(dependencies, { + creationConfig: { + queueName: AmqpPermissionConsumer.QUEUE_NAME, + queueOptions: { + durable: true, + autoDelete: false, + }, + }, + deletionConfig: { + deleteIfExists: true, + }, + messageSchema: MY_MESSAGE_SCHEMA, + messageTypeField: 'messageType', + }) + } + + override async processMessage( + message: MyType, + ): Promise> { + // Your handling code + return { result: 'success' } + } +} + +// Updated code +export class MyConsumer extends AbstractAmqpConsumer { + public static QUEUE_NAME = 'my-queue-name' + + constructor(dependencies: AMQPConsumerDependencies) { + super( + dependencies, + { + creationConfig: { + queueName: AmqpPermissionConsumer.QUEUE_NAME, + queueOptions: { + durable: true, + autoDelete: false, + }, + }, + deletionConfig: { + deleteIfExists: true, + }, + messageTypeField: 'messageType', + handlers: new MessageHandlerConfigBuilder() + .addConfig( + MY_MESSAGE_SCHEMA, + async (message) => { + // Your handling code + return { + result: 'success', + } + }, + ) + .build(), + }, + undefined + ) + } +} +``` +> **_NOTE:_** on this example code we are omitting the barrier pattern (`preHandlerBarrier`) and pre handlers (`preHandlers`) +to simplify the example. If you are using them, please check [SqsPermissionConsumer.ts](./packages/sqs/test/consumers/SqsPermissionConsumer.ts) +to see how to update your code. diff --git a/packages/amqp/index.ts b/packages/amqp/index.ts index 7b1078bf..7d1fae09 100644 --- a/packages/amqp/index.ts +++ b/packages/amqp/index.ts @@ -1,13 +1,9 @@ -export type { CommonMessage } from './lib/types/MessageTypes' - export type { AMQPQueueConfig } from './lib/AbstractAmqpService' -export { AbstractAmqpConsumerMonoSchema } from './lib/AbstractAmqpConsumerMonoSchema' -export { AbstractAmqpConsumerMultiSchema } from './lib/AbstractAmqpConsumerMultiSchema' +export { AbstractAmqpConsumer, AMQPConsumerOptions } from './lib/AbstractAmqpConsumer' export { AmqpConsumerErrorResolver } from './lib/errors/AmqpConsumerErrorResolver' -export { AbstractAmqpPublisherMonoSchema } from './lib/AbstractAmqpPublisherMonoSchema' -export { AbstractAmqpPublisherMultiSchema } from './lib/AbstractAmqpPublisherMultiSchema' +export { AbstractAmqpPublisher, AMQPPublisherOptions } from './lib/AbstractAmqpPublisher' export type { AmqpConfig } from './lib/amqpConnectionResolver' diff --git a/packages/amqp/lib/AbstractAmqpBasePublisher.ts b/packages/amqp/lib/AbstractAmqpBasePublisher.ts deleted file mode 100644 index cc7335dd..00000000 --- a/packages/amqp/lib/AbstractAmqpBasePublisher.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - BarrierResult, - MessageInvalidFormatError, - MessageValidationError, -} from '@message-queue-toolkit/core' -import { objectToBuffer } from '@message-queue-toolkit/core' - -import { AbstractAmqpService } from './AbstractAmqpService' - -export abstract class AbstractAmqpBasePublisher< - MessagePayloadType extends object, -> extends AbstractAmqpService { - protected sendToQueue(message: MessagePayloadType): void { - try { - this.channel.sendToQueue(this.queueName, objectToBuffer(message)) - } catch (err) { - // Unfortunately, reliable retry mechanism can't be implemented with try-catch block, - // as not all failures end up here. If connection is closed programmatically, it works fine, - // but if server closes connection unexpectedly (e. g. RabbitMQ is shut down), then we don't land here - // @ts-ignore - if (err.message === 'Channel closed') { - this.logger.error(`AMQP channel closed`) - void this.reconnect() - } else { - throw err - } - } - } - - /* c8 ignore start */ - protected resolveMessage(): Either { - throw new Error('Not implemented for publisher') - } - - /* c8 ignore start */ - protected override processPrehandlers(): Promise { - throw new Error('Not implemented for publisher') - } - - protected override preHandlerBarrier(): Promise> { - throw new Error('Not implemented for publisher') - } - - protected override resolveNextFunction(): () => void { - throw new Error('Not implemented for publisher') - } - - override processMessage(): Promise> { - throw new Error('Not implemented for publisher') - } - /* c8 ignore stop */ -} diff --git a/packages/amqp/lib/AbstractAmqpBaseConsumer.ts b/packages/amqp/lib/AbstractAmqpConsumer.ts similarity index 58% rename from packages/amqp/lib/AbstractAmqpBaseConsumer.ts rename to packages/amqp/lib/AbstractAmqpConsumer.ts index 80a9aad4..9ab5df3b 100644 --- a/packages/amqp/lib/AbstractAmqpBaseConsumer.ts +++ b/packages/amqp/lib/AbstractAmqpConsumer.ts @@ -1,77 +1,151 @@ import type { Either, ErrorResolver } from '@lokalise/node-core' import type { + BarrierResult, + Prehandler, + PrehandlingOutputs, QueueConsumer, - NewQueueOptions, + QueueConsumerOptions, TransactionObservabilityManager, - ExistingQueueOptions, - Prehandler, } from '@message-queue-toolkit/core' -import { isMessageError, parseMessage } from '@message-queue-toolkit/core' +import { + isMessageError, + parseMessage, + HandlerContainer, + MessageSchemaContainer, +} from '@message-queue-toolkit/core' import type { Connection, Message } from 'amqplib' -import type { AMQPConsumerDependencies, CreateAMQPQueueOptions } from './AbstractAmqpService' +import type { + AMQPConsumerDependencies, + AMQPLocator, + AMQPCreationConfig, +} from './AbstractAmqpService' import { AbstractAmqpService } from './AbstractAmqpService' import { readAmqpMessage } from './amqpMessageReader' -const ABORT_EARLY_EITHER: Either<'abort', never> = { - error: 'abort', -} - -export type AMQPLocatorType = { queueName: string } - -export type CommonQueueOptions< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput, -> = { - prehandlers?: Prehandler[] -} - -export type NewAMQPConsumerOptions< - MessagePayloadType extends object, - ExecutionContext = undefined, - PrehandlerOutput = undefined, -> = NewQueueOptions & - CommonQueueOptions +const ABORT_EARLY_EITHER: Either<'abort', never> = { error: 'abort' } -export type ExistingAMQPConsumerOptions< +export type AMQPConsumerOptions< MessagePayloadType extends object, ExecutionContext = undefined, PrehandlerOutput = undefined, -> = ExistingQueueOptions & - CommonQueueOptions +> = QueueConsumerOptions< + AMQPCreationConfig, + AMQPLocator, + MessagePayloadType, + ExecutionContext, + PrehandlerOutput +> -export abstract class AbstractAmqpBaseConsumer< +export abstract class AbstractAmqpConsumer< MessagePayloadType extends object, ExecutionContext, - PrehandlerOutput, - BarrierOutput, + PrehandlerOutput = undefined, > extends AbstractAmqpService< MessagePayloadType, AMQPConsumerDependencies, ExecutionContext, - PrehandlerOutput, - BarrierOutput + PrehandlerOutput > implements QueueConsumer { private readonly transactionObservabilityManager?: TransactionObservabilityManager - protected readonly errorResolver: ErrorResolver + private readonly errorResolver: ErrorResolver + + private readonly messageSchemaContainer: MessageSchemaContainer + private readonly handlerContainer: HandlerContainer< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput + > + private readonly executionContext: ExecutionContext constructor( dependencies: AMQPConsumerDependencies, - options: - | NewAMQPConsumerOptions - | ExistingAMQPConsumerOptions, + options: AMQPConsumerOptions, + executionContext: ExecutionContext, ) { super(dependencies, options) + this.transactionObservabilityManager = dependencies.transactionObservabilityManager this.errorResolver = dependencies.consumerErrorResolver - if (!options.locatorConfig?.queueName && !options.creationConfig?.queueName) { - throw new Error('queueName must be set in either locatorConfig or creationConfig') - } + const messageSchemas = options.handlers.map((entry) => entry.schema) + this.messageSchemaContainer = new MessageSchemaContainer({ + messageSchemas, + messageTypeField: options.messageTypeField, + }) + this.handlerContainer = new HandlerContainer< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput + >({ + messageTypeField: this.messageTypeField, + messageHandlers: options.handlers, + }) + this.executionContext = executionContext + } + + async start() { + await this.init() + if (!this.channel) throw new Error('Channel is not set') + await this.consume() + } + + async receiveNewConnection(connection: Connection): Promise { + await super.receiveNewConnection(connection) + await this.consume() + } + + private async consume() { + await this.channel.consume(this.queueName, (message) => { + if (message === null) { + return + } + + const deserializedMessage = this.deserializeMessage(message) + if (deserializedMessage.error === 'abort') { + this.channel.nack(message, false, false) + const messageId = this.tryToExtractId(message) + this.handleMessageProcessed(null, 'invalid_message', messageId.result) + return + } + // @ts-ignore + const messageType = deserializedMessage.result[this.messageTypeField] + const transactionSpanId = `queue_${this.queueName}:${ + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + deserializedMessage.result[this.messageTypeField] + }` + + this.transactionObservabilityManager?.start(transactionSpanId) + if (this.logMessages) { + const resolvedLogMessage = this.resolveMessageLog(deserializedMessage.result, messageType) + this.logMessage(resolvedLogMessage) + } + this.internalProcessMessage(deserializedMessage.result, messageType) + .then((result) => { + if (result.error === 'retryLater') { + this.channel.nack(message, false, true) + this.handleMessageProcessed(deserializedMessage.result, 'retryLater') + } + if (result.result === 'success') { + this.channel.ack(message) + this.handleMessageProcessed(deserializedMessage.result, 'consumed') + } + }) + .catch((err) => { + // ToDo we need sanity check to stop trying at some point, perhaps some kind of Redis counter + // If we fail due to unknown reason, let's retry + this.channel.nack(message, false, true) + this.handleMessageProcessed(deserializedMessage.result, 'retryLater') + this.handleError(err) + }) + .finally(() => { + this.transactionObservabilityManager?.stop(transactionSpanId) + }) + }) } private async internalProcessMessage( @@ -90,11 +164,69 @@ export abstract class AbstractAmqpBaseConsumer< return { error: 'retryLater' } } - private deserializeMessage(message: Message | null): Either<'abort', MessagePayloadType> { - if (message === null) { - return ABORT_EARLY_EITHER - } + protected override async processMessage( + message: MessagePayloadType, + messageType: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prehandlingOutputs: PrehandlingOutputs, + ): Promise> { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.handler(message, this.executionContext, prehandlingOutputs) + } + + protected override processPrehandlers(message: MessagePayloadType, messageType: string) { + const handlerConfig = this.handlerContainer.resolveHandler(messageType) + + return this.processPrehandlersInternal(handlerConfig.prehandlers, message) + } + + protected override async preHandlerBarrier( + message: MessagePayloadType, + messageType: string, + prehandlerOutput: PrehandlerOutput, + ): Promise> { + const handler = this.handlerContainer.resolveHandler( + messageType, + ) + + return this.preHandlerBarrierInternal( + handler.preHandlerBarrier, + message, + this.executionContext, + prehandlerOutput, + ) + } + + protected override resolveSchema(message: MessagePayloadType) { + return this.messageSchemaContainer.resolveSchema(message) + } + protected override resolveMessageLog(message: MessagePayloadType, messageType: string): unknown { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.messageLogFormatter(message) + } + + // eslint-disable-next-line max-params + protected override resolveNextFunction( + prehandlers: Prehandler[], + message: MessagePayloadType, + index: number, + prehandlerOutput: PrehandlerOutput, + resolve: (value: PrehandlerOutput | PromiseLike) => void, + reject: (err: Error) => void, + ) { + return this.resolveNextPreHandlerFunctionInternal( + prehandlers, + this.executionContext, + message, + index, + prehandlerOutput, + resolve, + reject, + ) + } + + private deserializeMessage(message: Message): Either<'abort', MessagePayloadType> { const resolveMessageResult = this.resolveMessage(message) if (isMessageError(resolveMessageResult.error)) { this.handleError(resolveMessageResult.error) @@ -134,16 +266,7 @@ export abstract class AbstractAmqpBaseConsumer< } } - async receiveNewConnection(connection: Connection): Promise { - await super.receiveNewConnection(connection) - await this.consume() - } - - private tryToExtractId(message: Message | null): Either<'abort', string> { - if (message === null) { - return ABORT_EARLY_EITHER - } - + private tryToExtractId(message: Message): Either<'abort', string> { const resolveMessageResult = this.resolveMessage(message) if (isMessageError(resolveMessageResult.error)) { this.handleError(resolveMessageResult.error) @@ -165,65 +288,6 @@ export abstract class AbstractAmqpBaseConsumer< return ABORT_EARLY_EITHER } - private async consume() { - await this.channel.consume(this.queueName, (message) => { - if (message === null) { - return - } - - const deserializedMessage = this.deserializeMessage(message) - if (deserializedMessage.error === 'abort') { - this.channel.nack(message, false, false) - const messageId = this.tryToExtractId(message) - this.handleMessageProcessed(null, 'invalid_message', messageId.result) - return - } - // @ts-ignore - const messageType = deserializedMessage.result[this.messageTypeField] - const transactionSpanId = `queue_${this.queueName}:${ - // @ts-ignore - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - deserializedMessage.result[this.messageTypeField] - }` - - this.transactionObservabilityManager?.start(transactionSpanId) - if (this.logMessages) { - const resolvedLogMessage = this.resolveMessageLog(deserializedMessage.result, messageType) - this.logMessage(resolvedLogMessage) - } - this.internalProcessMessage(deserializedMessage.result, messageType) - .then((result) => { - if (result.error === 'retryLater') { - this.channel.nack(message, false, true) - this.handleMessageProcessed(deserializedMessage.result, 'retryLater') - } - if (result.result === 'success') { - this.channel.ack(message) - this.handleMessageProcessed(deserializedMessage.result, 'consumed') - } - }) - .catch((err) => { - // ToDo we need sanity check to stop trying at some point, perhaps some kind of Redis counter - // If we fail due to unknown reason, let's retry - this.channel.nack(message, false, true) - this.handleMessageProcessed(deserializedMessage.result, 'retryLater') - this.handleError(err) - }) - .finally(() => { - this.transactionObservabilityManager?.stop(transactionSpanId) - }) - }) - } - - async start() { - await this.init() - if (!this.channel) { - throw new Error('Channel is not set') - } - - await this.consume() - } - protected resolveMessage(message: Message) { return readAmqpMessage(message, this.errorResolver) } diff --git a/packages/amqp/lib/AbstractAmqpConsumerMonoSchema.ts b/packages/amqp/lib/AbstractAmqpConsumerMonoSchema.ts deleted file mode 100644 index e0ead054..00000000 --- a/packages/amqp/lib/AbstractAmqpConsumerMonoSchema.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - QueueConsumer, - MonoSchemaQueueOptions, - BarrierResult, - Prehandler, -} from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' - -import type { - ExistingAMQPConsumerOptions, - NewAMQPConsumerOptions, -} from './AbstractAmqpBaseConsumer' -import { AbstractAmqpBaseConsumer } from './AbstractAmqpBaseConsumer' -import type { AMQPConsumerDependencies } from './AbstractAmqpService' - -const DEFAULT_BARRIER_RESULT = { - isPassing: true, - output: undefined, -} as const - -export abstract class AbstractAmqpConsumerMonoSchema< - MessagePayloadType extends object, - ExecutionContext = undefined, - PrehandlerOutput = undefined, - BarrierOutput = undefined, - > - extends AbstractAmqpBaseConsumer< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - BarrierOutput - > - implements QueueConsumer -{ - private readonly messageSchema: ZodSchema - private readonly schemaEither: Either> - private readonly prehandlers: Prehandler[] - - constructor( - dependencies: AMQPConsumerDependencies, - options: - | (NewAMQPConsumerOptions & - MonoSchemaQueueOptions) - | (ExistingAMQPConsumerOptions & - MonoSchemaQueueOptions), - ) { - super(dependencies, options) - - this.prehandlers = options.prehandlers ?? [] - this.messageSchema = options.messageSchema - this.schemaEither = { - result: this.messageSchema, - } - } - - protected override processPrehandlers(message: MessagePayloadType, _messageType: string) { - return this.processPrehandlersInternal(this.prehandlers, message) - } - - // MonoSchema support is going away in the next semver major, so we don't care about coverage strongly - /* c8 ignore start */ - // eslint-disable-next-line max-params - protected override resolveNextFunction( - prehandlers: Prehandler[], - message: MessagePayloadType, - index: number, - prehandlerOutput: PrehandlerOutput, - resolve: (value: PrehandlerOutput | PromiseLike) => void, - reject: (err: Error) => void, - ) { - return this.resolveNextPreHandlerFunctionInternal( - prehandlers, - this as unknown as ExecutionContext, - message, - index, - prehandlerOutput, - resolve, - reject, - ) - } - /* c8 ignore stop */ - - protected override resolveSchema(_message: MessagePayloadType) { - return this.schemaEither - } - - /** - * Override to implement barrier pattern - */ - protected preHandlerBarrier(_message: MessagePayloadType): Promise> { - // @ts-ignore - return Promise.resolve(DEFAULT_BARRIER_RESULT) - } -} diff --git a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts b/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts deleted file mode 100644 index 5d14f987..00000000 --- a/packages/amqp/lib/AbstractAmqpConsumerMultiSchema.ts +++ /dev/null @@ -1,118 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - QueueConsumer, - MultiSchemaConsumerOptions, - BarrierResult, - Prehandler, - PrehandlingOutputs, -} from '@message-queue-toolkit/core' -import { HandlerContainer, MessageSchemaContainer } from '@message-queue-toolkit/core' - -import type { NewAMQPConsumerOptions } from './AbstractAmqpBaseConsumer' -import { AbstractAmqpBaseConsumer } from './AbstractAmqpBaseConsumer' -import type { AMQPConsumerDependencies } from './AbstractAmqpService' - -export abstract class AbstractAmqpConsumerMultiSchema< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput = undefined, - BarrierOutput = undefined, - > - extends AbstractAmqpBaseConsumer< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - BarrierOutput - > - implements QueueConsumer -{ - messageSchemaContainer: MessageSchemaContainer - handlerContainer: HandlerContainer - protected readonly executionContext: ExecutionContext - - constructor( - dependencies: AMQPConsumerDependencies, - options: NewAMQPConsumerOptions & - MultiSchemaConsumerOptions, - executionContext: ExecutionContext, - ) { - super(dependencies, options) - const messageSchemas = options.handlers.map((entry) => entry.schema) - - this.messageSchemaContainer = new MessageSchemaContainer({ - messageSchemas, - messageTypeField: options.messageTypeField, - }) - this.handlerContainer = new HandlerContainer< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput - >({ - messageTypeField: this.messageTypeField, - messageHandlers: options.handlers, - }) - this.executionContext = executionContext - } - - protected override resolveSchema(message: MessagePayloadType) { - return this.messageSchemaContainer.resolveSchema(message) - } - - public override async processMessage( - message: MessagePayloadType, - messageType: string, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - prehandlingOutputs: PrehandlingOutputs, - ): Promise> { - const handler = this.handlerContainer.resolveHandler(messageType) - return handler.handler(message, this.executionContext, prehandlingOutputs) - } - - protected override resolveMessageLog(message: MessagePayloadType, messageType: string): unknown { - const handler = this.handlerContainer.resolveHandler(messageType) - return handler.messageLogFormatter(message) - } - - protected override processPrehandlers(message: MessagePayloadType, messageType: string) { - const handlerConfig = this.handlerContainer.resolveHandler(messageType) - - return this.processPrehandlersInternal(handlerConfig.prehandlers, message) - } - - // eslint-disable-next-line max-params - protected override resolveNextFunction( - prehandlers: Prehandler[], - message: MessagePayloadType, - index: number, - prehandlerOutput: PrehandlerOutput, - resolve: (value: PrehandlerOutput | PromiseLike) => void, - reject: (err: Error) => void, - ) { - return this.resolveNextPreHandlerFunctionInternal( - prehandlers, - this.executionContext, - message, - index, - prehandlerOutput, - resolve, - reject, - ) - } - - protected override async preHandlerBarrier( - message: MessagePayloadType, - messageType: string, - prehandlerOutput: PrehandlerOutput, - ): Promise> { - const handler = this.handlerContainer.resolveHandler( - messageType, - ) - // @ts-ignore - return handler.preHandlerBarrier - ? await handler.preHandlerBarrier(message, this.executionContext, prehandlerOutput) - : { - isPassing: true, - output: undefined, - } - } -} diff --git a/packages/amqp/lib/AbstractAmqpPublisher.ts b/packages/amqp/lib/AbstractAmqpPublisher.ts new file mode 100644 index 00000000..9439f125 --- /dev/null +++ b/packages/amqp/lib/AbstractAmqpPublisher.ts @@ -0,0 +1,94 @@ +import type { Either } from '@lokalise/node-core' +import type { + BarrierResult, + MessageInvalidFormatError, + MessageValidationError, + QueuePublisherOptions, + SyncPublisher, +} from '@message-queue-toolkit/core' +import { MessageSchemaContainer, objectToBuffer } from '@message-queue-toolkit/core' +import type { ZodSchema } from 'zod' + +import type { AMQPLocator, AMQPCreationConfig, AMQPDependencies } from './AbstractAmqpService' +import { AbstractAmqpService } from './AbstractAmqpService' + +export type AMQPPublisherOptions = QueuePublisherOptions< + AMQPCreationConfig, + AMQPLocator, + MessagePayloadType +> + +export abstract class AbstractAmqpPublisher + extends AbstractAmqpService + implements SyncPublisher +{ + private readonly messageSchemaContainer: MessageSchemaContainer + + constructor(dependencies: AMQPDependencies, options: AMQPPublisherOptions) { + super(dependencies, options) + + const messageSchemas = options.messageSchemas + this.messageSchemaContainer = new MessageSchemaContainer({ + messageSchemas, + messageTypeField: options.messageTypeField, + }) + } + + publish(message: MessagePayloadType): void { + const resolveSchemaResult = this.resolveSchema(message) + if (resolveSchemaResult.error) { + throw resolveSchemaResult.error + } + resolveSchemaResult.result.parse(message) + + if (this.logMessages) { + // @ts-ignore + const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) + this.logMessage(resolvedLogMessage) + } + + try { + this.channel.sendToQueue(this.queueName, objectToBuffer(message)) + } catch (err) { + // Unfortunately, reliable retry mechanism can't be implemented with try-catch block, + // as not all failures end up here. If connection is closed programmatically, it works fine, + // but if server closes connection unexpectedly (e. g. RabbitMQ is shut down), then we don't land here + // @ts-ignore + if (err.message === 'Channel closed') { + this.logger.error(`AMQP channel closed`) + void this.reconnect() + } else { + throw err + } + } + } + + protected override resolveSchema( + message: MessagePayloadType, + ): Either> { + return this.messageSchemaContainer.resolveSchema(message) + } + + /* c8 ignore start */ + protected resolveMessage(): Either { + throw new Error('Not implemented for publisher') + } + + /* c8 ignore start */ + protected override processPrehandlers(): Promise { + throw new Error('Not implemented for publisher') + } + + protected override preHandlerBarrier(): Promise> { + throw new Error('Not implemented for publisher') + } + + protected override resolveNextFunction(): () => void { + throw new Error('Not implemented for publisher') + } + + override processMessage(): Promise> { + throw new Error('Not implemented for publisher') + } + /* c8 ignore stop */ +} diff --git a/packages/amqp/lib/AbstractAmqpPublisherMonoSchema.ts b/packages/amqp/lib/AbstractAmqpPublisherMonoSchema.ts deleted file mode 100644 index 05f2aea4..00000000 --- a/packages/amqp/lib/AbstractAmqpPublisherMonoSchema.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - ExistingQueueOptions, - MonoSchemaQueueOptions, - NewQueueOptions, - SyncPublisher, -} from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' - -import type { AMQPLocatorType } from './AbstractAmqpBaseConsumer' -import { AbstractAmqpBasePublisher } from './AbstractAmqpBasePublisher' -import type { AMQPDependencies, CreateAMQPQueueOptions } from './AbstractAmqpService' - -export abstract class AbstractAmqpPublisherMonoSchema - extends AbstractAmqpBasePublisher - implements SyncPublisher -{ - private readonly messageSchema: ZodSchema - - constructor( - dependencies: AMQPDependencies, - options: (NewQueueOptions | ExistingQueueOptions) & - MonoSchemaQueueOptions, - ) { - super(dependencies, options) - - this.messageSchema = options.messageSchema - } - - publish(message: MessagePayloadType): void { - this.messageSchema.parse(message) - - if (this.logMessages) { - // @ts-ignore - const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) - this.logMessage(resolvedLogMessage) - } - - this.sendToQueue(message) - } - - /* c8 ignore start */ - protected override resolveSchema(): Either> { - throw new Error('Not implemented for publisher') - } - /* c8 ignore stop */ -} diff --git a/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts b/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts deleted file mode 100644 index e196d132..00000000 --- a/packages/amqp/lib/AbstractAmqpPublisherMultiSchema.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - ExistingQueueOptions, - NewQueueOptions, - SyncPublisher, - MultiSchemaPublisherOptions, -} from '@message-queue-toolkit/core' -import { MessageSchemaContainer } from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' - -import type { AMQPLocatorType } from './AbstractAmqpBaseConsumer' -import { AbstractAmqpBasePublisher } from './AbstractAmqpBasePublisher' -import type { AMQPDependencies, CreateAMQPQueueOptions } from './AbstractAmqpService' - -export abstract class AbstractAmqpPublisherMultiSchema - extends AbstractAmqpBasePublisher - implements SyncPublisher -{ - private readonly messageSchemaContainer: MessageSchemaContainer - - constructor( - dependencies: AMQPDependencies, - options: (NewQueueOptions | ExistingQueueOptions) & - MultiSchemaPublisherOptions, - ) { - super(dependencies, options) - - const messageSchemas = options.messageSchemas - this.messageSchemaContainer = new MessageSchemaContainer({ - messageSchemas, - messageTypeField: options.messageTypeField, - }) - } - - publish(message: MessagePayloadType): void { - const resolveSchemaResult = this.resolveSchema(message) - if (resolveSchemaResult.error) { - throw resolveSchemaResult.error - } - resolveSchemaResult.result.parse(message) - - if (this.logMessages) { - // @ts-ignore - const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) - this.logMessage(resolvedLogMessage) - } - - this.sendToQueue(message) - } - - protected override resolveSchema( - message: MessagePayloadType, - ): Either> { - return this.messageSchemaContainer.resolveSchema(message) - } -} diff --git a/packages/amqp/lib/AbstractAmqpService.ts b/packages/amqp/lib/AbstractAmqpService.ts index 0480ca7e..4fa8243a 100644 --- a/packages/amqp/lib/AbstractAmqpService.ts +++ b/packages/amqp/lib/AbstractAmqpService.ts @@ -1,14 +1,12 @@ import type { QueueConsumerDependencies, QueueDependencies, - NewQueueOptions, - ExistingQueueOptions, + QueueOptions, } from '@message-queue-toolkit/core' import { AbstractQueueService } from '@message-queue-toolkit/core' import type { Channel, Connection, Message } from 'amqplib' import type { Options } from 'amqplib/properties' -import type { AMQPLocatorType } from './AbstractAmqpBaseConsumer' import type { AmqpConnectionManager, ConnectionReceiver } from './AmqpConnectionManager' import { deleteAmqp } from './utils/amqpInitter' @@ -19,13 +17,13 @@ export type AMQPDependencies = QueueDependencies & { export type AMQPConsumerDependencies = AMQPDependencies & QueueConsumerDependencies export type AMQPQueueConfig = Options.AssertQueue -export type CreateAMQPQueueOptions = { +export type AMQPCreationConfig = { queueOptions: AMQPQueueConfig queueName: string updateAttributesIfExists?: boolean } -export type AMQPQueueLocatorType = { +export type AMQPLocator = { queueName: string } @@ -34,18 +32,16 @@ export abstract class AbstractAmqpService< DependenciesType extends AMQPDependencies = AMQPDependencies, ExecutionContext = unknown, PrehandlerOutput = unknown, - BarrierOutput = unknown, > extends AbstractQueueService< MessagePayloadType, Message, DependenciesType, - CreateAMQPQueueOptions, - AMQPQueueLocatorType, - NewQueueOptions | ExistingQueueOptions, + AMQPCreationConfig, + AMQPLocator, + QueueOptions, ExecutionContext, - PrehandlerOutput, - BarrierOutput + PrehandlerOutput > implements ConnectionReceiver { @@ -58,7 +54,7 @@ export abstract class AbstractAmqpService< constructor( dependencies: DependenciesType, - options: NewQueueOptions | ExistingQueueOptions, + options: QueueOptions, ) { super(dependencies, options) diff --git a/packages/amqp/lib/types/MessageTypes.ts b/packages/amqp/lib/types/MessageTypes.ts deleted file mode 100644 index 9a6f9faf..00000000 --- a/packages/amqp/lib/types/MessageTypes.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type CommonMessage = { - messageType: string -} diff --git a/packages/amqp/lib/utils/amqpInitter.ts b/packages/amqp/lib/utils/amqpInitter.ts index d135c637..fb8def01 100644 --- a/packages/amqp/lib/utils/amqpInitter.ts +++ b/packages/amqp/lib/utils/amqpInitter.ts @@ -2,12 +2,12 @@ import type { DeletionConfig } from '@message-queue-toolkit/core' import { isProduction } from '@message-queue-toolkit/core' import type { Channel } from 'amqplib' -import type { CreateAMQPQueueOptions } from '../AbstractAmqpService' +import type { AMQPCreationConfig } from '../AbstractAmqpService' export async function deleteAmqp( channel: Channel, deletionConfig: DeletionConfig, - creationConfig: CreateAMQPQueueOptions, + creationConfig: AMQPCreationConfig, ) { if (!deletionConfig.deleteIfExists) { return diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 57875d4b..1a9e2a11 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "11.1.0", + "version": "12.0.0", "private": false, "license": "MIT", "description": "AMQP adapter for message-queue-toolkit", diff --git a/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts b/packages/amqp/test/consumers/AmqpPermissionConsumer.spec.ts similarity index 70% rename from packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts rename to packages/amqp/test/consumers/AmqpPermissionConsumer.spec.ts index ce08004b..e8a0ad4c 100644 --- a/packages/amqp/test/consumers/AmqpPermissionsConsumerMultiSchema.spec.ts +++ b/packages/amqp/test/consumers/AmqpPermissionConsumer.spec.ts @@ -1,33 +1,35 @@ -import { waitAndRetry } from '@message-queue-toolkit/core' +import { objectToBuffer, waitAndRetry } from '@message-queue-toolkit/core' +import type { Channel } from 'amqplib' import type { AwilixContainer } from 'awilix' import { asClass, asFunction } from 'awilix' import { describe, beforeEach, afterEach, expect, it } from 'vitest' +import { ZodError } from 'zod' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' import { FakeLogger } from '../fakes/FakeLogger' -import type { AmqpPermissionPublisherMultiSchema } from '../publishers/AmqpPermissionPublisherMultiSchema' +import type { AmqpPermissionPublisher } from '../publishers/AmqpPermissionPublisher' import { TEST_AMQP_CONFIG } from '../utils/testAmqpConfig' import type { Dependencies } from '../utils/testContext' import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' -import { AmqpPermissionConsumerMultiSchema } from './AmqpPermissionConsumerMultiSchema' +import { AmqpPermissionConsumer } from './AmqpPermissionConsumer' -describe('PermissionsConsumerMultiSchema', () => { +describe('AmqpPermissionConsumer', () => { describe('logging', () => { let logger: FakeLogger let diContainer: AwilixContainer - let publisher: AmqpPermissionPublisherMultiSchema + let publisher: AmqpPermissionPublisher beforeAll(async () => { logger = new FakeLogger() diContainer = await registerDependencies(TEST_AMQP_CONFIG, { logger: asFunction(() => logger), }) - await diContainer.cradle.permissionConsumerMultiSchema.close() - publisher = diContainer.cradle.permissionPublisherMultiSchema + await diContainer.cradle.permissionConsumer.close() + publisher = diContainer.cradle.permissionPublisher }) it('logs a message when logging is enabled', async () => { - const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new AmqpPermissionConsumer(diContainer.cradle, { logMessages: true, }) await newConsumer.start() @@ -62,17 +64,17 @@ describe('PermissionsConsumerMultiSchema', () => { describe('preHandlerBarrier', () => { let diContainer: AwilixContainer - let publisher: AmqpPermissionPublisherMultiSchema + let publisher: AmqpPermissionPublisher beforeAll(async () => { diContainer = await registerDependencies(TEST_AMQP_CONFIG) - await diContainer.cradle.permissionConsumerMultiSchema.close() - publisher = diContainer.cradle.permissionPublisherMultiSchema + await diContainer.cradle.permissionConsumer.close() + publisher = diContainer.cradle.permissionPublisher }) it('blocks first try', async () => { let barrierCounter = 0 - const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new AmqpPermissionConsumer(diContainer.cradle, { addPreHandlerBarrier: (_msg) => { barrierCounter++ if (barrierCounter < 2) { @@ -103,7 +105,7 @@ describe('PermissionsConsumerMultiSchema', () => { it('throws an error on first try', async () => { let barrierCounter = 0 - const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new AmqpPermissionConsumer(diContainer.cradle, { addPreHandlerBarrier: (_msg) => { barrierCounter++ if (barrierCounter === 1) { @@ -132,10 +134,10 @@ describe('PermissionsConsumerMultiSchema', () => { describe('prehandlers', () => { let diContainer: AwilixContainer - let publisher: AmqpPermissionPublisherMultiSchema + let publisher: AmqpPermissionPublisher beforeEach(async () => { diContainer = await registerDependencies(TEST_AMQP_CONFIG, undefined, false) - publisher = diContainer.cradle.permissionPublisherMultiSchema + publisher = diContainer.cradle.permissionPublisher await publisher.init() }) @@ -147,7 +149,7 @@ describe('PermissionsConsumerMultiSchema', () => { it('processes one prehandler', async () => { expect.assertions(1) - const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new AmqpPermissionConsumer(diContainer.cradle, { removeHandlerOverride: async (message, _context, prehandlerOutputs) => { expect(prehandlerOutputs.prehandlerOutput.prehandlerCount).toBe(1) return { @@ -180,7 +182,7 @@ describe('PermissionsConsumerMultiSchema', () => { it('processes two prehandlers', async () => { expect.assertions(1) - const newConsumer = new AmqpPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new AmqpPermissionConsumer(diContainer.cradle, { removeHandlerOverride: async (message, _context, prehandlerOutputs) => { expect(prehandlerOutputs.prehandlerOutput.prehandlerCount).toBe(11) return { @@ -222,24 +224,73 @@ describe('PermissionsConsumerMultiSchema', () => { describe('consume', () => { let diContainer: AwilixContainer - let publisher: AmqpPermissionPublisherMultiSchema - let consumer: AmqpPermissionConsumerMultiSchema + let consumer: AmqpPermissionConsumer + let publisher: AmqpPermissionPublisher + let channel: Channel + let consumerErrorResolver: FakeConsumerErrorResolver beforeEach(async () => { diContainer = await registerDependencies(TEST_AMQP_CONFIG, { consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), }) - publisher = diContainer.cradle.permissionPublisherMultiSchema - consumer = diContainer.cradle.permissionConsumerMultiSchema + publisher = diContainer.cradle.permissionPublisher + consumer = diContainer.cradle.permissionConsumer + consumerErrorResolver = diContainer.cradle.consumerErrorResolver as FakeConsumerErrorResolver + channel = await ( + await diContainer.cradle.amqpConnectionManager.getConnection() + ).createChannel() }) afterEach(async () => { const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() await diContainer.dispose() }) + it('Invalid message in the queue', async () => { + channel.sendToQueue( + AmqpPermissionConsumer.QUEUE_NAME, + objectToBuffer({ + id: 1, // invalid type + messageType: 'add', + }), + ) + + await waitAndRetry(() => consumerErrorResolver.errors.length > 0) + + expect(consumerErrorResolver.errors).toHaveLength(1) + expect(consumerErrorResolver.errors[0] instanceof ZodError).toBe(true) + }) + + it('message with invalid message type', async () => { + const errorReporterSpy = vi.spyOn(diContainer.cradle.errorReporter, 'report') + channel.sendToQueue( + AmqpPermissionConsumer.QUEUE_NAME, + objectToBuffer({ + id: '1', + messageType: 'bad', + }), + ) + + await waitAndRetry(() => errorReporterSpy.mock.calls.length > 0) + + expect(errorReporterSpy.mock.calls).toHaveLength(1) + expect(errorReporterSpy.mock.calls[0][0].error).toMatchObject({ + message: 'Unsupported message type: bad', + }) + }) + + it('message in the queue is not JSON', async () => { + channel.sendToQueue(AmqpPermissionConsumer.QUEUE_NAME, Buffer.from('not a JSON')) + + await waitAndRetry(() => consumerErrorResolver.errors.length > 0) + + expect(consumerErrorResolver.errors.length).toBeGreaterThan(0) + expect((consumerErrorResolver.errors[0] as Error).message).toContain('Unexpected token') + }) + it('Processes messages', async () => { publisher.publish({ id: '10', diff --git a/packages/amqp/test/consumers/AmqpPermissionConsumer.ts b/packages/amqp/test/consumers/AmqpPermissionConsumer.ts index 15956ffe..47784db6 100644 --- a/packages/amqp/test/consumers/AmqpPermissionConsumer.ts +++ b/packages/amqp/test/consumers/AmqpPermissionConsumer.ts @@ -1,56 +1,115 @@ import type { Either } from '@lokalise/node-core' +import type { BarrierResult, Prehandler, PrehandlingOutputs } from '@message-queue-toolkit/core' +import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' -import { AbstractAmqpConsumerMonoSchema } from '../../lib/AbstractAmqpConsumerMonoSchema' +import type { AMQPConsumerOptions } from '../../lib/AbstractAmqpConsumer' +import { AbstractAmqpConsumer } from '../../lib/AbstractAmqpConsumer' import type { AMQPConsumerDependencies } from '../../lib/AbstractAmqpService' -import { userPermissionMap } from '../repositories/PermissionRepository' -import type { PERMISSIONS_MESSAGE_TYPE } from './userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from './userConsumerSchemas' +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from './userConsumerSchemas' +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from './userConsumerSchemas' -export class AmqpPermissionConsumer extends AbstractAmqpConsumerMonoSchema { - public static QUEUE_NAME = 'user_permissions' +type SupportedEvents = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE +type ExecutionContext = { + incrementAmount: number +} +type PrehandlerOutput = { + prehandlerCount: number +} - constructor(dependencies: AMQPConsumerDependencies) { - super(dependencies, { - creationConfig: { - queueName: AmqpPermissionConsumer.QUEUE_NAME, - queueOptions: { - durable: true, - autoDelete: false, - }, - }, - deletionConfig: { - deleteIfExists: true, - }, - messageSchema: PERMISSIONS_MESSAGE_SCHEMA, - messageTypeField: 'messageType', - }) - } +type AmqpPermissionConsumerOptions = Pick< + AMQPConsumerOptions, + 'creationConfig' | 'locatorConfig' | 'logMessages' +> & { + addPreHandlerBarrier?: (message: SupportedEvents) => Promise> + removeHandlerOverride?: ( + _message: SupportedEvents, + context: ExecutionContext, + prehandlingOutputs: PrehandlingOutputs, + ) => Promise> + removePreHandlers?: Prehandler[] +} - override async processMessage( - message: PERMISSIONS_MESSAGE_TYPE, - ): Promise> { - const matchedUserPermissions = message.userIds.reduce((acc, userId) => { - if (userPermissionMap[userId]) { - acc.push(userPermissionMap[userId]) - } - return acc - }, [] as string[][]) +export class AmqpPermissionConsumer extends AbstractAmqpConsumer< + SupportedEvents, + ExecutionContext, + PrehandlerOutput +> { + public static readonly QUEUE_NAME = 'user_permissions_multi' - if (!matchedUserPermissions || matchedUserPermissions.length < message.userIds.length) { - // not all users were already created, we need to wait to be able to set permissions + public addCounter = 0 + public removeCounter = 0 + + constructor(dependencies: AMQPConsumerDependencies, options?: AmqpPermissionConsumerOptions) { + const defaultRemoveHandler = async ( + _message: SupportedEvents, + context: ExecutionContext, + _prehandlingOutputs: PrehandlingOutputs, + ): Promise> => { + this.removeCounter += context.incrementAmount return { - error: 'retryLater', + result: 'success', } } - // Do not do this in production, some kind of bulk insertion is needed here - for (const userPermissions of matchedUserPermissions) { - userPermissions.push(...message.permissions) - } - - return { - result: 'success', - } + super( + dependencies, + { + ...(options?.locatorConfig + ? { locatorConfig: options.locatorConfig } + : { + creationConfig: options?.creationConfig ?? { + queueName: AmqpPermissionConsumer.QUEUE_NAME, + queueOptions: { + durable: true, + autoDelete: false, + }, + }, + }), + logMessages: options?.logMessages, + handlerSpy: true, + messageTypeField: 'messageType', + deletionConfig: { + deleteIfExists: true, + }, + handlers: new MessageHandlerConfigBuilder< + SupportedEvents, + ExecutionContext, + PrehandlerOutput + >() + .addConfig( + PERMISSIONS_ADD_MESSAGE_SCHEMA, + async (_message, context, barrierOutput) => { + if (options?.addPreHandlerBarrier && !barrierOutput) { + throw new Error('barrier is not working') + } + this.addCounter += context.incrementAmount + return { + result: 'success', + } + }, + { + preHandlerBarrier: options?.addPreHandlerBarrier, + }, + ) + .addConfig( + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, + options?.removeHandlerOverride ?? defaultRemoveHandler, + { + prehandlers: options?.removePreHandlers, + }, + ) + .build(), + }, + { + incrementAmount: 1, + }, + ) } } diff --git a/packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts b/packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts deleted file mode 100644 index 4d687f59..00000000 --- a/packages/amqp/test/consumers/AmqpPermissionConsumerMultiSchema.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { BarrierResult, Prehandler, PrehandlingOutputs } from '@message-queue-toolkit/core' -import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' - -import type { NewAMQPConsumerOptions } from '../../lib/AbstractAmqpBaseConsumer' -import { AbstractAmqpConsumerMultiSchema } from '../../lib/AbstractAmqpConsumerMultiSchema' -import type { AMQPConsumerDependencies } from '../../lib/AbstractAmqpService' - -import type { - PERMISSIONS_ADD_MESSAGE_TYPE, - PERMISSIONS_REMOVE_MESSAGE_TYPE, -} from './userConsumerSchemas' -import { - PERMISSIONS_ADD_MESSAGE_SCHEMA, - PERMISSIONS_REMOVE_MESSAGE_SCHEMA, -} from './userConsumerSchemas' - -type SupportedEvents = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE -type ExecutionContext = { - incrementAmount: number -} -type PrehandlerOutput = { - prehandlerCount: number -} - -export class AmqpPermissionConsumerMultiSchema extends AbstractAmqpConsumerMultiSchema< - SupportedEvents, - ExecutionContext, - PrehandlerOutput -> { - public static QUEUE_NAME = 'user_permissions_multi' - - public addCounter = 0 - public removeCounter = 0 - - constructor( - dependencies: AMQPConsumerDependencies, - options: Partial< - NewAMQPConsumerOptions - > & { - addPreHandlerBarrier?: (message: SupportedEvents) => Promise> - removeHandlerOverride?: ( - _message: SupportedEvents, - context: ExecutionContext, - prehandlingOutputs: PrehandlingOutputs, - ) => Promise> - removePreHandlers?: Prehandler[] - }, - ) { - const defaultRemoveHandler = async ( - _message: SupportedEvents, - context: ExecutionContext, - _prehandlingOutputs: PrehandlingOutputs, - ): Promise> => { - this.removeCounter += context.incrementAmount - return { - result: 'success', - } - } - - super( - dependencies, - { - handlerSpy: true, - creationConfig: { - queueName: AmqpPermissionConsumerMultiSchema.QUEUE_NAME, - queueOptions: { - durable: true, - autoDelete: false, - }, - }, - deletionConfig: { - deleteIfExists: true, - }, - handlers: new MessageHandlerConfigBuilder< - SupportedEvents, - ExecutionContext, - PrehandlerOutput - >() - .addConfig( - PERMISSIONS_ADD_MESSAGE_SCHEMA, - async (_message, context, barrierOutput) => { - if (options?.addPreHandlerBarrier && !barrierOutput) { - return { error: 'retryLater' } - } - this.addCounter += context.incrementAmount - return { - result: 'success', - } - }, - { - preHandlerBarrier: options?.addPreHandlerBarrier, - }, - ) - .addConfig( - PERMISSIONS_REMOVE_MESSAGE_SCHEMA, - options?.removeHandlerOverride ?? defaultRemoveHandler, - { - prehandlers: options?.removePreHandlers, - }, - ) - .build(), - messageTypeField: 'messageType', - ...options, - }, - { - incrementAmount: 1, - }, - ) - } -} diff --git a/packages/amqp/test/consumers/AmqpPermissionsConsumer.spec.ts b/packages/amqp/test/consumers/AmqpPermissionsConsumer.spec.ts deleted file mode 100644 index c988dcc5..00000000 --- a/packages/amqp/test/consumers/AmqpPermissionsConsumer.spec.ts +++ /dev/null @@ -1,243 +0,0 @@ -import type { Channel } from 'amqplib' -import type { AwilixContainer } from 'awilix' -import { asClass } from 'awilix' -import { describe, beforeEach, afterEach, expect, it } from 'vitest' - -import { objectToBuffer } from '../../../core/lib/utils/queueUtils' -import { waitAndRetry } from '../../../core/lib/utils/waitUtils' -import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' -import type { AmqpPermissionPublisher } from '../publishers/AmqpPermissionPublisher' -import { userPermissionMap } from '../repositories/PermissionRepository' -import { TEST_AMQP_CONFIG } from '../utils/testAmqpConfig' -import type { Dependencies } from '../utils/testContext' -import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' - -import { AmqpPermissionConsumer } from './AmqpPermissionConsumer' -import type { PERMISSIONS_MESSAGE_TYPE } from './userConsumerSchemas' - -const userIds = [100, 200, 300] -const perms: [string, ...string[]] = ['perm1', 'perm2'] - -function checkPermissions(userIds: number[]) { - const usersPerms = userIds.reduce((acc, userId) => { - if (userPermissionMap[userId]) { - acc.push(userPermissionMap[userId]) - } - return acc - }, [] as string[][]) - - if (usersPerms && usersPerms.length !== userIds.length) { - return null - } - - for (const userPerms of usersPerms) - if (userPerms.length !== perms.length) { - return null - } - - return usersPerms -} - -async function waitForPermissions(userIds: number[]) { - return await waitAndRetry(async () => { - return checkPermissions(userIds) - }) -} - -describe('PermissionsConsumer', () => { - describe('consume', () => { - let diContainer: AwilixContainer - let channel: Channel - let publisher: AmqpPermissionPublisher - let consumer: AmqpPermissionConsumer - beforeEach(async () => { - delete userPermissionMap[100] - delete userPermissionMap[200] - delete userPermissionMap[300] - diContainer = await registerDependencies(TEST_AMQP_CONFIG, { - consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), - }) - - channel = await diContainer.cradle.amqpConnectionManager.getConnectionSync()!.createChannel() - publisher = diContainer.cradle.permissionPublisher - consumer = diContainer.cradle.permissionConsumer - await diContainer.cradle.permissionConsumer.start() - }) - - afterEach(async () => { - const connection = await diContainer.cradle.amqpConnectionManager.getConnection() - channel = await connection.createChannel() - - await channel.deleteQueue(AmqpPermissionConsumer.QUEUE_NAME) - await channel.close() - const { awilixManager } = diContainer.cradle - await awilixManager.executeDispose() - await diContainer.dispose() - }) - - it('Creates permissions', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - void channel.sendToQueue( - AmqpPermissionConsumer.QUEUE_NAME, - objectToBuffer({ - id: '1', - messageType: 'add', - userIds, - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE), - ) - - const updatedUsersPermissions = await waitForPermissions(userIds) - - if (null === updatedUsersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(updatedUsersPermissions).toBeDefined() - expect(updatedUsersPermissions[0]).toHaveLength(2) - }) - - it('Reconnects if connection is lost', async () => { - await (await diContainer.cradle.amqpConnectionManager.getConnection()).close() - - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - publisher.publish({ - id: '2', - messageType: 'add', - userIds, - permissions: perms, - }) - - const updatedUsersPermissions = await waitAndRetry(() => { - const checkResult = checkPermissions(userIds) - if (checkResult) { - return checkResult - } - - publisher.publish({ - id: '3', - messageType: 'add', - userIds, - permissions: perms, - }) - return null - }, 50) - - if (null === updatedUsersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(updatedUsersPermissions).toBeDefined() - expect(updatedUsersPermissions[0]).toHaveLength(2) - - await publisher.close() - await consumer.close() - }) - - it('Wait for users to be created and then create permissions', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - channel.sendToQueue( - AmqpPermissionConsumer.QUEUE_NAME, - objectToBuffer({ - id: '4', - userIds, - messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE), - ) - - // no users in the database, so message will go back to the queue - const usersFromDb = await waitForPermissions(userIds) - expect(usersFromDb).toBeNull() - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - const usersPermissions = await waitForPermissions(userIds) - - if (null === usersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(usersPermissions).toBeDefined() - expect(usersPermissions[0]).toHaveLength(2) - }) - - it('Not all users exist, no permissions were created', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - - channel.sendToQueue( - AmqpPermissionConsumer.QUEUE_NAME, - objectToBuffer({ - id: '5', - userIds, - messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE), - ) - - // not all users are in the database, so message will go back to the queue - const usersFromDb = await waitForPermissions(userIds) - expect(usersFromDb).toBeNull() - - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - const usersPermissions = await waitForPermissions(userIds) - - if (null === usersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(usersPermissions).toBeDefined() - expect(usersPermissions[0]).toHaveLength(2) - }) - - it('Invalid message in the queue', async () => { - const { consumerErrorResolver } = diContainer.cradle - - channel.sendToQueue( - AmqpPermissionConsumer.QUEUE_NAME, - objectToBuffer({ - id: '6', - messageType: 'add', - permissions: perms, - } as PERMISSIONS_MESSAGE_TYPE), - ) - - const fakeResolver = consumerErrorResolver as FakeConsumerErrorResolver - await waitAndRetry(() => fakeResolver.handleErrorCallsCount) - - expect(fakeResolver.handleErrorCallsCount).toBe(1) - }) - - it('Non-JSON message in the queue', async () => { - const { consumerErrorResolver } = diContainer.cradle - - channel.sendToQueue(AmqpPermissionConsumer.QUEUE_NAME, Buffer.from('dummy')) - - const fakeResolver = consumerErrorResolver as FakeConsumerErrorResolver - await waitAndRetry(() => fakeResolver.handleErrorCallsCount) - - expect(fakeResolver.handleErrorCallsCount).toBe(2) - }) - }) -}) diff --git a/packages/amqp/test/fakes/FakeConsumer.ts b/packages/amqp/test/fakes/FakeConsumer.ts index 4e1f4449..d02b7cb6 100644 --- a/packages/amqp/test/fakes/FakeConsumer.ts +++ b/packages/amqp/test/fakes/FakeConsumer.ts @@ -1,28 +1,32 @@ -import type { Either } from '@lokalise/node-core' -import type { ZodType } from 'zod' +import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' +import z from 'zod' -import { AbstractAmqpConsumerMonoSchema } from '../../lib/AbstractAmqpConsumerMonoSchema' +import { AbstractAmqpConsumer } from '../../lib/AbstractAmqpConsumer' import type { AMQPConsumerDependencies } from '../../lib/AbstractAmqpService' -import type { CommonMessage } from '../../lib/types/MessageTypes' -export class FakeConsumer extends AbstractAmqpConsumerMonoSchema { - constructor(dependencies: AMQPConsumerDependencies, queueName = 'dummy', messageSchema: ZodType) { - super(dependencies, { - creationConfig: { - queueName: queueName, - queueOptions: { - durable: true, - autoDelete: false, +export const COMMON_MESSAGE_SCHEMA = z.object({ + messageType: z.string(), +}) + +export type CommonMessage = z.infer +export class FakeConsumer extends AbstractAmqpConsumer { + constructor(dependencies: AMQPConsumerDependencies) { + super( + dependencies, + { + creationConfig: { + queueName: 'dummy', + queueOptions: { + durable: true, + autoDelete: false, + }, }, + messageTypeField: 'messageType', + handlers: new MessageHandlerConfigBuilder() + .addConfig(COMMON_MESSAGE_SCHEMA, () => Promise.resolve({ result: 'success' })) + .build(), }, - messageSchema, - messageTypeField: 'messageType', - }) - } - - processMessage(): Promise> { - return Promise.resolve({ - result: 'success', - }) + {}, + ) } } diff --git a/packages/amqp/test/fakes/FakeConsumerErrorResolver.ts b/packages/amqp/test/fakes/FakeConsumerErrorResolver.ts index a8564b9d..78f65dae 100644 --- a/packages/amqp/test/fakes/FakeConsumerErrorResolver.ts +++ b/packages/amqp/test/fakes/FakeConsumerErrorResolver.ts @@ -1,19 +1,23 @@ import { AmqpConsumerErrorResolver } from '../../lib/errors/AmqpConsumerErrorResolver' export class FakeConsumerErrorResolver extends AmqpConsumerErrorResolver { - public handleErrorCallsCount: number + private _errors: unknown[] constructor() { super() - this.handleErrorCallsCount = 0 + this._errors = [] } public override processError(error: unknown) { - this.handleErrorCallsCount++ + this._errors.push(error) return super.processError(error) } public clear() { - this.handleErrorCallsCount = 0 + this._errors = [] + } + + get errors() { + return this._errors } } diff --git a/packages/amqp/test/fakes/FakeConsumerMultiSchema.ts b/packages/amqp/test/fakes/FakeConsumerMultiSchema.ts deleted file mode 100644 index 2b0400cc..00000000 --- a/packages/amqp/test/fakes/FakeConsumerMultiSchema.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { MessageHandlerConfig } from '@message-queue-toolkit/core' - -import { AbstractAmqpConsumerMultiSchema } from '../../lib/AbstractAmqpConsumerMultiSchema' -import type { AMQPConsumerDependencies } from '../../lib/AbstractAmqpService' -import type { CommonMessage } from '../../lib/types/MessageTypes' - -export class FakeConsumerMultiSchema extends AbstractAmqpConsumerMultiSchema< - CommonMessage, - ExecutionContext -> { - constructor( - dependencies: AMQPConsumerDependencies, - queueName = 'dummy', - handlers: MessageHandlerConfig[], - executionContext: ExecutionContext, - ) { - super( - dependencies, - { - creationConfig: { - queueName: queueName, - queueOptions: { - durable: true, - autoDelete: false, - }, - }, - deletionConfig: { - deleteIfExists: true, - }, - handlers, - messageTypeField: 'messageType', - }, - executionContext, - ) - } - - processMessage(): Promise> { - return Promise.resolve({ - result: 'success', - }) - } -} diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts index 682f3902..c31d5400 100644 --- a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts +++ b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts @@ -1,72 +1,42 @@ +import { waitAndRetry } from '@lokalise/node-core' import type { Channel } from 'amqplib' import type { AwilixContainer } from 'awilix' import { asClass, asFunction, Lifetime } from 'awilix' import { describe, beforeAll, beforeEach, afterAll, afterEach, expect, it } from 'vitest' +import { ZodError } from 'zod' -import { waitAndRetry } from '../../../core/lib/utils/waitUtils' import { deserializeAmqpMessage } from '../../lib/amqpMessageDeserializer' import { AmqpPermissionConsumer } from '../consumers/AmqpPermissionConsumer' -import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' +import { PERMISSIONS_ADD_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' import { FakeConsumer } from '../fakes/FakeConsumer' import { FakeConsumerErrorResolver } from '../fakes/FakeConsumerErrorResolver' import { FakeLogger } from '../fakes/FakeLogger' -import { userPermissionMap } from '../repositories/PermissionRepository' import { TEST_AMQP_CONFIG } from '../utils/testAmqpConfig' import type { Dependencies } from '../utils/testContext' import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' import { AmqpPermissionPublisher } from './AmqpPermissionPublisher' -import type { AmqpPermissionPublisherMultiSchema } from './AmqpPermissionPublisherMultiSchema' - -const perms: [string, ...string[]] = ['perm1', 'perm2'] -const userIds = [100, 200, 300] - -function checkPermissions(userIds: number[]) { - const usersPerms = userIds.reduce((acc, userId) => { - if (userPermissionMap[userId]) { - acc.push(userPermissionMap[userId]) - } - return acc - }, [] as string[][]) - - if (usersPerms.length > userIds.length) { - return usersPerms.slice(0, userIds.length - 1) - } - - if (usersPerms && usersPerms.length !== userIds.length) { - return null - } - - for (const userPerms of usersPerms) - if (userPerms.length !== perms.length) { - return null - } - - return usersPerms -} describe('PermissionPublisher', () => { describe('logging', () => { let logger: FakeLogger let diContainer: AwilixContainer - let publisher: AmqpPermissionPublisherMultiSchema + let publisher: AmqpPermissionPublisher beforeAll(async () => { logger = new FakeLogger() diContainer = await registerDependencies(TEST_AMQP_CONFIG, { logger: asFunction(() => logger), }) - await diContainer.cradle.permissionConsumerMultiSchema.close() - publisher = diContainer.cradle.permissionPublisherMultiSchema + await diContainer.cradle.permissionConsumer.close() + publisher = diContainer.cradle.permissionPublisher }) it('logs a message when logging is enabled', async () => { const message = { id: '1', - userIds, messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE + } satisfies PERMISSIONS_ADD_MESSAGE_TYPE publisher.publish(message) @@ -77,8 +47,6 @@ describe('PermissionPublisher', () => { expect(logger.loggedMessages[1]).toEqual({ id: '1', messageType: 'add', - permissions: ['perm1', 'perm2'], - userIds: [100, 200, 300], }) }) }) @@ -135,21 +103,27 @@ describe('PermissionPublisher', () => { describe('publish', () => { let diContainer: AwilixContainer let channel: Channel + let permissionPublisher: AmqpPermissionPublisher + let permissionConsumer: AmqpPermissionConsumer + beforeAll(async () => { diContainer = await registerDependencies(TEST_AMQP_CONFIG, { consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), }) + permissionPublisher = diContainer.cradle.permissionPublisher + permissionConsumer = diContainer.cradle.permissionConsumer }) beforeEach(async () => { const connection = await diContainer.cradle.amqpConnectionManager.getConnection() channel = await connection.createChannel() + await permissionConsumer.start() }) afterEach(async () => { const connection = await diContainer.cradle.amqpConnectionManager.getConnection() channel = await connection.createChannel() - await channel.deleteQueue(AmqpPermissionConsumer.QUEUE_NAME) + await channel.deleteQueue(AmqpPermissionPublisher.QUEUE_NAME) await channel.close() }) @@ -159,24 +133,55 @@ describe('PermissionPublisher', () => { await diContainer.dispose() }) + it('publish unexpected message', async () => { + let error: unknown + try { + permissionPublisher.publish({ + hello: 'world', + messageType: 'add', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any) + } catch (e) { + error = e + } + expect(error).toBeDefined() + expect(error).toBeInstanceOf(Error) + expect(error).toBeInstanceOf(ZodError) + }) + + it('publish message with uns Unsupported message type', async () => { + let error: unknown + try { + permissionPublisher.publish({ + id: '124', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messageType: 'bad' as any, + }) + } catch (e) { + error = e + } + console.log(error) + expect(error).toBeDefined() + expect(error).toBeInstanceOf(Error) + expect((error as Error).message).toBe('Unsupported message type: bad') + }) + it('publishes a message', async () => { - const { permissionPublisher } = diContainer.cradle + await permissionConsumer.close() const message = { id: '2', - userIds, messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE + } satisfies PERMISSIONS_ADD_MESSAGE_TYPE - let receivedMessage: PERMISSIONS_MESSAGE_TYPE | null = null + let receivedMessage: PERMISSIONS_ADD_MESSAGE_TYPE | null = null await channel.consume(AmqpPermissionPublisher.QUEUE_NAME, (message) => { if (message === null) { return } const decodedMessage = deserializeAmqpMessage( message, - PERMISSIONS_MESSAGE_SCHEMA, + PERMISSIONS_ADD_MESSAGE_SCHEMA, new FakeConsumerErrorResolver(), ) receivedMessage = decodedMessage.result! @@ -191,28 +196,14 @@ describe('PermissionPublisher', () => { expect(receivedMessage).toEqual({ id: '2', messageType: 'add', - permissions: ['perm1', 'perm2'], - userIds: [100, 200, 300], }) }) it('reconnects on lost connection', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - const { permissionPublisher, permissionConsumer } = diContainer.cradle - await permissionConsumer.start() - const message = { - id: '3', - userIds, + id: '4', messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE + } satisfies PERMISSIONS_ADD_MESSAGE_TYPE await diContainer.cradle.amqpConnectionManager.getConnectionSync()!.close() @@ -220,7 +211,7 @@ describe('PermissionPublisher', () => { () => { permissionPublisher.publish(message) - return checkPermissions(userIds) + return permissionConsumer.addCounter > 0 }, 100, 20, @@ -230,8 +221,7 @@ describe('PermissionPublisher', () => { throw new Error('Users permissions unexpectedly null') } - expect(updatedUsersPermissions).toBeDefined() - expect(updatedUsersPermissions[0]).toHaveLength(2) + expect(permissionConsumer.addCounter).toBeGreaterThan(0) }) }) }) diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisher.ts b/packages/amqp/test/publishers/AmqpPermissionPublisher.ts index f67bcc2f..3cbf59dc 100644 --- a/packages/amqp/test/publishers/AmqpPermissionPublisher.ts +++ b/packages/amqp/test/publishers/AmqpPermissionPublisher.ts @@ -1,20 +1,26 @@ -import type { - ExistingAMQPConsumerOptions, - NewAMQPConsumerOptions, -} from '../../lib/AbstractAmqpBaseConsumer' -import { AbstractAmqpPublisherMonoSchema } from '../../lib/AbstractAmqpPublisherMonoSchema' +import type { AMQPPublisherOptions } from '../../lib/AbstractAmqpPublisher' +import { AbstractAmqpPublisher } from '../../lib/AbstractAmqpPublisher' import type { AMQPDependencies } from '../../lib/AbstractAmqpService' -import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from '../consumers/userConsumerSchemas' +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from '../consumers/userConsumerSchemas' + +type SupportedTypes = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE -export class AmqpPermissionPublisher extends AbstractAmqpPublisherMonoSchema { - public static QUEUE_NAME = 'user_permissions' +export class AmqpPermissionPublisher extends AbstractAmqpPublisher { + public static QUEUE_NAME = 'user_permissions_multi' constructor( dependencies: AMQPDependencies, - options: - | Pick, 'creationConfig'> - | Pick, 'locatorConfig'> = { + options: Pick< + AMQPPublisherOptions, + 'creationConfig' | 'logMessages' | 'locatorConfig' + > = { creationConfig: { queueName: AmqpPermissionPublisher.QUEUE_NAME, queueOptions: { @@ -25,9 +31,20 @@ export class AmqpPermissionPublisher extends AbstractAmqpPublisherMonoSchema { - public static QUEUE_NAME = 'user_permissions_multi' - - constructor( - dependencies: AMQPDependencies, - options: - | Pick, 'creationConfig' | 'logMessages'> - | Pick, 'locatorConfig' | 'logMessages'> = { - creationConfig: { - queueName: AmqpPermissionPublisherMultiSchema.QUEUE_NAME, - queueOptions: { - durable: true, - autoDelete: false, - }, - }, - }, - ) { - super(dependencies, { - messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], - messageTypeField: 'messageType', - logMessages: true, - ...options, - }) - } -} diff --git a/packages/amqp/test/repositories/PermissionRepository.ts b/packages/amqp/test/repositories/PermissionRepository.ts deleted file mode 100644 index 89b09f10..00000000 --- a/packages/amqp/test/repositories/PermissionRepository.ts +++ /dev/null @@ -1 +0,0 @@ -export const userPermissionMap: Record = {} diff --git a/packages/amqp/test/utils/testContext.ts b/packages/amqp/test/utils/testContext.ts index 6b61bee7..79602577 100644 --- a/packages/amqp/test/utils/testContext.ts +++ b/packages/amqp/test/utils/testContext.ts @@ -8,9 +8,7 @@ import { AmqpConnectionManager } from '../../lib/AmqpConnectionManager' import type { AmqpConfig } from '../../lib/amqpConnectionResolver' import { AmqpConsumerErrorResolver } from '../../lib/errors/AmqpConsumerErrorResolver' import { AmqpPermissionConsumer } from '../consumers/AmqpPermissionConsumer' -import { AmqpPermissionConsumerMultiSchema } from '../consumers/AmqpPermissionConsumerMultiSchema' import { AmqpPermissionPublisher } from '../publishers/AmqpPermissionPublisher' -import { AmqpPermissionPublisherMultiSchema } from '../publishers/AmqpPermissionPublisherMultiSchema' export const SINGLETON_CONFIG = { lifetime: Lifetime.SINGLETON } @@ -71,21 +69,6 @@ export async function registerDependencies( enabled: queuesEnabled, }), - permissionConsumerMultiSchema: asClass(AmqpPermissionConsumerMultiSchema, { - lifetime: Lifetime.SINGLETON, - asyncInit: 'start', - asyncDispose: 'close', - asyncDisposePriority: 10, - enabled: queuesEnabled, - }), - permissionPublisherMultiSchema: asClass(AmqpPermissionPublisherMultiSchema, { - lifetime: Lifetime.SINGLETON, - asyncInit: 'init', - asyncDispose: 'close', - asyncDisposePriority: 20, - enabled: queuesEnabled, - }), - // vendor-specific dependencies transactionObservabilityManager: asFunction(() => { return undefined @@ -94,7 +77,7 @@ export async function registerDependencies( return { report: () => {}, } satisfies ErrorReporter - }), + }, SINGLETON_CONFIG), } diContainer.register(diConfig) @@ -122,6 +105,4 @@ export interface Dependencies { consumerErrorResolver: ErrorResolver permissionConsumer: AmqpPermissionConsumer permissionPublisher: AmqpPermissionPublisher - permissionConsumerMultiSchema: AmqpPermissionConsumerMultiSchema - permissionPublisherMultiSchema: AmqpPermissionPublisherMultiSchema } diff --git a/packages/core/index.ts b/packages/core/index.ts index 16cf5c68..061fe59c 100644 --- a/packages/core/index.ts +++ b/packages/core/index.ts @@ -9,21 +9,8 @@ export type { ExtraParams, } from './lib/types/MessageQueueTypes' -export { AbstractQueueService } from './lib/queues/AbstractQueueService' -export type { - NewQueueOptions, - ExistingQueueOptions, - NewQueueOptionsMultiSchema, - ExistingQueueOptionsMultiSchema, - MonoSchemaQueueOptions, - MultiSchemaConsumerOptions, - QueueDependencies, - QueueConsumerDependencies, - Deserializer, - CommonQueueLocator, - DeletionConfig, - MultiSchemaPublisherOptions, -} from './lib/queues/AbstractQueueService' +export { AbstractQueueService, Deserializer } from './lib/queues/AbstractQueueService' +export * from './lib/types/queueOptionsTypes' export { isMessageError, @@ -43,7 +30,7 @@ export { MessageHandlerConfigBuilder, } from './lib/queues/HandlerContainer' export type { - BarrierCallbackMultiConsumers, + BarrierCallback, BarrierResult, BarrierResultPositive, BarrierResultNegative, diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index bf3b0406..45baa77d 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -5,100 +5,26 @@ import { resolveGlobalErrorLogObject } from '@lokalise/node-core' import type { ZodSchema, ZodType } from 'zod' import type { MessageInvalidFormatError, MessageValidationError } from '../errors/Errors' -import type { - Logger, - TransactionObservabilityManager, - MessageProcessingResult, -} from '../types/MessageQueueTypes' +import type { Logger, MessageProcessingResult } from '../types/MessageQueueTypes' +import type { DeletionConfig, QueueDependencies, QueueOptions } from '../types/queueOptionsTypes' import type { + BarrierCallback, BarrierResult, - MessageHandlerConfig, Prehandler, PrehandlerResult, PrehandlingOutputs, } from './HandlerContainer' -import type { HandlerSpy, PublicHandlerSpy, HandlerSpyParams } from './HandlerSpy' +import type { HandlerSpy, PublicHandlerSpy } from './HandlerSpy' import { resolveHandlerSpy } from './HandlerSpy' -export type QueueDependencies = { - errorReporter: ErrorReporter - logger: Logger -} - -export type QueueConsumerDependencies = { - consumerErrorResolver: ErrorResolver - transactionObservabilityManager: TransactionObservabilityManager -} - export type Deserializer = ( message: unknown, type: ZodType, errorProcessor: ErrorResolver, ) => Either -export type NewQueueOptionsMultiSchema< - MessagePayloadSchemas extends object, - CreationConfigType extends object, - ExecutionContext, - PrehandlerOutput = undefined, -> = NewQueueOptions & - MultiSchemaConsumerOptions - -export type ExistingQueueOptionsMultiSchema< - MessagePayloadSchemas extends object, - QueueLocatorType extends object, - ExecutionContext, - PrehandlerOutput = undefined, -> = ExistingQueueOptions & - MultiSchemaConsumerOptions - -export type DeletionConfig = { - deleteIfExists?: boolean - waitForConfirmation?: boolean - forceDeleteInProduction?: boolean -} - -export type CommonQueueOptions = { - messageTypeField: string - messageIdField?: string - handlerSpy?: HandlerSpy | HandlerSpyParams | boolean - logMessages?: boolean -} - -export type CommonCreationConfigType = { - updateAttributesIfExists?: boolean -} - -export type NewQueueOptions = { - locatorConfig?: never - deletionConfig?: DeletionConfig - creationConfig: CreationConfigType -} & CommonQueueOptions - -export type ExistingQueueOptions = { - locatorConfig: QueueLocatorType - deletionConfig?: DeletionConfig - creationConfig?: never -} & CommonQueueOptions - -export type MultiSchemaPublisherOptions = { - messageSchemas: readonly ZodSchema[] -} - -export type MultiSchemaConsumerOptions< - MessagePayloadSchemas extends object, - ExecutionContext, - PrehandlerOutput = undefined, -> = { - handlers: MessageHandlerConfig[] -} - -export type MonoSchemaQueueOptions = { - messageSchema: ZodSchema -} - -export type CommonQueueLocator = { +type CommonQueueLocator = { queueName: string } @@ -108,14 +34,12 @@ export abstract class AbstractQueueService< DependenciesType extends QueueDependencies, QueueConfiguration extends object, QueueLocatorType extends object = CommonQueueLocator, - OptionsType extends - | NewQueueOptions - | ExistingQueueOptions = - | NewQueueOptions - | ExistingQueueOptions, + OptionsType extends QueueOptions = QueueOptions< + QueueConfiguration, + QueueLocatorType + >, ExecutionContext = undefined, PrehandlerOutput = undefined, - BarrierOutput = undefined, > { protected readonly errorReporter: ErrorReporter public readonly logger: Logger @@ -249,6 +173,26 @@ export abstract class AbstractQueueService< }) } + protected async preHandlerBarrierInternal( + barrier: + | BarrierCallback + | undefined, + message: MessagePayloadSchemas, + executionContext: ExecutionContext, + prehandlerOutput: PrehandlerOutput, + ): Promise> { + if (!barrier) { + // @ts-ignore + return { + isPassing: true, + output: undefined, + } + } + + // @ts-ignore + return await barrier(message, executionContext, prehandlerOutput) + } + protected abstract resolveNextFunction( prehandlers: Prehandler[], message: MessagePayloadSchemas, @@ -301,16 +245,17 @@ export abstract class AbstractQueueService< messageType: string, ): Promise - protected abstract preHandlerBarrier( + protected abstract preHandlerBarrier( message: MessagePayloadSchemas, messageType: string, prehandlerOutput: PrehandlerOutput, ): Promise> - abstract processMessage( + protected abstract processMessage( message: MessagePayloadSchemas, messageType: string, - prehandlingOutputs: PrehandlingOutputs, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prehandlingOutputs: PrehandlingOutputs, ): Promise> public abstract close(): Promise diff --git a/packages/core/lib/queues/HandlerContainer.ts b/packages/core/lib/queues/HandlerContainer.ts index e11c684a..765b4a05 100644 --- a/packages/core/lib/queues/HandlerContainer.ts +++ b/packages/core/lib/queues/HandlerContainer.ts @@ -27,7 +27,7 @@ export type BarrierResultNegative = { export type PrehandlerResult = Either -export type BarrierCallbackMultiConsumers< +export type BarrierCallback< MessagePayloadSchema extends object, ExecutionContext, PrehandlerOutput, @@ -54,7 +54,7 @@ export type HandlerConfigOptions< BarrierOutput, > = { messageLogFormatter?: LogFormatter - preHandlerBarrier?: BarrierCallbackMultiConsumers< + preHandlerBarrier?: BarrierCallback< MessagePayloadSchema, ExecutionContext, PrehandlerOutput, @@ -77,7 +77,7 @@ export class MessageHandlerConfig< BarrierOutput > public readonly messageLogFormatter: LogFormatter - public readonly preHandlerBarrier?: BarrierCallbackMultiConsumers< + public readonly preHandlerBarrier?: BarrierCallback< MessagePayloadSchema, ExecutionContext, PrehandlerOutput, diff --git a/packages/core/lib/queues/HandlerSpy.ts b/packages/core/lib/queues/HandlerSpy.ts index 89602097..7ece1949 100644 --- a/packages/core/lib/queues/HandlerSpy.ts +++ b/packages/core/lib/queues/HandlerSpy.ts @@ -4,10 +4,9 @@ import { isObject } from '@lokalise/node-core' import { Fifo } from 'toad-cache' import type { MessageProcessingResult } from '../types/MessageQueueTypes' +import type { CommonQueueOptions } from '../types/queueOptionsTypes' import { objectMatches } from '../utils/matchUtils' -import type { CommonQueueOptions } from './AbstractQueueService' - export type HandlerSpyParams = { bufferSize?: number messageIdField?: string diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts new file mode 100644 index 00000000..93578481 --- /dev/null +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -0,0 +1,70 @@ +import type { ErrorReporter, ErrorResolver } from '@lokalise/node-core' +import type { ZodSchema } from 'zod' + +import type { MessageHandlerConfig } from '../queues/HandlerContainer' +import type { HandlerSpy, HandlerSpyParams } from '../queues/HandlerSpy' + +import type { Logger, TransactionObservabilityManager } from './MessageQueueTypes' + +export type QueueDependencies = { + errorReporter: ErrorReporter + logger: Logger +} + +export type QueueConsumerDependencies = { + consumerErrorResolver: ErrorResolver + transactionObservabilityManager: TransactionObservabilityManager +} + +export type CommonQueueOptions = { + messageTypeField: string + messageIdField?: string + handlerSpy?: HandlerSpy | HandlerSpyParams | boolean + logMessages?: boolean +} + +type CommonCreationConfigType = { + updateAttributesIfExists?: boolean +} + +export type DeletionConfig = { + deleteIfExists?: boolean + waitForConfirmation?: boolean + forceDeleteInProduction?: boolean +} + +type NewQueueOptions = { + locatorConfig?: never + deletionConfig?: DeletionConfig + creationConfig: CreationConfigType +} & CommonQueueOptions + +type ExistingQueueOptions = { + locatorConfig: QueueLocatorType + deletionConfig?: DeletionConfig + creationConfig?: never +} & CommonQueueOptions + +export type QueueOptions< + CreationConfigType extends CommonCreationConfigType, + QueueLocatorType extends object, +> = CommonQueueOptions & + (NewQueueOptions | ExistingQueueOptions) + +export type QueuePublisherOptions< + CreationConfigType extends CommonCreationConfigType, + QueueLocatorType extends object, + MessagePayloadSchemas extends object, +> = QueueOptions & { + messageSchemas: readonly ZodSchema[] +} + +export type QueueConsumerOptions< + CreationConfigType extends object, + QueueLocatorType extends object, + MessagePayloadSchemas extends object, + ExecutionContext, + PrehandlerOutput = undefined, +> = QueueOptions & { + handlers: MessageHandlerConfig[] +} diff --git a/packages/core/package.json b/packages/core/package.json index 18f3525c..444fd520 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "9.1.0", + "version": "10.0.0", "private": false, "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", diff --git a/packages/sns/index.ts b/packages/sns/index.ts index d11b1ae2..e2ec8a0e 100644 --- a/packages/sns/index.ts +++ b/packages/sns/index.ts @@ -1,39 +1,22 @@ export type { SNSTopicAWSConfig, SNSTopicConfig, - SNSConsumerDependencies, SNSQueueLocatorType, SNSCreationConfig, SNSDependencies, - NewSNSOptions, - ExistingSNSOptions, - ExistingSNSOptionsMultiSchema, - NewSNSOptionsMultiSchema, } from './lib/sns/AbstractSnsService' export { AbstractSnsService } from './lib/sns/AbstractSnsService' export { SnsConsumerErrorResolver } from './lib/errors/SnsConsumerErrorResolver' -export { AbstractSnsPublisherMonoSchema } from './lib/sns/AbstractSnsPublisherMonoSchema' -export { AbstractSnsPublisherMultiSchema } from './lib/sns/AbstractSnsPublisherMultiSchema' +export { AbstractSnsPublisher } from './lib/sns/AbstractSnsPublisher' -export { AbstractSnsSqsConsumerMonoSchema } from './lib/sns/AbstractSnsSqsConsumerMonoSchema' -export { AbstractSnsSqsConsumerMultiSchema } from './lib/sns/AbstractSnsSqsConsumerMultiSchema' export type { - ExistingSnsSqsConsumerOptions, - NewSnsSqsConsumerOptions, + SNSSQSConsumerOptions, SNSSQSConsumerDependencies, - ExistingSnsSqsConsumerOptionsMono, - NewSnsSqsConsumerOptionsMono, - SNSSQSQueueLocatorType, -} from './lib/sns/AbstractSnsSqsConsumerMonoSchema' -export type { - NewSnsSqsConsumerOptionsMulti, - ExistingSnsSqsConsumerOptionsMulti, -} from './lib/sns/AbstractSnsSqsConsumerMultiSchema' - -export type { SNSMessageOptions } from './lib/sns/AbstractSnsPublisherMonoSchema' +} from './lib/sns/AbstractSnsSqsConsumer' +export { AbstractSnsSqsConsumer } from './lib/sns/AbstractSnsSqsConsumer' export type { CommonMessage } from './lib/types/MessageTypes' diff --git a/packages/sns/lib/sns/AbstractSnsPublisherMultiSchema.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts similarity index 53% rename from packages/sns/lib/sns/AbstractSnsPublisherMultiSchema.ts rename to packages/sns/lib/sns/AbstractSnsPublisher.ts index 9afccfa6..2d4b4b04 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisherMultiSchema.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -1,15 +1,16 @@ +import { PublishCommand } from '@aws-sdk/client-sns' +import type { PublishCommandInput } from '@aws-sdk/client-sns/dist-types/commands/PublishCommand' import type { Either } from '@lokalise/node-core' import type { AsyncPublisher, BarrierResult, MessageInvalidFormatError, MessageValidationError, - NewQueueOptions, + QueuePublisherOptions, } from '@message-queue-toolkit/core' import { MessageSchemaContainer } from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' -import type { ExistingSNSOptions, SNSCreationConfig, SNSDependencies } from './AbstractSnsService' +import type { SNSCreationConfig, SNSDependencies, SNSQueueLocatorType } from './AbstractSnsService' import { AbstractSnsService } from './AbstractSnsService' export type SNSMessageOptions = { @@ -17,18 +18,22 @@ export type SNSMessageOptions = { MessageDeduplicationId?: string } -export abstract class AbstractSnsPublisherMultiSchema +export type SNSPublisherOptions = QueuePublisherOptions< + SNSCreationConfig, + SNSQueueLocatorType, + MessagePayloadType +> + +export abstract class AbstractSnsPublisher extends AbstractSnsService implements AsyncPublisher { private readonly messageSchemaContainer: MessageSchemaContainer - constructor( - dependencies: SNSDependencies, - options: (ExistingSNSOptions | NewQueueOptions) & { - messageSchemas: readonly ZodSchema[] - }, - ) { + private isInitted: boolean + private initPromise?: Promise + + constructor(dependencies: SNSDependencies, options: SNSPublisherOptions) { super(dependencies, options) const messageSchemas = options.messageSchemas @@ -36,6 +41,7 @@ export abstract class AbstractSnsPublisherMultiSchema { @@ -44,7 +50,36 @@ export abstract class AbstractSnsPublisherMultiSchema> { + protected override preHandlerBarrier(): Promise> { throw new Error('Not implemented for publisher') } diff --git a/packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts b/packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts deleted file mode 100644 index aa931e0e..00000000 --- a/packages/sns/lib/sns/AbstractSnsPublisherMonoSchema.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - AsyncPublisher, - BarrierResult, - MessageInvalidFormatError, - MessageValidationError, - MonoSchemaQueueOptions, - NewQueueOptions, -} from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' - -import type { ExistingSNSOptions, SNSCreationConfig, SNSDependencies } from './AbstractSnsService' -import { AbstractSnsService } from './AbstractSnsService' - -export type SNSMessageOptions = { - MessageGroupId?: string - MessageDeduplicationId?: string -} - -export abstract class AbstractSnsPublisherMonoSchema - extends AbstractSnsService - implements AsyncPublisher -{ - private readonly messageSchema: ZodSchema - private readonly schemaEither: Either> - - constructor( - dependencies: SNSDependencies, - options: (ExistingSNSOptions | NewQueueOptions) & - MonoSchemaQueueOptions, - ) { - super(dependencies, options) - this.messageSchema = options.messageSchema - this.schemaEither = { - result: this.messageSchema, - } - } - - publish(message: MessagePayloadType, options: SNSMessageOptions = {}): Promise { - return this.internalPublish(message, this.messageSchema, options) - } - - /* c8 ignore start */ - protected override resolveMessage(): Either< - MessageInvalidFormatError | MessageValidationError, - unknown - > { - throw new Error('Not implemented for publisher') - } - - protected override resolveNextFunction(): () => void { - throw new Error('Not implemented for publisher') - } - - protected override processPrehandlers(): Promise { - throw new Error('Not implemented for publisher') - } - - protected override preHandlerBarrier(): Promise> { - throw new Error('Not implemented for publisher') - } - - override processMessage(): Promise> { - throw new Error('Not implemented for publisher') - } - - protected override resolveSchema() { - return this.schemaEither - } - /* c8 ignore stop */ -} diff --git a/packages/sns/lib/sns/AbstractSnsService.ts b/packages/sns/lib/sns/AbstractSnsService.ts index c9af2200..190d21bd 100644 --- a/packages/sns/lib/sns/AbstractSnsService.ts +++ b/packages/sns/lib/sns/AbstractSnsService.ts @@ -1,32 +1,14 @@ import type { SNSClient, CreateTopicCommandInput, Tag } from '@aws-sdk/client-sns' -import { PublishCommand } from '@aws-sdk/client-sns' -import type { PublishCommandInput } from '@aws-sdk/client-sns/dist-types/commands/PublishCommand' -import type { - QueueConsumerDependencies, - QueueDependencies, - NewQueueOptions, - ExistingQueueOptions, - NewQueueOptionsMultiSchema, - ExistingQueueOptionsMultiSchema, -} from '@message-queue-toolkit/core' +import type { QueueDependencies, QueueOptions } from '@message-queue-toolkit/core' import { AbstractQueueService } from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' import type { SNS_MESSAGE_BODY_TYPE } from '../types/MessageTypes' import { deleteSns, initSns } from '../utils/snsInitter' -import type { SNSMessageOptions } from './AbstractSnsPublisherMonoSchema' - export type SNSDependencies = QueueDependencies & { snsClient: SNSClient } -export type SNSQueueLocatorType = { - topicArn: string -} - -export type SNSConsumerDependencies = SNSDependencies & QueueConsumerDependencies - export type SNSTopicAWSConfig = CreateTopicCommandInput export type SNSTopicConfig = { tags?: Tag[] @@ -43,32 +25,6 @@ export type SNSTopicConfig = { } } -export type NewSNSOptions = NewQueueOptions - -export type ExistingSNSOptions = ExistingQueueOptions - -export type NewSNSOptionsMultiSchema< - MessagePayloadSchemas extends object, - ExecutionContext, - PrehandlerOutput, -> = NewQueueOptionsMultiSchema< - MessagePayloadSchemas, - SNSCreationConfig, - ExecutionContext, - PrehandlerOutput -> - -export type ExistingSNSOptionsMultiSchema< - MessagePayloadSchemas extends object, - ExecutionContext, - PrehandlerOutput, -> = ExistingQueueOptionsMultiSchema< - MessagePayloadSchemas, - SNSQueueLocatorType, - ExecutionContext, - PrehandlerOutput -> - export type ExtraSNSCreationParams = { queueUrlsWithSubscribePermissionsPrefix?: string allowedSourceOwner?: string @@ -79,12 +35,16 @@ export type SNSCreationConfig = { updateAttributesIfExists?: boolean } & ExtraSNSCreationParams +export type SNSQueueLocatorType = { + topicArn: string +} + +export type SNSOptions = QueueOptions + export abstract class AbstractSnsService< MessagePayloadType extends object, MessageEnvelopeType extends object = SNS_MESSAGE_BODY_TYPE, - SNSOptionsType extends - | ExistingQueueOptions - | NewQueueOptions = ExistingSNSOptions | NewQueueOptions, + SNSOptionsType extends SNSOptions = SNSOptions, DependenciesType extends SNSDependencies = SNSDependencies, > extends AbstractQueueService< MessagePayloadType, @@ -96,15 +56,11 @@ export abstract class AbstractSnsService< > { protected readonly snsClient: SNSClient // @ts-ignore - public topicArn: string - - private isInitted: boolean - private initPromise?: Promise + protected topicArn: string constructor(dependencies: DependenciesType, options: SNSOptionsType) { super(dependencies, options) - this.isInitted = false this.snsClient = dependencies.snsClient } @@ -115,46 +71,9 @@ export abstract class AbstractSnsService< const initResult = await initSns(this.snsClient, this.locatorConfig, this.creationConfig) this.topicArn = initResult.topicArn - this.isInitted = true } - // eslint-disable-next-line @typescript-eslint/require-await - public override async close(): Promise {} - - protected async internalPublish( - message: MessagePayloadType, - messageSchema: ZodSchema, - options: SNSMessageOptions = {}, - ): Promise { - // If it's not initted yet, do the lazy init - if (!this.isInitted) { - // avoid multiple concurrent inits - if (!this.initPromise) { - this.initPromise = this.init() - } - await this.initPromise - } - - try { - messageSchema.parse(message) - - if (this.logMessages) { - // @ts-ignore - const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) - this.logMessage(resolvedLogMessage) - } - - const input = { - Message: JSON.stringify(message), - TopicArn: this.topicArn, - ...options, - } satisfies PublishCommandInput - const command = new PublishCommand(input) - await this.snsClient.send(command) - this.handleMessageProcessed(message, 'published') - } catch (error) { - this.handleError(error) - throw error - } + public override close(): Promise { + return Promise.resolve() } } diff --git a/packages/sns/lib/sns/AbstractSnsSqsConsumerMultiSchema.ts b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts similarity index 63% rename from packages/sns/lib/sns/AbstractSnsSqsConsumerMultiSchema.ts rename to packages/sns/lib/sns/AbstractSnsSqsConsumer.ts index 6aac4bcf..f6873c55 100644 --- a/packages/sns/lib/sns/AbstractSnsSqsConsumerMultiSchema.ts +++ b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts @@ -1,60 +1,67 @@ import type { SNSClient } from '@aws-sdk/client-sns' -import type { MultiSchemaConsumerOptions } from '@message-queue-toolkit/core' -import type { SQSCreationConfig, SQSMessage } from '@message-queue-toolkit/sqs' -import { AbstractSqsConsumerMultiSchema, deleteSqs } from '@message-queue-toolkit/sqs' +import type { + SQSConsumerDependencies, + SQSConsumerOptions, + SQSCreationConfig, + SQSMessage, + SQSQueueLocatorType, +} from '@message-queue-toolkit/sqs' +import { AbstractSqsConsumer, deleteSqs } from '@message-queue-toolkit/sqs' import { deleteSnsSqs, initSnsSqs } from '../utils/snsInitter' import { readSnsMessage } from '../utils/snsMessageReader' import type { SNSSubscriptionOptions } from '../utils/snsSubscriber' -import type { SNSCreationConfig } from './AbstractSnsService' -import type { - ExistingSnsSqsConsumerOptions, - NewSnsSqsConsumerOptions, - SNSSQSConsumerDependencies, - SNSSQSQueueLocatorType, -} from './AbstractSnsSqsConsumerMonoSchema' +import type { SNSCreationConfig, SNSOptions, SNSQueueLocatorType } from './AbstractSnsService' -export type ExistingSnsSqsConsumerOptionsMulti< +export type SNSSQSConsumerDependencies = SQSConsumerDependencies & { + snsClient: SNSClient +} +export type SNSSQSCreationConfig = SQSCreationConfig & SNSCreationConfig + +export type SNSSQSQueueLocatorType = SQSQueueLocatorType & + SNSQueueLocatorType & { + subscriptionArn?: string + } + +export type SNSSQSConsumerOptions< MessagePayloadType extends object, ExecutionContext, PrehandlerOutput, -> = ExistingSnsSqsConsumerOptions & - MultiSchemaConsumerOptions - -export type NewSnsSqsConsumerOptionsMulti< - MessagePayloadType extends object, +> = SQSConsumerOptions< + MessagePayloadType, ExecutionContext, PrehandlerOutput, -> = NewSnsSqsConsumerOptions & - MultiSchemaConsumerOptions + SNSSQSCreationConfig, + SNSSQSQueueLocatorType +> & + SNSOptions & { + subscriptionConfig?: SNSSubscriptionOptions + } -export abstract class AbstractSnsSqsConsumerMultiSchema< +export abstract class AbstractSnsSqsConsumer< MessagePayloadSchemas extends object, ExecutionContext, PrehandlerOutput = undefined, -> extends AbstractSqsConsumerMultiSchema< +> extends AbstractSqsConsumer< MessagePayloadSchemas, ExecutionContext, PrehandlerOutput, + SNSSQSCreationConfig, SNSSQSQueueLocatorType, - SNSCreationConfig & SQSCreationConfig, - NewSnsSqsConsumerOptionsMulti + SNSSQSConsumerOptions > { private readonly subscriptionConfig?: SNSSubscriptionOptions private readonly snsClient: SNSClient + // @ts-ignore - public topicArn: string + protected topicArn: string // @ts-ignore - public subscriptionArn: string + protected subscriptionArn: string protected constructor( dependencies: SNSSQSConsumerDependencies, - options: NewSnsSqsConsumerOptionsMulti< - MessagePayloadSchemas, - ExecutionContext, - PrehandlerOutput - >, + options: SNSSQSConsumerOptions, executionContext: ExecutionContext, ) { super( diff --git a/packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts b/packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts deleted file mode 100644 index 26974b5c..00000000 --- a/packages/sns/lib/sns/AbstractSnsSqsConsumerMonoSchema.ts +++ /dev/null @@ -1,186 +0,0 @@ -import type { SNSClient } from '@aws-sdk/client-sns' -import type { Either } from '@lokalise/node-core' -import type { MonoSchemaQueueOptions, BarrierResult, Prehandler } from '@message-queue-toolkit/core' -import type { - SQSConsumerDependencies, - NewSQSConsumerOptions, - ExistingSQSConsumerOptions, - SQSQueueLocatorType, - SQSCreationConfig, - SQSMessage, - CommonSQSConsumerOptionsMono, -} from '@message-queue-toolkit/sqs' -import { AbstractSqsConsumer, deleteSqs } from '@message-queue-toolkit/sqs' -import type { ZodSchema } from 'zod' - -import { deleteSnsSqs, initSnsSqs } from '../utils/snsInitter' -import { readSnsMessage } from '../utils/snsMessageReader' -import type { SNSSubscriptionOptions } from '../utils/snsSubscriber' - -import type { - ExistingSNSOptions, - NewSNSOptions, - SNSCreationConfig, - SNSQueueLocatorType, -} from './AbstractSnsService' - -export type NewSnsSqsConsumerOptions = NewSQSConsumerOptions< - SQSCreationConfig & SNSCreationConfig -> & - NewSNSOptions & { - subscriptionConfig?: SNSSubscriptionOptions - } - -export type NewSnsSqsConsumerOptionsMono< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput = undefined, -> = NewSnsSqsConsumerOptions & - MonoSchemaQueueOptions & - CommonSQSConsumerOptionsMono - -export type ExistingSnsSqsConsumerOptions = ExistingSQSConsumerOptions & - ExistingSNSOptions & { - subscriptionConfig?: SNSSubscriptionOptions - } - -export type ExistingSnsSqsConsumerOptionsMono< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput, -> = ExistingSnsSqsConsumerOptions & - MonoSchemaQueueOptions & - CommonSQSConsumerOptionsMono - -export type SNSSQSConsumerDependencies = SQSConsumerDependencies & { - snsClient: SNSClient -} - -export type SNSSQSQueueLocatorType = SQSQueueLocatorType & - SNSQueueLocatorType & { - subscriptionArn?: string - } - -const DEFAULT_BARRIER_RESULT = { - isPassing: true, - output: undefined, -} as const - -export abstract class AbstractSnsSqsConsumerMonoSchema< - MessagePayloadType extends object, - ExecutionContext = undefined, - PrehandlerOutput = undefined, - BarrierOutput = undefined, -> extends AbstractSqsConsumer< - MessagePayloadType, - SNSSQSQueueLocatorType, - SNSCreationConfig & SQSCreationConfig, - | NewSnsSqsConsumerOptions - | ExistingSnsSqsConsumerOptionsMono, - ExecutionContext, - PrehandlerOutput, - BarrierOutput -> { - private readonly subscriptionConfig?: SNSSubscriptionOptions - private readonly snsClient: SNSClient - private readonly messageSchema: ZodSchema - private readonly schemaEither: Either> - private readonly prehandlers: Prehandler[] - - // @ts-ignore - public topicArn: string - // @ts-ignore - public subscriptionArn: string - - protected constructor( - dependencies: SNSSQSConsumerDependencies, - options: NewSnsSqsConsumerOptionsMono, - ) { - super(dependencies, { - ...options, - }) - - this.subscriptionConfig = options.subscriptionConfig - this.snsClient = dependencies.snsClient - this.prehandlers = options.prehandlers ?? [] - this.messageSchema = options.messageSchema - this.schemaEither = { - result: this.messageSchema, - } - } - - protected override resolveSchema() { - return this.schemaEither - } - - protected override resolveMessage(message: SQSMessage) { - return readSnsMessage(message, this.errorResolver) - } - - /** - * Override to implement barrier pattern - */ - protected preHandlerBarrier( - _message: MessagePayloadType, - _messageType: string, - ): Promise> { - // @ts-ignore - return Promise.resolve(DEFAULT_BARRIER_RESULT) - } - - protected override processPrehandlers(message: MessagePayloadType) { - return this.processPrehandlersInternal(this.prehandlers, message) - } - - // eslint-disable-next-line max-params - protected override resolveNextFunction( - prehandlers: Prehandler[], - message: MessagePayloadType, - index: number, - prehandlerOutput: PrehandlerOutput, - resolve: (value: PrehandlerOutput | PromiseLike) => void, - reject: (err: Error) => void, - ) { - return this.resolveNextPreHandlerFunctionInternal( - prehandlers, - this as unknown as ExecutionContext, - message, - index, - prehandlerOutput, - resolve, - reject, - ) - } - - override async init(): Promise { - if (this.deletionConfig && this.creationConfig && this.subscriptionConfig) { - await deleteSnsSqs( - this.sqsClient, - this.snsClient, - this.deletionConfig, - this.creationConfig.queue, - this.creationConfig.topic, - this.subscriptionConfig, - { - logger: this.logger, - }, - ) - } else if (this.deletionConfig && this.creationConfig) { - await deleteSqs(this.sqsClient, this.deletionConfig, this.creationConfig) - } - - const initSnsSqsResult = await initSnsSqs( - this.sqsClient, - this.snsClient, - this.locatorConfig, - this.creationConfig, - this.subscriptionConfig, - { - logger: this.logger, - }, - ) - this.queueUrl = initSnsSqsResult.queueUrl - this.topicArn = initSnsSqsResult.topicArn - this.subscriptionArn = initSnsSqsResult.subscriptionArn - } -} diff --git a/packages/sns/lib/utils/snsInitter.ts b/packages/sns/lib/utils/snsInitter.ts index efd8368f..a2cb30a2 100644 --- a/packages/sns/lib/utils/snsInitter.ts +++ b/packages/sns/lib/utils/snsInitter.ts @@ -6,7 +6,7 @@ import type { SQSCreationConfig } from '@message-queue-toolkit/sqs' import { deleteQueue, getQueueAttributes } from '@message-queue-toolkit/sqs' import type { SNSCreationConfig, SNSQueueLocatorType } from '../sns/AbstractSnsService' -import type { SNSSQSQueueLocatorType } from '../sns/AbstractSnsSqsConsumerMonoSchema' +import type { SNSSQSQueueLocatorType } from '../sns/AbstractSnsSqsConsumer' import type { SNSSubscriptionOptions } from './snsSubscriber' import { subscribeToTopic } from './snsSubscriber' diff --git a/packages/sns/package.json b/packages/sns/package.json index d1c78ae4..7961f934 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "11.1.0", + "version": "12.0.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", diff --git a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMultiSchema.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.spec.ts similarity index 83% rename from packages/sns/test/consumers/SnsSqsPermissionsConsumerMultiSchema.spec.ts rename to packages/sns/test/consumers/SnsSqsPermissionConsumer.spec.ts index b9c4fbd8..d615f40a 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMultiSchema.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.spec.ts @@ -5,13 +5,13 @@ import type { AwilixContainer } from 'awilix' import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest' import { assertTopic, deleteTopic } from '../../lib/utils/snsUtils' -import type { SnsPermissionPublisherMultiSchema } from '../publishers/SnsPermissionPublisherMultiSchema' +import type { SnsPermissionPublisher } from '../publishers/SnsPermissionPublisher' import { registerDependencies } from '../utils/testContext' import type { Dependencies } from '../utils/testContext' -import { SnsSqsPermissionConsumerMultiSchema } from './SnsSqsPermissionConsumerMultiSchema' +import { SnsSqsPermissionConsumer } from './SnsSqsPermissionConsumer' -describe('SNS PermissionsConsumerMultiSchema', () => { +describe('SnsSqsPermissionConsumer', () => { describe('init', () => { let diContainer: AwilixContainer let sqsClient: SQSClient @@ -31,7 +31,7 @@ describe('SNS PermissionsConsumerMultiSchema', () => { QueueName: 'existingQueue', }) - const newConsumer = new SnsSqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { locatorConfig: { queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', subscriptionArn: 'dummy', @@ -51,7 +51,7 @@ describe('SNS PermissionsConsumerMultiSchema', () => { Name: 'existingTopic', }) - const newConsumer = new SnsSqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { locatorConfig: { topicArn: arn, queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', @@ -61,11 +61,11 @@ describe('SNS PermissionsConsumerMultiSchema', () => { }) await newConsumer.init() - expect(newConsumer.queueUrl).toBe( + expect(newConsumer.subscriptionProps.queueUrl).toBe( 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', ) - expect(newConsumer.topicArn).toEqual(arn) - expect(newConsumer.subscriptionArn).toBe( + expect(newConsumer.subscriptionProps.topicArn).toEqual(arn) + expect(newConsumer.subscriptionProps.subscriptionArn).toBe( 'arn:aws:sns:eu-west-1:000000000000:user_permissions:bdf640a2-bedf-475a-98b8-758b88c87395', ) await deleteTopic(snsClient, 'existingTopic') @@ -79,7 +79,7 @@ describe('SNS PermissionsConsumerMultiSchema', () => { }, }) - const newConsumer = new SnsSqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { creationConfig: { topic: { Name: 'sometopic', @@ -98,12 +98,12 @@ describe('SNS PermissionsConsumerMultiSchema', () => { }) await newConsumer.init() - expect(newConsumer.queueUrl).toBe( + expect(newConsumer.subscriptionProps.queueUrl).toBe( 'http://sqs.eu-west-1.localstack:4566/000000000000/existingQueue', ) const attributes = await getQueueAttributes(sqsClient, { - queueUrl: newConsumer.queueUrl, + queueUrl: newConsumer.subscriptionProps.queueUrl, }) expect(attributes.result?.attributes!.KmsMasterKeyId).toBe('othervalue') @@ -117,7 +117,7 @@ describe('SNS PermissionsConsumerMultiSchema', () => { }, }) - const newConsumer = new SnsSqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { creationConfig: { topic: { Name: 'sometopic', @@ -137,12 +137,12 @@ describe('SNS PermissionsConsumerMultiSchema', () => { }) await newConsumer.init() - expect(newConsumer.queueUrl).toBe( + expect(newConsumer.subscriptionProps.queueUrl).toBe( 'http://sqs.eu-west-1.localstack:4566/000000000000/existingQueue', ) const attributes = await getQueueAttributes(sqsClient, { - queueUrl: newConsumer.queueUrl, + queueUrl: newConsumer.subscriptionProps.queueUrl, }) expect(attributes.result?.attributes!.Policy).toBe( @@ -151,7 +151,7 @@ describe('SNS PermissionsConsumerMultiSchema', () => { }) it('does not attempt to update non-existing queue when passing update param', async () => { - const newConsumer = new SnsSqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { creationConfig: { topic: { Name: 'sometopic', @@ -170,12 +170,12 @@ describe('SNS PermissionsConsumerMultiSchema', () => { }) await newConsumer.init() - expect(newConsumer.queueUrl).toBe( + expect(newConsumer.subscriptionProps.queueUrl).toBe( 'http://sqs.eu-west-1.localstack:4566/000000000000/existingQueue', ) const attributes = await getQueueAttributes(sqsClient, { - queueUrl: newConsumer.queueUrl, + queueUrl: newConsumer.subscriptionProps.queueUrl, }) expect(attributes.result?.attributes!.KmsMasterKeyId).toBe('othervalue') @@ -184,10 +184,10 @@ describe('SNS PermissionsConsumerMultiSchema', () => { describe('prehandlers', () => { let diContainer: AwilixContainer - let publisher: SnsPermissionPublisherMultiSchema + let publisher: SnsPermissionPublisher beforeEach(async () => { diContainer = await registerDependencies({}, false) - publisher = diContainer.cradle.permissionPublisherMultiSchema + publisher = diContainer.cradle.permissionPublisher await publisher.init() }) @@ -199,13 +199,13 @@ describe('SNS PermissionsConsumerMultiSchema', () => { it('processes one prehandler', async () => { expect.assertions(1) - const newConsumer = new SnsSqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { creationConfig: { topic: { - Name: SnsSqsPermissionConsumerMultiSchema.SUBSCRIBED_TOPIC_NAME, + Name: SnsSqsPermissionConsumer.SUBSCRIBED_TOPIC_NAME, }, queue: { - QueueName: SnsSqsPermissionConsumerMultiSchema.CONSUMED_QUEUE_NAME, + QueueName: SnsSqsPermissionConsumer.CONSUMED_QUEUE_NAME, }, updateAttributesIfExists: true, }, @@ -244,13 +244,13 @@ describe('SNS PermissionsConsumerMultiSchema', () => { it('processes two prehandlers', async () => { expect.assertions(1) - const newConsumer = new SnsSqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { creationConfig: { topic: { - Name: SnsSqsPermissionConsumerMultiSchema.SUBSCRIBED_TOPIC_NAME, + Name: SnsSqsPermissionConsumer.SUBSCRIBED_TOPIC_NAME, }, queue: { - QueueName: SnsSqsPermissionConsumerMultiSchema.CONSUMED_QUEUE_NAME, + QueueName: SnsSqsPermissionConsumer.CONSUMED_QUEUE_NAME, }, updateAttributesIfExists: true, }, @@ -298,12 +298,12 @@ describe('SNS PermissionsConsumerMultiSchema', () => { describe('consume', () => { let diContainer: AwilixContainer - let publisher: SnsPermissionPublisherMultiSchema - let consumer: SnsSqsPermissionConsumerMultiSchema + let publisher: SnsPermissionPublisher + let consumer: SnsSqsPermissionConsumer beforeEach(async () => { diContainer = await registerDependencies() - publisher = diContainer.cradle.permissionPublisherMultiSchema - consumer = diContainer.cradle.permissionConsumerMultiSchema + publisher = diContainer.cradle.permissionPublisher + consumer = diContainer.cradle.permissionConsumer }) afterEach(async () => { diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts similarity index 75% rename from packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts rename to packages/sns/test/consumers/SnsSqsPermissionConsumer.ts index d67fdb30..e2005e54 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumerMultiSchema.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts @@ -9,11 +9,9 @@ import type { PrehandlerResult } from '@message-queue-toolkit/core/dist/lib/queu import type { SNSSQSConsumerDependencies, - NewSnsSqsConsumerOptions, - ExistingSnsSqsConsumerOptions, -} from '../../lib/sns/AbstractSnsSqsConsumerMonoSchema' -import type { NewSnsSqsConsumerOptionsMulti } from '../../lib/sns/AbstractSnsSqsConsumerMultiSchema' -import { AbstractSnsSqsConsumerMultiSchema } from '../../lib/sns/AbstractSnsSqsConsumerMultiSchema' + SNSSQSConsumerOptions, +} from '../../lib/sns/AbstractSnsSqsConsumer' +import { AbstractSnsSqsConsumer } from '../../lib/sns/AbstractSnsSqsConsumer' import type { PERMISSIONS_ADD_MESSAGE_TYPE, @@ -24,10 +22,18 @@ import { PERMISSIONS_REMOVE_MESSAGE_SCHEMA, } from './userConsumerSchemas' -type SnsSqsPermissionConsumerMultiSchemaOptions = ( - | Pick - | Pick -) & { +type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE +type ExecutionContext = { + incrementAmount: number +} +type PrehandlerOutput = { + prehandlerCount: number +} + +type SnsSqsPermissionConsumerOptions = Pick< + SNSSQSConsumerOptions, + 'creationConfig' | 'locatorConfig' | 'deletionConfig' +> & { addPreHandlerBarrier?: ( message: SupportedMessages, _executionContext: ExecutionContext, @@ -41,21 +47,13 @@ type SnsSqsPermissionConsumerMultiSchemaOptions = ( removePreHandlers?: Prehandler[] } -type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE -type ExecutionContext = { - incrementAmount: number -} -type PrehandlerOutput = { - prehandlerCount: number -} - -export class SnsSqsPermissionConsumerMultiSchema extends AbstractSnsSqsConsumerMultiSchema< +export class SnsSqsPermissionConsumer extends AbstractSnsSqsConsumer< SupportedMessages, ExecutionContext, PrehandlerOutput > { - public static CONSUMED_QUEUE_NAME = 'user_permissions_multi' - public static SUBSCRIBED_TOPIC_NAME = 'user_permissions_multi' + public static readonly CONSUMED_QUEUE_NAME = 'user_permissions_multi' + public static readonly SUBSCRIBED_TOPIC_NAME = 'user_permissions_multi' public addCounter = 0 public addBarrierCounter = 0 @@ -64,13 +62,13 @@ export class SnsSqsPermissionConsumerMultiSchema extends AbstractSnsSqsConsumerM constructor( dependencies: SNSSQSConsumerDependencies, - options: SnsSqsPermissionConsumerMultiSchemaOptions = { + options: SnsSqsPermissionConsumerOptions = { creationConfig: { queue: { - QueueName: SnsSqsPermissionConsumerMultiSchema.CONSUMED_QUEUE_NAME, + QueueName: SnsSqsPermissionConsumer.CONSUMED_QUEUE_NAME, }, topic: { - Name: SnsSqsPermissionConsumerMultiSchema.SUBSCRIBED_TOPIC_NAME, + Name: SnsSqsPermissionConsumer.SUBSCRIBED_TOPIC_NAME, }, }, }, @@ -142,25 +140,36 @@ export class SnsSqsPermissionConsumerMultiSchema extends AbstractSnsSqsConsumerM }, ) .build(), - messageTypeField: 'messageType', - deletionConfig: { + deletionConfig: options.deletionConfig ?? { deleteIfExists: true, }, + ...(options.locatorConfig + ? { locatorConfig: options.locatorConfig } + : { + creationConfig: options.creationConfig ?? { + queue: { QueueName: SnsSqsPermissionConsumer.CONSUMED_QUEUE_NAME }, + topic: { Name: SnsSqsPermissionConsumer.SUBSCRIBED_TOPIC_NAME }, + }, + }), + messageTypeField: 'messageType', consumerOverrides: { terminateVisibilityTimeout: true, // this allows to retry failed messages immediately }, subscriptionConfig: { updateAttributesIfExists: false, }, - // FixMe this casting shouldn't be necessary - ...(options as Pick< - NewSnsSqsConsumerOptionsMulti, - 'creationConfig' | 'logMessages' - >), }, { incrementAmount: 1, }, ) } + + get subscriptionProps() { + return { + topicArn: this.topicArn, + queueUrl: this.queueUrl, + subscriptionArn: this.subscriptionArn, + } + } } diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts deleted file mode 100644 index 372a8cf4..00000000 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumerMonoSchema.ts +++ /dev/null @@ -1,99 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { BarrierResult } from '@message-queue-toolkit/core' - -import type { - SNSSQSConsumerDependencies, - NewSnsSqsConsumerOptions, - ExistingSnsSqsConsumerOptions, -} from '../../lib/sns/AbstractSnsSqsConsumerMonoSchema' -import { AbstractSnsSqsConsumerMonoSchema } from '../../lib/sns/AbstractSnsSqsConsumerMonoSchema' -import { userPermissionMap } from '../repositories/PermissionRepository' - -import type { PERMISSIONS_MESSAGE_TYPE } from './userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from './userConsumerSchemas' - -type Options = - | Pick - | Pick - -export class SnsSqsPermissionConsumerMonoSchema extends AbstractSnsSqsConsumerMonoSchema< - PERMISSIONS_MESSAGE_TYPE, - undefined, - undefined, - number -> { - public static CONSUMED_QUEUE_NAME = 'user_permissions' - public static SUBSCRIBED_TOPIC_NAME = 'user_permissions' - - public preHandlerBarrierCounter: number = 0 - - constructor( - dependencies: SNSSQSConsumerDependencies, - options: Options = { - creationConfig: { - queue: { - QueueName: SnsSqsPermissionConsumerMonoSchema.CONSUMED_QUEUE_NAME, - }, - topic: { - Name: SnsSqsPermissionConsumerMonoSchema.SUBSCRIBED_TOPIC_NAME, - }, - }, - }, - ) { - super(dependencies, { - messageSchema: PERMISSIONS_MESSAGE_SCHEMA, - handlerSpy: true, - messageTypeField: 'messageType', - deletionConfig: { - deleteIfExists: true, - }, - consumerOverrides: { - terminateVisibilityTimeout: true, // this allows to retry failed messages immediately - }, - subscriptionConfig: { - updateAttributesIfExists: false, - }, - ...(options as Omit), - }) - } - - override async processMessage( - message: PERMISSIONS_MESSAGE_TYPE, - ): Promise> { - const matchedUserPermissions = message.userIds.reduce((acc, userId) => { - if (userPermissionMap[userId]) { - acc.push(userPermissionMap[userId]) - } - return acc - }, [] as string[][]) - - if (!matchedUserPermissions || matchedUserPermissions.length < message.userIds.length) { - // not all users were already created, we need to wait to be able to set permissions - return { - error: 'retryLater', - } - } - - // Do not do this in production, some kind of bulk insertion is needed here - for (const userPermissions of matchedUserPermissions) { - userPermissions.push(...message.permissions) - } - - return { - result: 'success', - } - } - - async preHandlerBarrier(_message: PERMISSIONS_MESSAGE_TYPE): Promise> { - this.preHandlerBarrierCounter++ - if (this.preHandlerBarrierCounter < 3) { - return { - isPassing: false, - } - } - return { - isPassing: true, - output: this.preHandlerBarrierCounter, - } - } -} diff --git a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts deleted file mode 100644 index d6b1786d..00000000 --- a/packages/sns/test/consumers/SnsSqsPermissionsConsumerMonoSchema.spec.ts +++ /dev/null @@ -1,277 +0,0 @@ -import type { SNSClient } from '@aws-sdk/client-sns' -import type { SQSClient } from '@aws-sdk/client-sqs' -import { waitAndRetry } from '@message-queue-toolkit/core' -import type { FakeConsumerErrorResolver } from '@message-queue-toolkit/sqs' -import { assertQueue, deleteQueue, getQueueAttributes } from '@message-queue-toolkit/sqs' -import type { AwilixContainer } from 'awilix' -import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest' -import z from 'zod' - -import { assertTopic, deleteTopic, getTopicAttributes } from '../../lib/utils/snsUtils' -import type { SnsPermissionPublisherMonoSchema } from '../publishers/SnsPermissionPublisherMonoSchema' -import { userPermissionMap } from '../repositories/PermissionRepository' -import { registerDependencies } from '../utils/testContext' -import type { Dependencies } from '../utils/testContext' - -import { SnsSqsPermissionConsumerMonoSchema } from './SnsSqsPermissionConsumerMonoSchema' - -const userIds = [100, 200, 300] -const perms: [string, ...string[]] = ['perm1', 'perm2'] - -async function resolvePermissions(userIds: number[]) { - const usersPerms = userIds.reduce((acc, userId) => { - if (userPermissionMap[userId]) { - acc.push(userPermissionMap[userId]) - } - return acc - }, [] as string[][]) - - if (usersPerms && usersPerms.length !== userIds.length) { - return null - } - - for (const userPerms of usersPerms) - if (userPerms.length !== perms.length) { - return null - } - - return usersPerms -} - -describe('SNS PermissionsConsumer', () => { - describe('init', () => { - let diContainer: AwilixContainer - let sqsClient: SQSClient - let snsClient: SNSClient - beforeAll(async () => { - diContainer = await registerDependencies({}, false) - sqsClient = diContainer.cradle.sqsClient - snsClient = diContainer.cradle.snsClient - await deleteQueue(sqsClient, SnsSqsPermissionConsumerMonoSchema.CONSUMED_QUEUE_NAME) - }) - - it('sets correct policy when policy fields are set', async () => { - const newConsumer = new SnsSqsPermissionConsumerMonoSchema(diContainer.cradle, { - creationConfig: { - queue: { - QueueName: 'policy-queue', - }, - topic: { - Name: 'policy-topic', - }, - topicArnsWithPublishPermissionsPrefix: 'dummy*', - queueUrlsWithSubscribePermissionsPrefix: 'dummy*', - }, - }) - await newConsumer.init() - - const queue = await getQueueAttributes( - sqsClient, - { - queueUrl: newConsumer.queueUrl, - }, - ['Policy'], - ) - const topic = await getTopicAttributes(snsClient, newConsumer.topicArn) - - expect(queue.result?.attributes?.Policy).toBe( - `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSNSPublish","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sqs:SendMessage","Resource":"arn:aws:sqs:eu-west-1:000000000000:policy-queue","Condition":{"ArnLike":{"aws:SourceArn":"dummy*"}}}]}`, - ) - expect(topic.result?.attributes?.Policy).toBe( - `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"arn:aws:sns:eu-west-1:000000000000:policy-topic","Condition":{"StringLike":{"sns:Endpoint":"dummy*"}}}]}`, - ) - }) - - // FixMe https://github.com/localstack/localstack/issues/9306 - it.skip('throws an error when invalid queue locator is passed', async () => { - await assertQueue(sqsClient, { - QueueName: 'existingQueue', - }) - - const newConsumer = new SnsSqsPermissionConsumerMonoSchema(diContainer.cradle, { - locatorConfig: { - queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', - subscriptionArn: 'dummy', - topicArn: 'dummy', - }, - }) - - await expect(() => newConsumer.init()).rejects.toThrow(/does not exist/) - }) - - it('does not create a new queue when queue locator is passed', async () => { - await assertQueue(sqsClient, { - QueueName: 'existingQueue', - }) - - const arn = await assertTopic(snsClient, { - Name: 'existingTopic', - }) - - const newConsumer = new SnsSqsPermissionConsumerMonoSchema(diContainer.cradle, { - locatorConfig: { - topicArn: arn, - queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', - subscriptionArn: - 'arn:aws:sns:eu-west-1:000000000000:user_permissions:bdf640a2-bedf-475a-98b8-758b88c87395', - }, - }) - - await newConsumer.init() - expect(newConsumer.queueUrl).toBe( - 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', - ) - expect(newConsumer.topicArn).toEqual(arn) - expect(newConsumer.subscriptionArn).toBe( - 'arn:aws:sns:eu-west-1:000000000000:user_permissions:bdf640a2-bedf-475a-98b8-758b88c87395', - ) - await deleteTopic(snsClient, 'existingTopic') - }) - }) - - describe('consume', () => { - let diContainer: AwilixContainer - let publisher: SnsPermissionPublisherMonoSchema - let consumer: SnsSqsPermissionConsumerMonoSchema - let fakeResolver: FakeConsumerErrorResolver - - beforeEach(async () => { - diContainer = await registerDependencies() - publisher = diContainer.cradle.permissionPublisher - consumer = diContainer.cradle.permissionConsumer - fakeResolver = diContainer.cradle.consumerErrorResolver as FakeConsumerErrorResolver - - delete userPermissionMap[100] - delete userPermissionMap[200] - delete userPermissionMap[300] - }) - - afterEach(async () => { - const { awilixManager } = diContainer.cradle - - await awilixManager.executeDispose() - await diContainer.dispose() - }) - - describe('happy path', () => { - it('Creates permissions', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - await publisher.publish({ - id: '1', - messageType: 'add', - userIds, - permissions: perms, - }) - - await consumer.handlerSpy.waitForMessageWithId('1', 'consumed') - const updatedUsersPermissions = await resolvePermissions(userIds) - - if (null === updatedUsersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(consumer.preHandlerBarrierCounter).toBe(3) - expect(updatedUsersPermissions).toBeDefined() - expect(updatedUsersPermissions[0]).toHaveLength(2) - }) - - it('Wait for users to be created and then create permissions', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - await publisher.publish({ - id: '2', - messageType: 'add', - userIds, - permissions: perms, - }) - - await consumer.handlerSpy.waitForMessageWithId('2', 'retryLater') - // no users in the database, so message will go back to the queue - const usersFromDb = await resolvePermissions(userIds) - expect(usersFromDb).toBeNull() - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - await consumer.handlerSpy.waitForMessageWithId('2', 'consumed') - const usersPermissions = await resolvePermissions(userIds) - - if (null === usersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(usersPermissions).toBeDefined() - expect(usersPermissions[0]).toHaveLength(2) - }) - - it('Not all users exist, no permissions were created initially', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - - await publisher.publish({ - id: '3', - messageType: 'add', - userIds, - permissions: perms, - }) - - await consumer.handlerSpy.waitForMessageWithId('3', 'retryLater') - // not all users are in the database, so message will go back to the queue - const usersFromDb = await resolvePermissions(userIds) - expect(usersFromDb).toBeNull() - - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - await consumer.handlerSpy.waitForMessageWithId('3', 'consumed') - const usersPermissions = await resolvePermissions(userIds) - - if (null === usersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(usersPermissions).toBeDefined() - expect(usersPermissions[0]).toHaveLength(2) - }) - }) - - describe('error handling', () => { - it('Invalid message in the queue', async () => { - // @ts-ignore - publisher['messageSchema'] = z.any() - await publisher.publish({ - id: 'abc', - messageType: 'add', - permissions: perms, - } as any) - - const messageResult = await consumer.handlerSpy.waitForMessageWithId('abc') - expect(messageResult.processingResult).toBe('invalid_message') - - expect(fakeResolver.handleErrorCallsCount).toBe(1) - }) - - it('Non-JSON message in the queue', async () => { - // @ts-ignore - publisher['messageSchema'] = z.any() - await publisher.publish('dummy' as any) - - await waitAndRetry(() => { - return fakeResolver.handleErrorCallsCount > 0 - }) - - expect(fakeResolver.handleErrorCallsCount).toBe(1) - }) - }) - }) -}) diff --git a/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts b/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts index e4665c4c..7bfa3776 100644 --- a/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts +++ b/packages/sns/test/publishers/SnsPermissionPublisher.spec.ts @@ -12,17 +12,16 @@ import { subscribeToTopic } from '../../lib/utils/snsSubscriber' import { assertTopic, deleteTopic, getTopicAttributes } from '../../lib/utils/snsUtils' import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' -import { userPermissionMap } from '../repositories/PermissionRepository' import { registerDependencies } from '../utils/testContext' import type { Dependencies } from '../utils/testContext' -import { SnsPermissionPublisherMonoSchema } from './SnsPermissionPublisherMonoSchema' +import { SnsPermissionPublisher } from './SnsPermissionPublisher' const perms: [string, ...string[]] = ['perm1', 'perm2'] const userIds = [100, 200, 300] const queueName = 'someQueue' -describe('SNSPermissionPublisher', () => { +describe('SnsPermissionPublisher', () => { describe('init', () => { let diContainer: AwilixContainer let snsClient: SNSClient @@ -32,7 +31,7 @@ describe('SNSPermissionPublisher', () => { }) it('sets correct policy when policy fields are set', async () => { - const newPublisher = new SnsPermissionPublisherMonoSchema(diContainer.cradle, { + const newPublisher = new SnsPermissionPublisher(diContainer.cradle, { creationConfig: { topic: { Name: 'policy-topic', @@ -43,7 +42,7 @@ describe('SNSPermissionPublisher', () => { await newPublisher.init() - const topic = await getTopicAttributes(snsClient, newPublisher.topicArn) + const topic = await getTopicAttributes(snsClient, newPublisher.topicArnProp) expect(topic.result?.attributes?.Policy).toBe( `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"arn:aws:sns:eu-west-1:000000000000:policy-topic","Condition":{"StringLike":{"sns:Endpoint":"dummy*"}}}]}`, @@ -51,7 +50,7 @@ describe('SNSPermissionPublisher', () => { }) it('sets correct policy when two policy fields are set', async () => { - const newPublisher = new SnsPermissionPublisherMonoSchema(diContainer.cradle, { + const newPublisher = new SnsPermissionPublisher(diContainer.cradle, { creationConfig: { topic: { Name: 'policy-topic', @@ -63,7 +62,7 @@ describe('SNSPermissionPublisher', () => { await newPublisher.init() - const topic = await getTopicAttributes(snsClient, newPublisher.topicArn) + const topic = await getTopicAttributes(snsClient, newPublisher.topicArnProp) expect(topic.result?.attributes?.Policy).toBe( `{"Version":"2012-10-17","Id":"__default_policy_ID","Statement":[{"Sid":"AllowSQSSubscription","Effect":"Allow","Principal":{"AWS":"*"},"Action":["sns:Subscribe"],"Resource":"arn:aws:sns:eu-west-1:000000000000:policy-topic","Condition":{"StringEquals":{"AWS:SourceOwner":"111111111111"},"StringLike":{"sns:Endpoint":"dummy*"}}}]}`, @@ -72,7 +71,7 @@ describe('SNSPermissionPublisher', () => { // FixMe https://github.com/localstack/localstack/issues/9306 it.skip('throws an error when invalid queue locator is passed', async () => { - const newPublisher = new SnsPermissionPublisherMonoSchema(diContainer.cradle, { + const newPublisher = new SnsPermissionPublisher(diContainer.cradle, { locatorConfig: { topicArn: 'dummy', }, @@ -86,14 +85,14 @@ describe('SNSPermissionPublisher', () => { Name: 'existingTopic', }) - const newPublisher = new SnsPermissionPublisherMonoSchema(diContainer.cradle, { + const newPublisher = new SnsPermissionPublisher(diContainer.cradle, { locatorConfig: { topicArn: arn, }, }) await newPublisher.init() - expect(newPublisher.topicArn).toEqual(arn) + expect(newPublisher.topicArnProp).toEqual(arn) await deleteTopic(snsClient, 'existingTopic') }) }) @@ -111,11 +110,7 @@ describe('SNSPermissionPublisher', () => { await diContainer.cradle.permissionConsumer.close() await deleteQueue(sqsClient, queueName) - await deleteTopic(snsClient, SnsPermissionPublisherMonoSchema.TOPIC_NAME) - - delete userPermissionMap[100] - delete userPermissionMap[200] - delete userPermissionMap[300] + await deleteTopic(snsClient, SnsPermissionPublisher.TOPIC_NAME) }) afterEach(async () => { @@ -145,7 +140,7 @@ describe('SNSPermissionPublisher', () => { QueueName: queueName, }, { - Name: SnsPermissionPublisherMonoSchema.TOPIC_NAME, + Name: SnsPermissionPublisher.TOPIC_NAME, }, { updateAttributesIfExists: false, @@ -189,7 +184,7 @@ describe('SNSPermissionPublisher', () => { }) it('publish message with lazy loading', async () => { - const newPublisher = new SnsPermissionPublisherMonoSchema(diContainer.cradle) + const newPublisher = new SnsPermissionPublisher(diContainer.cradle) const message = { id: '1', diff --git a/packages/sns/test/publishers/SnsPermissionPublisher.ts b/packages/sns/test/publishers/SnsPermissionPublisher.ts new file mode 100644 index 00000000..12717541 --- /dev/null +++ b/packages/sns/test/publishers/SnsPermissionPublisher.ts @@ -0,0 +1,41 @@ +import { AbstractSnsPublisher } from '../../lib/sns/AbstractSnsPublisher' +import type { SNSDependencies, SNSOptions } from '../../lib/sns/AbstractSnsService' +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from '../consumers/userConsumerSchemas' +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from '../consumers/userConsumerSchemas' + +type SupportedTypes = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE + +export class SnsPermissionPublisher extends AbstractSnsPublisher { + public static readonly TOPIC_NAME = 'user_permissions_multi' + + constructor( + dependencies: SNSDependencies, + options?: Pick, + ) { + super(dependencies, { + ...(options?.locatorConfig + ? { locatorConfig: options?.locatorConfig } + : { + creationConfig: options?.creationConfig ?? { + topic: { Name: SnsPermissionPublisher.TOPIC_NAME }, + }, + }), + deletionConfig: { + deleteIfExists: false, + }, + messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], + handlerSpy: true, + messageTypeField: 'messageType', + }) + } + + get topicArnProp(): string { + return this.topicArn + } +} diff --git a/packages/sns/test/publishers/SnsPermissionPublisherMonoSchema.ts b/packages/sns/test/publishers/SnsPermissionPublisherMonoSchema.ts deleted file mode 100644 index 349ca934..00000000 --- a/packages/sns/test/publishers/SnsPermissionPublisherMonoSchema.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { AbstractSnsPublisherMonoSchema } from '../../lib/sns/AbstractSnsPublisherMonoSchema' -import type { - SNSDependencies, - NewSNSOptions, - ExistingSNSOptions, -} from '../../lib/sns/AbstractSnsService' -import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' - -export class SnsPermissionPublisherMonoSchema extends AbstractSnsPublisherMonoSchema { - public static TOPIC_NAME = 'user_permissions' - - constructor( - dependencies: SNSDependencies, - options: Pick | Pick = { - creationConfig: { - topic: { - Name: SnsPermissionPublisherMonoSchema.TOPIC_NAME, - }, - }, - }, - ) { - super(dependencies, { - deletionConfig: { - deleteIfExists: false, - }, - handlerSpy: true, - messageSchema: PERMISSIONS_MESSAGE_SCHEMA, - messageTypeField: 'messageType', - ...options, - }) - } -} diff --git a/packages/sns/test/publishers/SnsPermissionPublisherMultiSchema.ts b/packages/sns/test/publishers/SnsPermissionPublisherMultiSchema.ts deleted file mode 100644 index cadbcab8..00000000 --- a/packages/sns/test/publishers/SnsPermissionPublisherMultiSchema.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { AbstractSnsPublisherMultiSchema } from '../../lib/sns/AbstractSnsPublisherMultiSchema' -import type { - SNSDependencies, - NewSNSOptions, - ExistingSNSOptions, -} from '../../lib/sns/AbstractSnsService' -import type { - PERMISSIONS_ADD_MESSAGE_TYPE, - PERMISSIONS_REMOVE_MESSAGE_TYPE, -} from '../consumers/userConsumerSchemas' -import { - PERMISSIONS_ADD_MESSAGE_SCHEMA, - PERMISSIONS_REMOVE_MESSAGE_SCHEMA, -} from '../consumers/userConsumerSchemas' - -type SupportedTypes = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE - -export class SnsPermissionPublisherMultiSchema extends AbstractSnsPublisherMultiSchema { - public static TOPIC_NAME = 'user_permissions_multi' - - constructor( - dependencies: SNSDependencies, - options: Pick | Pick = { - creationConfig: { - topic: { - Name: SnsPermissionPublisherMultiSchema.TOPIC_NAME, - }, - }, - }, - ) { - super(dependencies, { - deletionConfig: { - deleteIfExists: false, - }, - messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], - handlerSpy: true, - messageTypeField: 'messageType', - ...options, - }) - } -} diff --git a/packages/sns/test/repositories/PermissionRepository.ts b/packages/sns/test/repositories/PermissionRepository.ts deleted file mode 100644 index 89b09f10..00000000 --- a/packages/sns/test/repositories/PermissionRepository.ts +++ /dev/null @@ -1 +0,0 @@ -export const userPermissionMap: Record = {} diff --git a/packages/sns/test/utils/testContext.ts b/packages/sns/test/utils/testContext.ts index 6d4d9517..aed73d4a 100644 --- a/packages/sns/test/utils/testContext.ts +++ b/packages/sns/test/utils/testContext.ts @@ -7,10 +7,8 @@ import type { Resolver } from 'awilix' import { asClass, asFunction, createContainer, Lifetime } from 'awilix' import { AwilixManager } from 'awilix-manager' -import { SnsSqsPermissionConsumerMonoSchema } from '../consumers/SnsSqsPermissionConsumerMonoSchema' -import { SnsSqsPermissionConsumerMultiSchema } from '../consumers/SnsSqsPermissionConsumerMultiSchema' -import { SnsPermissionPublisherMonoSchema } from '../publishers/SnsPermissionPublisherMonoSchema' -import { SnsPermissionPublisherMultiSchema } from '../publishers/SnsPermissionPublisherMultiSchema' +import { SnsSqsPermissionConsumer } from '../consumers/SnsSqsPermissionConsumer' +import { SnsPermissionPublisher } from '../publishers/SnsPermissionPublisher' import { TEST_AWS_CONFIG } from './testSnsConfig' @@ -63,7 +61,7 @@ export async function registerDependencies( consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), - permissionConsumer: asClass(SnsSqsPermissionConsumerMonoSchema, { + permissionConsumer: asClass(SnsSqsPermissionConsumer, { lifetime: Lifetime.SINGLETON, asyncInit: 'start', asyncDispose: 'close', @@ -71,23 +69,7 @@ export async function registerDependencies( asyncDisposePriority: 30, enabled: queuesEnabled, }), - permissionConsumerMultiSchema: asClass(SnsSqsPermissionConsumerMultiSchema, { - lifetime: Lifetime.SINGLETON, - asyncInit: 'start', - asyncDispose: 'close', - asyncInitPriority: 30, - asyncDisposePriority: 30, - enabled: queuesEnabled, - }), - permissionPublisher: asClass(SnsPermissionPublisherMonoSchema, { - lifetime: Lifetime.SINGLETON, - asyncInit: 'init', - asyncDispose: 'close', - asyncInitPriority: 40, - asyncDisposePriority: 40, - enabled: queuesEnabled, - }), - permissionPublisherMultiSchema: asClass(SnsPermissionPublisherMultiSchema, { + permissionPublisher: asClass(SnsPermissionPublisher, { lifetime: Lifetime.SINGLETON, asyncInit: 'init', asyncDispose: 'close', @@ -131,8 +113,6 @@ export interface Dependencies { errorReporter: ErrorReporter consumerErrorResolver: ErrorResolver - permissionConsumer: SnsSqsPermissionConsumerMonoSchema - permissionConsumerMultiSchema: SnsSqsPermissionConsumerMultiSchema - permissionPublisher: SnsPermissionPublisherMonoSchema - permissionPublisherMultiSchema: SnsPermissionPublisherMultiSchema + permissionConsumer: SnsSqsPermissionConsumer + permissionPublisher: SnsPermissionPublisher } diff --git a/packages/sqs/index.ts b/packages/sqs/index.ts index 32a768c5..01ce2ad8 100644 --- a/packages/sqs/index.ts +++ b/packages/sqs/index.ts @@ -1,25 +1,14 @@ export type { - SQSQueueConfig, SQSConsumerDependencies, SQSQueueLocatorType, SQSDependencies, } from './lib/sqs/AbstractSqsService' -export { AbstractSqsConsumer } from './lib/sqs/AbstractSqsConsumer' -export type { SQSCreationConfig, ExtraSQSCreationParams } from './lib/sqs/AbstractSqsConsumer' -export { AbstractSqsConsumerMultiSchema } from './lib/sqs/AbstractSqsConsumerMultiSchema' -export { AbstractSqsConsumerMonoSchema } from './lib/sqs/AbstractSqsConsumerMonoSchema' -export type { CommonSQSConsumerOptionsMono } from './lib/sqs/AbstractSqsConsumerMonoSchema' - -export type { - NewSQSConsumerOptions, - ExistingSQSConsumerOptions, -} from './lib/sqs/AbstractSqsConsumer' +export * from './lib/sqs/AbstractSqsConsumer' export { SqsConsumerErrorResolver } from './lib/errors/SqsConsumerErrorResolver' -export { AbstractSqsPublisherMonoSchema } from './lib/sqs/AbstractSqsPublisherMonoSchema' -export { AbstractSqsPublisherMultiSchema } from './lib/sqs/AbstractSqsPublisherMultiSchema' -export type { SQSMessageOptions } from './lib/sqs/AbstractSqsPublisherMultiSchema' +export { AbstractSqsPublisher } from './lib/sqs/AbstractSqsPublisher' +export type { SQSMessageOptions } from './lib/sqs/AbstractSqsPublisher' export { assertQueue, deleteQueue, getQueueAttributes, getQueueUrl } from './lib/utils/sqsUtils' export { deleteSqs, updateQueueAttributes } from './lib/utils/sqsInitter' diff --git a/packages/sqs/lib/errors/SqsConsumerErrorResolver.ts b/packages/sqs/lib/errors/SqsConsumerErrorResolver.ts index d68dd7a7..1aa7bb26 100644 --- a/packages/sqs/lib/errors/SqsConsumerErrorResolver.ts +++ b/packages/sqs/lib/errors/SqsConsumerErrorResolver.ts @@ -26,9 +26,11 @@ export class SqsConsumerErrorResolver implements ErrorResolver { errorCode: error.code, }) } + /* c8 ignore start */ return new InternalError({ message: 'Error processing message', errorCode: 'INTERNAL_ERROR', }) + /* c8 ignore stop */ } } diff --git a/packages/sqs/lib/fakes/FakeConsumerErrorResolver.ts b/packages/sqs/lib/fakes/FakeConsumerErrorResolver.ts index 41718466..85fe2653 100644 --- a/packages/sqs/lib/fakes/FakeConsumerErrorResolver.ts +++ b/packages/sqs/lib/fakes/FakeConsumerErrorResolver.ts @@ -1,24 +1,22 @@ import { SqsConsumerErrorResolver } from '../errors/SqsConsumerErrorResolver' export class FakeConsumerErrorResolver extends SqsConsumerErrorResolver { - public errors: Error[] - - get handleErrorCallsCount() { - return this.errors.length - } + private _errors: unknown[] constructor() { super() - - this.errors = [] + this._errors = [] } public override processError(error: unknown) { - this.errors.push(error as Error) + this._errors.push(error) return super.processError(error) } + get errors() { + return this._errors + } public clear() { - this.errors = [] + this._errors = [] } } diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 55275281..926ec1f8 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -1,10 +1,18 @@ +import type { CreateQueueRequest } from '@aws-sdk/client-sqs' import type { Either, ErrorResolver } from '@lokalise/node-core' -import { isMessageError, parseMessage } from '@message-queue-toolkit/core' import type { QueueConsumer as QueueConsumer, - NewQueueOptions, - ExistingQueueOptions, TransactionObservabilityManager, + QueueConsumerOptions, + PrehandlingOutputs, + Prehandler, + BarrierResult, +} from '@message-queue-toolkit/core' +import { + isMessageError, + parseMessage, + HandlerContainer, + MessageSchemaContainer, } from '@message-queue-toolkit/core' import { Consumer } from 'sqs-consumer' import type { ConsumerOptions } from 'sqs-consumer/src/types' @@ -12,11 +20,7 @@ import type { ConsumerOptions } from 'sqs-consumer/src/types' import type { SQSMessage } from '../types/MessageTypes' import { readSqsMessage } from '../utils/sqsMessageReader' -import type { - SQSConsumerDependencies, - SQSQueueConfig, - SQSQueueLocatorType, -} from './AbstractSqsService' +import type { SQSConsumerDependencies, SQSQueueLocatorType } from './AbstractSqsService' import { AbstractSqsService } from './AbstractSqsService' const ABORT_EARLY_EITHER: Either<'abort', never> = { @@ -29,33 +33,44 @@ export type ExtraSQSCreationParams = { } export type SQSCreationConfig = { - queue: SQSQueueConfig + queue: CreateQueueRequest updateAttributesIfExists?: boolean } & ExtraSQSCreationParams -export type NewSQSConsumerOptions = - NewQueueOptions & { - consumerOverrides?: Partial - } - -export type ExistingSQSConsumerOptions< +export type SQSConsumerOptions< + MessagePayloadSchemas extends object, + ExecutionContext, + PrehandlerOutput, + CreationConfigType extends SQSCreationConfig = SQSCreationConfig, QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, -> = ExistingQueueOptions & { +> = QueueConsumerOptions< + CreationConfigType, + QueueLocatorType, + MessagePayloadSchemas, + ExecutionContext, + PrehandlerOutput +> & { consumerOverrides?: Partial } - export abstract class AbstractSqsConsumer< MessagePayloadType extends object, - QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, + ExecutionContext, + PrehandlerOutput = undefined, CreationConfigType extends SQSCreationConfig = SQSCreationConfig, - ConsumerOptionsType extends - | NewSQSConsumerOptions - | ExistingSQSConsumerOptions = - | NewSQSConsumerOptions - | ExistingSQSConsumerOptions, - ExecutionContext = unknown, - PrehandlerOutput = unknown, - BarrierOutput = unknown, + QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, + ConsumerOptionsType extends SQSConsumerOptions< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput, + CreationConfigType, + QueueLocatorType + > = SQSConsumerOptions< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput, + CreationConfigType, + QueueLocatorType + >, > extends AbstractSqsService< MessagePayloadType, @@ -64,23 +79,129 @@ export abstract class AbstractSqsConsumer< ConsumerOptionsType, SQSConsumerDependencies, ExecutionContext, - PrehandlerOutput, - BarrierOutput + PrehandlerOutput > implements QueueConsumer { + private consumer?: Consumer private readonly transactionObservabilityManager?: TransactionObservabilityManager - protected readonly errorResolver: ErrorResolver - // @ts-ignore - protected consumer: Consumer + private readonly consumerOptionsOverride: Partial + private readonly handlerContainer: HandlerContainer< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput + > - protected constructor(dependencies: SQSConsumerDependencies, options: ConsumerOptionsType) { + protected readonly errorResolver: ErrorResolver + protected readonly messageSchemaContainer: MessageSchemaContainer + protected readonly executionContext: ExecutionContext + + protected constructor( + dependencies: SQSConsumerDependencies, + options: ConsumerOptionsType, + executionContext: ExecutionContext, + ) { super(dependencies, options) this.transactionObservabilityManager = dependencies.transactionObservabilityManager this.errorResolver = dependencies.consumerErrorResolver this.consumerOptionsOverride = options.consumerOverrides ?? {} + const messageSchemas = options.handlers.map((entry) => entry.schema) + + this.messageSchemaContainer = new MessageSchemaContainer({ + messageSchemas, + messageTypeField: options.messageTypeField, + }) + this.handlerContainer = new HandlerContainer< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput + >({ + messageTypeField: this.messageTypeField, + messageHandlers: options.handlers, + }) + this.executionContext = executionContext + } + + public async start() { + await this.init() + + if (this.consumer) { + this.consumer.stop() + } + this.consumer = Consumer.create({ + queueUrl: this.queueUrl, + handleMessage: async (message: SQSMessage) => { + /* c8 ignore next */ + if (message === null) return + + const deserializedMessage = this.deserializeMessage(message) + if (deserializedMessage.error === 'abort') { + await this.failProcessing(message) + + const messageId = this.tryToExtractId(message) + this.handleMessageProcessed(null, 'invalid_message', messageId.result) + return + } + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + const messageType = deserializedMessage.result[this.messageTypeField] + const transactionSpanId = `queue_${this.queueName}:${messageType}` + + this.transactionObservabilityManager?.start(transactionSpanId) + if (this.logMessages) { + const resolvedLogMessage = this.resolveMessageLog(deserializedMessage.result, messageType) + this.logMessage(resolvedLogMessage) + } + const result: Either<'retryLater' | Error, 'success'> = await this.internalProcessMessage( + deserializedMessage.result, + messageType, + ) + .catch((err) => { + // ToDo we need sanity check to stop trying at some point, perhaps some kind of Redis counter + // If we fail due to unknown reason, let's retry + this.handleError(err) + return { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + error: err, + } + }) + .finally(() => { + this.transactionObservabilityManager?.stop(transactionSpanId) + }) + + // success + if (result.result) { + this.handleMessageProcessed(deserializedMessage.result, 'consumed') + return message + } + + // failure + this.handleMessageProcessed( + deserializedMessage.result, + result.error === 'retryLater' ? 'retryLater' : 'error', + ) + return Promise.reject(result.error) + }, + sqs: this.sqsClient, + ...this.consumerOptionsOverride, + }) + + this.consumer.on('error', (err) => { + this.handleError(err, { + queueName: this.queueName, + }) + }) + + this.consumer.start() + } + + public override async close(abort?: boolean): Promise { + await super.close() + this.consumer?.stop({ + abort: abort ?? false, + }) } private async internalProcessMessage( @@ -96,23 +217,86 @@ export abstract class AbstractSqsConsumer< barrierOutput: barrierResult.output, }) } + return { error: 'retryLater' } } - private tryToExtractId(message: SQSMessage): Either<'abort', string> { - if (message === null) { - return ABORT_EARLY_EITHER - } + protected override async processMessage( + message: MessagePayloadType, + messageType: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prehandlingOutputs: PrehandlingOutputs, + ): Promise> { + const handler = this.handlerContainer.resolveHandler(messageType) + + return handler.handler(message, this.executionContext, prehandlingOutputs) + } + + protected override processPrehandlers(message: MessagePayloadType, messageType: string) { + const handlerConfig = this.handlerContainer.resolveHandler(messageType) + + return this.processPrehandlersInternal(handlerConfig.prehandlers, message) + } + protected override async preHandlerBarrier( + message: MessagePayloadType, + messageType: string, + prehandlerOutput: PrehandlerOutput, + ): Promise> { + const handler = this.handlerContainer.resolveHandler( + messageType, + ) + + return this.preHandlerBarrierInternal( + handler.preHandlerBarrier, + message, + this.executionContext, + prehandlerOutput, + ) + } + + protected override resolveSchema(message: MessagePayloadType) { + return this.messageSchemaContainer.resolveSchema(message) + } + + // eslint-disable-next-line max-params + protected override resolveNextFunction( + prehandlers: Prehandler[], + message: MessagePayloadType, + index: number, + prehandlerOutput: PrehandlerOutput, + resolve: (value: PrehandlerOutput | PromiseLike) => void, + reject: (err: Error) => void, + ) { + return this.resolveNextPreHandlerFunctionInternal( + prehandlers, + this.executionContext, + message, + index, + prehandlerOutput, + resolve, + reject, + ) + } + + protected override resolveMessageLog(message: MessagePayloadType, messageType: string): unknown { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.messageLogFormatter(message) + } + + protected override resolveMessage(message: SQSMessage) { + return readSqsMessage(message, this.errorResolver) + } + + private tryToExtractId(message: SQSMessage): Either<'abort', string> { const resolveMessageResult = this.resolveMessage(message) if (isMessageError(resolveMessageResult.error)) { this.handleError(resolveMessageResult.error) return ABORT_EARLY_EITHER } // Empty content for whatever reason - if (!resolveMessageResult.result) { - return ABORT_EARLY_EITHER - } + /* c8 ignore next */ + if (!resolveMessageResult.result) return ABORT_EARLY_EITHER // @ts-ignore if (this.messageIdField in resolveMessageResult.result) { @@ -171,89 +355,4 @@ export abstract class AbstractSqsConsumer< private async failProcessing(_message: SQSMessage) { // Not implemented yet - needs dead letter queue } - - async start() { - await this.init() - - if (this.consumer) { - this.consumer.stop() - } - this.consumer = Consumer.create({ - queueUrl: this.queueUrl, - handleMessage: async (message: SQSMessage) => { - if (message === null) { - return - } - - const deserializedMessage = this.deserializeMessage(message) - if (deserializedMessage.error === 'abort') { - await this.failProcessing(message) - - const messageId = this.tryToExtractId(message) - this.handleMessageProcessed(null, 'invalid_message', messageId.result) - return - } - // @ts-ignore - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - const messageType = deserializedMessage.result[this.messageTypeField] - const transactionSpanId = `queue_${this.queueName}:${messageType}` - - this.transactionObservabilityManager?.start(transactionSpanId) - if (this.logMessages) { - const resolvedLogMessage = this.resolveMessageLog(deserializedMessage.result, messageType) - this.logMessage(resolvedLogMessage) - } - const result: Either<'retryLater' | Error, 'success'> = await this.internalProcessMessage( - deserializedMessage.result, - messageType, - ) - .catch((err) => { - // ToDo we need sanity check to stop trying at some point, perhaps some kind of Redis counter - // If we fail due to unknown reason, let's retry - this.handleError(err) - return { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - error: err, - } - }) - .finally(() => { - this.transactionObservabilityManager?.stop(transactionSpanId) - }) - - // success - if (result.result) { - this.handleMessageProcessed(deserializedMessage.result, 'consumed') - return message - } - - // failure - this.handleMessageProcessed( - deserializedMessage.result, - result.error === 'retryLater' ? 'retryLater' : 'error', - ) - return Promise.reject(result.error) - }, - sqs: this.sqsClient, - ...this.consumerOptionsOverride, - }) - - this.consumer.on('error', (err) => { - this.handleError(err, { - queueName: this.queueName, - }) - }) - - this.consumer.start() - } - - protected override resolveMessage(message: SQSMessage) { - return readSqsMessage(message, this.errorResolver) - } - - public override async close(abort?: boolean): Promise { - await super.close() - this.consumer?.stop({ - abort: abort ?? false, - }) - } } diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts b/packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts deleted file mode 100644 index 774cf7a7..00000000 --- a/packages/sqs/lib/sqs/AbstractSqsConsumerMonoSchema.ts +++ /dev/null @@ -1,151 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - MonoSchemaQueueOptions, - QueueConsumer as QueueConsumer, - BarrierResult, - Prehandler, - PrehandlingOutputs, -} from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' - -import type { - ExistingSQSConsumerOptions, - NewSQSConsumerOptions, - SQSCreationConfig, -} from './AbstractSqsConsumer' -import { AbstractSqsConsumer } from './AbstractSqsConsumer' -import type { SQSConsumerDependencies, SQSQueueLocatorType } from './AbstractSqsService' - -const DEFAULT_BARRIER_RESULT = { - isPassing: true, - output: undefined, -} as const - -export type CommonSQSConsumerOptionsMono< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput = undefined, -> = { - prehandlers?: Prehandler[] -} - -export type NewSQSConsumerOptionsMono< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput = undefined, - CreationConfigType extends SQSCreationConfig = SQSCreationConfig, -> = NewSQSConsumerOptions & - MonoSchemaQueueOptions & - CommonSQSConsumerOptionsMono - -export type ExistingSQSConsumerOptionsMono< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput = undefined, - QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, -> = ExistingSQSConsumerOptions & - MonoSchemaQueueOptions & - CommonSQSConsumerOptionsMono - -export abstract class AbstractSqsConsumerMonoSchema< - MessagePayloadType extends object, - PrehandlerOutput = undefined, - ExecutionContext = undefined, - BarrierOutput = undefined, - QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, - CreationConfigType extends SQSCreationConfig = SQSCreationConfig, - ConsumerOptionsType extends - | NewSQSConsumerOptionsMono< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - CreationConfigType - > - | ExistingSQSConsumerOptionsMono< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - QueueLocatorType - > = - | NewSQSConsumerOptionsMono< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - CreationConfigType - > - | ExistingSQSConsumerOptionsMono< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - QueueLocatorType - >, - > - extends AbstractSqsConsumer< - MessagePayloadType, - QueueLocatorType, - CreationConfigType, - ConsumerOptionsType, - ExecutionContext, - PrehandlerOutput, - BarrierOutput - > - implements QueueConsumer -{ - private readonly messageSchema: ZodSchema - private readonly schemaEither: Either> - private readonly prehandlers: Prehandler[] - - protected constructor(dependencies: SQSConsumerDependencies, options: ConsumerOptionsType) { - super(dependencies, options) - - this.prehandlers = options.prehandlers ?? [] - this.messageSchema = options.messageSchema - this.schemaEither = { - result: this.messageSchema, - } - } - - /** - * Override to implement barrier pattern - */ - /* c8 ignore start */ - protected override preHandlerBarrier(_message: MessagePayloadType, _messageType: string) { - // @ts-ignore - return Promise.resolve(DEFAULT_BARRIER_RESULT as BarrierResult) - } - /* c8 ignore end */ - - abstract override processMessage( - message: MessagePayloadType, - messageType: string, - prehandlingOutputs: PrehandlingOutputs, - ): Promise> - - protected override processPrehandlers(message: MessagePayloadType, _messageType: string) { - return this.processPrehandlersInternal(this.prehandlers, message) - } - - // eslint-disable-next-line max-params - protected override resolveNextFunction( - prehandlers: Prehandler[], - message: MessagePayloadType, - index: number, - prehandlerOutput: PrehandlerOutput, - resolve: (value: PrehandlerOutput | PromiseLike) => void, - reject: (err: Error) => void, - ) { - return this.resolveNextPreHandlerFunctionInternal( - prehandlers, - this as unknown as ExecutionContext, - message, - index, - prehandlerOutput, - resolve, - reject, - ) - } - - protected resolveSchema() { - return this.schemaEither - } -} diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts b/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts deleted file mode 100644 index b71350ff..00000000 --- a/packages/sqs/lib/sqs/AbstractSqsConsumerMultiSchema.ts +++ /dev/null @@ -1,160 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import { HandlerContainer, MessageSchemaContainer } from '@message-queue-toolkit/core' -import type { - ExistingQueueOptionsMultiSchema, - NewQueueOptionsMultiSchema, - BarrierResult, - Prehandler, - PrehandlingOutputs, -} from '@message-queue-toolkit/core' -import type { ConsumerOptions } from 'sqs-consumer/src/types' - -import type { SQSCreationConfig } from './AbstractSqsConsumer' -import { AbstractSqsConsumer } from './AbstractSqsConsumer' -import type { SQSConsumerDependencies, SQSQueueLocatorType } from './AbstractSqsService' - -export type NewSQSConsumerOptionsMultiSchema< - MessagePayloadSchemas extends object, - ExecutionContext, - PrehandlerOutput, - CreationConfigType extends SQSCreationConfig, -> = NewQueueOptionsMultiSchema< - MessagePayloadSchemas, - CreationConfigType, - ExecutionContext, - PrehandlerOutput -> & { - consumerOverrides?: Partial -} - -export type ExistingSQSConsumerOptionsMultiSchema< - MessagePayloadSchemas extends object, - ExecutionContext, - PrehandlerOutput, - QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, -> = ExistingQueueOptionsMultiSchema< - MessagePayloadSchemas, - QueueLocatorType, - ExecutionContext, - PrehandlerOutput -> & { - consumerOverrides?: Partial -} - -export abstract class AbstractSqsConsumerMultiSchema< - MessagePayloadType extends object, - ExecutionContext, - PrehandlerOutput = undefined, - QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, - CreationConfigType extends SQSCreationConfig = SQSCreationConfig, - ConsumerOptionsType extends NewSQSConsumerOptionsMultiSchema< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - CreationConfigType - > = NewSQSConsumerOptionsMultiSchema< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput, - CreationConfigType - >, -> extends AbstractSqsConsumer< - MessagePayloadType, - QueueLocatorType, - CreationConfigType, - ConsumerOptionsType, - ExecutionContext, - PrehandlerOutput -> { - messageSchemaContainer: MessageSchemaContainer - handlerContainer: HandlerContainer - protected readonly executionContext: ExecutionContext - - constructor( - dependencies: SQSConsumerDependencies, - options: ConsumerOptionsType, - executionContext: ExecutionContext, - ) { - super(dependencies, options) - - const messageSchemas = options.handlers.map((entry) => entry.schema) - - this.messageSchemaContainer = new MessageSchemaContainer({ - messageSchemas, - messageTypeField: options.messageTypeField, - }) - this.handlerContainer = new HandlerContainer< - MessagePayloadType, - ExecutionContext, - PrehandlerOutput - >({ - messageTypeField: this.messageTypeField, - messageHandlers: options.handlers, - }) - this.executionContext = executionContext - } - - protected override resolveSchema(message: MessagePayloadType) { - return this.messageSchemaContainer.resolveSchema(message) - } - - public override async processMessage( - message: MessagePayloadType, - messageType: string, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - prehandlingOutputs: PrehandlingOutputs, - ): Promise> { - const handler = this.handlerContainer.resolveHandler(messageType) - - return handler.handler(message, this.executionContext, prehandlingOutputs) - } - - protected override processPrehandlers(message: MessagePayloadType, messageType: string) { - const handlerConfig = this.handlerContainer.resolveHandler(messageType) - - return this.processPrehandlersInternal(handlerConfig.prehandlers, message) - } - - // eslint-disable-next-line max-params - protected override resolveNextFunction( - prehandlers: Prehandler[], - message: MessagePayloadType, - index: number, - prehandlerOutput: PrehandlerOutput, - resolve: (value: PrehandlerOutput | PromiseLike) => void, - reject: (err: Error) => void, - ) { - return this.resolveNextPreHandlerFunctionInternal( - prehandlers, - this.executionContext, - message, - index, - prehandlerOutput, - resolve, - reject, - ) - } - - protected override resolveMessageLog(message: MessagePayloadType, messageType: string): unknown { - const handler = this.handlerContainer.resolveHandler(messageType) - return handler.messageLogFormatter(message) - } - - protected override async preHandlerBarrier( - message: MessagePayloadType, - messageType: string, - prehandlerOutput: PrehandlerOutput, - ): Promise> { - const handler = this.handlerContainer.resolveHandler( - messageType, - ) - // @ts-ignore - return handler.preHandlerBarrier - ? // @ts-ignore - await handler.preHandlerBarrier(message, this.executionContext, prehandlerOutput) - : { - isPassing: true, - output: undefined, - } - } -} diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisherMultiSchema.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts similarity index 61% rename from packages/sqs/lib/sqs/AbstractSqsPublisherMultiSchema.ts rename to packages/sqs/lib/sqs/AbstractSqsPublisher.ts index f73e5054..60c24c66 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisherMultiSchema.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -1,12 +1,12 @@ +import type { SendMessageCommandInput } from '@aws-sdk/client-sqs' +import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import type { AsyncPublisher, MessageInvalidFormatError, MessageValidationError, - ExistingQueueOptions, - NewQueueOptions, - MultiSchemaPublisherOptions, BarrierResult, + QueuePublisherOptions, } from '@message-queue-toolkit/core' import { MessageSchemaContainer } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod' @@ -22,16 +22,17 @@ export type SQSMessageOptions = { MessageDeduplicationId?: string } -export abstract class AbstractSqsPublisherMultiSchema +export abstract class AbstractSqsPublisher extends AbstractSqsService implements AsyncPublisher { private readonly messageSchemaContainer: MessageSchemaContainer + private isInitted: boolean + private initPromise?: Promise constructor( dependencies: SQSDependencies, - options: (NewQueueOptions | ExistingQueueOptions) & - MultiSchemaPublisherOptions, + options: QueuePublisherOptions, ) { super(dependencies, options) @@ -40,6 +41,7 @@ export abstract class AbstractSqsPublisherMultiSchema { @@ -48,7 +50,37 @@ export abstract class AbstractSqsPublisherMultiSchema> { + protected override preHandlerBarrier(): Promise> { throw new Error('Not implemented for publisher') } diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts b/packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts deleted file mode 100644 index ca5c7ada..00000000 --- a/packages/sqs/lib/sqs/AbstractSqsPublisherMonoSchema.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { - AsyncPublisher, - MessageInvalidFormatError, - MessageValidationError, - ExistingQueueOptions, - MonoSchemaQueueOptions, - NewQueueOptions, - BarrierResult, -} from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' - -import type { SQSCreationConfig } from './AbstractSqsConsumer' -import type { SQSMessageOptions } from './AbstractSqsPublisherMultiSchema' -import type { SQSDependencies, SQSQueueLocatorType } from './AbstractSqsService' -import { AbstractSqsService } from './AbstractSqsService' - -export abstract class AbstractSqsPublisherMonoSchema - extends AbstractSqsService - implements AsyncPublisher -{ - private readonly messageSchema: ZodSchema - private readonly schemaEither: Either> - - constructor( - dependencies: SQSDependencies, - options: (NewQueueOptions | ExistingQueueOptions) & - MonoSchemaQueueOptions, - ) { - super(dependencies, options) - this.messageSchema = options.messageSchema - - this.schemaEither = { - result: this.messageSchema, - } - } - - async publish(message: MessagePayloadType, options: SQSMessageOptions = {}): Promise { - return this.internalPublish(message, this.messageSchema, options) - } - - /* c8 ignore start */ - protected override resolveNextFunction(): () => void { - throw new Error('Not implemented for publisher') - } - - protected override resolveMessage(): Either< - MessageInvalidFormatError | MessageValidationError, - unknown - > { - throw new Error('Not implemented for publisher') - } - - protected override processPrehandlers(): Promise { - throw new Error('Not implemented for publisher') - } - - protected override preHandlerBarrier(): Promise> { - throw new Error('Not implemented for publisher') - } - - override processMessage(): Promise> { - throw new Error('Not implemented for publisher') - } - - protected override resolveSchema() { - return this.schemaEither - } - /* c8 ignore stop */ -} diff --git a/packages/sqs/lib/sqs/AbstractSqsService.ts b/packages/sqs/lib/sqs/AbstractSqsService.ts index ed7d8be3..ed5054b6 100644 --- a/packages/sqs/lib/sqs/AbstractSqsService.ts +++ b/packages/sqs/lib/sqs/AbstractSqsService.ts @@ -1,19 +1,15 @@ -import type { SQSClient, CreateQueueRequest, SendMessageCommandInput } from '@aws-sdk/client-sqs' -import { SendMessageCommand } from '@aws-sdk/client-sqs' +import type { SQSClient } from '@aws-sdk/client-sqs' import type { QueueConsumerDependencies, QueueDependencies, - NewQueueOptions, - ExistingQueueOptions, + QueueOptions, } from '@message-queue-toolkit/core' import { AbstractQueueService } from '@message-queue-toolkit/core' -import type { ZodSchema } from 'zod' import type { SQSMessage } from '../types/MessageTypes' import { deleteSqs, initSqs } from '../utils/sqsInitter' import type { SQSCreationConfig } from './AbstractSqsConsumer' -import type { SQSMessageOptions } from './AbstractSqsPublisherMultiSchema' export type SQSDependencies = QueueDependencies & { sqsClient: SQSClient @@ -21,8 +17,6 @@ export type SQSDependencies = QueueDependencies & { export type SQSConsumerDependencies = SQSDependencies & QueueConsumerDependencies -export type SQSQueueConfig = CreateQueueRequest - export type SQSQueueLocatorType = { queueUrl: string } @@ -31,15 +25,13 @@ export abstract class AbstractSqsService< MessagePayloadType extends object, QueueLocatorType extends SQSQueueLocatorType = SQSQueueLocatorType, CreationConfigType extends SQSCreationConfig = SQSCreationConfig, - SQSOptionsType extends - | NewQueueOptions - | ExistingQueueOptions = - | NewQueueOptions - | ExistingQueueOptions, + SQSOptionsType extends QueueOptions = QueueOptions< + CreationConfigType, + QueueLocatorType + >, DependenciesType extends SQSDependencies = SQSDependencies, PrehandlerOutput = unknown, ExecutionContext = unknown, - BarrierOutput = unknown, > extends AbstractQueueService< MessagePayloadType, SQSMessage, @@ -48,24 +40,19 @@ export abstract class AbstractSqsService< QueueLocatorType, SQSOptionsType, PrehandlerOutput, - ExecutionContext, - BarrierOutput + ExecutionContext > { protected readonly sqsClient: SQSClient + // @ts-ignore - public queueUrl: string + protected queueName: string // @ts-ignore - public queueName: string + protected queueUrl: string // @ts-ignore - public queueArn: string - - private isInitted: boolean - private initPromise?: Promise + protected queueArn: string constructor(dependencies: DependenciesType, options: SQSOptionsType) { super(dependencies, options) - - this.isInitted = false this.sqsClient = dependencies.sqsClient } @@ -73,54 +60,14 @@ export abstract class AbstractSqsService< if (this.deletionConfig && this.creationConfig) { await deleteSqs(this.sqsClient, this.deletionConfig, this.creationConfig) } - const { queueUrl, queueName, queueArn } = await initSqs( + const { queueName, queueUrl, queueArn } = await initSqs( this.sqsClient, this.locatorConfig, this.creationConfig, ) - - this.queueArn = queueArn - this.queueUrl = queueUrl this.queueName = queueName - this.isInitted = true - } - - protected async internalPublish( - message: MessagePayloadType, - messageSchema: ZodSchema, - options: SQSMessageOptions = {}, - ): Promise { - // If it's not initted yet, do the lazy init - if (!this.isInitted) { - // avoid multiple concurrent inits - if (!this.initPromise) { - this.initPromise = this.init() - } - await this.initPromise - } - - try { - messageSchema.parse(message) - - if (this.logMessages) { - // @ts-ignore - const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) - this.logMessage(resolvedLogMessage) - } - - const input = { - // SendMessageRequest - QueueUrl: this.queueUrl, - MessageBody: JSON.stringify(message), - ...options, - } satisfies SendMessageCommandInput - const command = new SendMessageCommand(input) - await this.sqsClient.send(command) - this.handleMessageProcessed(message, 'published') - } catch (error) { - this.handleError(error) - throw error - } + this.queueUrl = queueUrl + this.queueArn = queueArn } // eslint-disable-next-line @typescript-eslint/require-await diff --git a/packages/sqs/lib/utils/sqsInitter.ts b/packages/sqs/lib/utils/sqsInitter.ts index b40e56d0..8f7fee82 100644 --- a/packages/sqs/lib/utils/sqsInitter.ts +++ b/packages/sqs/lib/utils/sqsInitter.ts @@ -67,11 +67,7 @@ export async function initSqs( const splitUrl = queueUrl.split('/') const queueName = splitUrl[splitUrl.length - 1] - return { - queueArn, - queueUrl, - queueName, - } + return { queueArn, queueUrl, queueName } } // create new queue if does not exist @@ -86,9 +82,5 @@ export async function initSqs( }) const queueName = creationConfig.queue.QueueName - return { - queueUrl, - queueArn, - queueName, - } + return { queueUrl, queueArn, queueName } } diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 793f0384..f29b635d 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "11.1.0", + "version": "12.0.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", diff --git a/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.spec.ts similarity index 72% rename from packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts rename to packages/sqs/test/consumers/SqsPermissionConsumer.spec.ts index 567e48d1..2f52a0a3 100644 --- a/packages/sqs/test/consumers/SqsPermissionsConsumerMultiSchema.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.spec.ts @@ -1,20 +1,22 @@ -import type { SQSClient } from '@aws-sdk/client-sqs' -import { ReceiveMessageCommand } from '@aws-sdk/client-sqs' +import type { SendMessageCommandInput, SQSClient } from '@aws-sdk/client-sqs' +import { SendMessageCommand, ReceiveMessageCommand } from '@aws-sdk/client-sqs' +import { waitAndRetry } from '@lokalise/node-core' import type { BarrierResult } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asClass, asFunction } from 'awilix' import { describe, beforeEach, afterEach, expect, it } from 'vitest' +import { ZodError } from 'zod' import { FakeConsumerErrorResolver } from '../../lib/fakes/FakeConsumerErrorResolver' import { assertQueue, deleteQueue, getQueueAttributes } from '../../lib/utils/sqsUtils' import { FakeLogger } from '../fakes/FakeLogger' -import type { SqsPermissionPublisherMultiSchema } from '../publishers/SqsPermissionPublisherMultiSchema' +import type { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher' import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' import type { Dependencies } from '../utils/testContext' -import { SqsPermissionConsumerMultiSchema } from './SqsPermissionConsumerMultiSchema' +import { SqsPermissionConsumer } from './SqsPermissionConsumer' -describe('SqsPermissionsConsumerMultiSchema', () => { +describe('SqsPermissionConsumer', () => { describe('init', () => { let diContainer: AwilixContainer let sqsClient: SQSClient @@ -30,7 +32,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { }) it('throws an error when invalid queue locator is passed', async () => { - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { locatorConfig: { queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', }, @@ -44,14 +46,14 @@ describe('SqsPermissionsConsumerMultiSchema', () => { QueueName: 'existingQueue', }) - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { locatorConfig: { queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', }, }) await newConsumer.init() - expect(newConsumer.queueUrl).toBe( + expect(newConsumer.queueProps.url).toBe( 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', ) }) @@ -64,7 +66,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { }, }) - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { QueueName: 'existingQueue', @@ -83,7 +85,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { const sqsSpy = vi.spyOn(sqsClient, 'send') await newConsumer.init() - expect(newConsumer.queueUrl).toBe( + expect(newConsumer.queueProps.url).toBe( 'http://sqs.eu-west-1.localstack:4566/000000000000/existingQueue', ) @@ -93,7 +95,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { expect(updateCall).toBeDefined() const attributes = await getQueueAttributes(sqsClient, { - queueUrl: newConsumer.queueUrl, + queueUrl: newConsumer.queueProps.url, }) expect(attributes.result?.attributes!.KmsMasterKeyId).toBe('othervalue') @@ -107,7 +109,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { }, }) - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { QueueName: 'existingQueue', @@ -126,7 +128,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { const sqsSpy = vi.spyOn(sqsClient, 'send') await newConsumer.init() - expect(newConsumer.queueUrl).toBe( + expect(newConsumer.queueProps.url).toBe( 'http://sqs.eu-west-1.localstack:4566/000000000000/existingQueue', ) @@ -136,7 +138,7 @@ describe('SqsPermissionsConsumerMultiSchema', () => { expect(updateCall).toBeUndefined() const attributes = await getQueueAttributes(sqsClient, { - queueUrl: newConsumer.queueUrl, + queueUrl: newConsumer.queueProps.url, }) expect(attributes.result?.attributes!.KmsMasterKeyId).toBe('somevalue') @@ -146,15 +148,15 @@ describe('SqsPermissionsConsumerMultiSchema', () => { describe('logging', () => { let logger: FakeLogger let diContainer: AwilixContainer - let publisher: SqsPermissionPublisherMultiSchema + let publisher: SqsPermissionPublisher beforeEach(async () => { logger = new FakeLogger() diContainer = await registerDependencies({ logger: asFunction(() => logger), }) - await diContainer.cradle.permissionConsumerMultiSchema.close() - publisher = diContainer.cradle.permissionPublisherMultiSchema + await diContainer.cradle.permissionConsumer.close() + publisher = diContainer.cradle.permissionPublisher }) afterEach(async () => { @@ -163,10 +165,10 @@ describe('SqsPermissionsConsumerMultiSchema', () => { }) it('logs a message when logging is enabled', async () => { - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { - QueueName: publisher.queueName, + QueueName: publisher.queueProps.name, }, }, logMessages: true, @@ -199,11 +201,11 @@ describe('SqsPermissionsConsumerMultiSchema', () => { describe('preHandlerBarrier', () => { let diContainer: AwilixContainer - let publisher: SqsPermissionPublisherMultiSchema + let publisher: SqsPermissionPublisher beforeEach(async () => { diContainer = await registerDependencies() - await diContainer.cradle.permissionConsumerMultiSchema.close() - publisher = diContainer.cradle.permissionPublisherMultiSchema + await diContainer.cradle.permissionConsumer.close() + publisher = diContainer.cradle.permissionPublisher }) afterEach(async () => { @@ -213,10 +215,10 @@ describe('SqsPermissionsConsumerMultiSchema', () => { it('blocks first try', async () => { let barrierCounter = 0 - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { - QueueName: publisher.queueName, + QueueName: publisher.queueProps.name, }, }, addPreHandlerBarrier: async (_msg): Promise> => { @@ -246,10 +248,10 @@ describe('SqsPermissionsConsumerMultiSchema', () => { it('can access prehandler output', async () => { expect.assertions(1) - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { - QueueName: publisher.queueName, + QueueName: publisher.queueProps.name, }, }, addPreHandlerBarrier: async ( @@ -275,10 +277,10 @@ describe('SqsPermissionsConsumerMultiSchema', () => { it('throws an error on first try', async () => { let barrierCounter = 0 - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { - QueueName: publisher.queueName, + QueueName: publisher.queueProps.name, }, }, addPreHandlerBarrier: (_msg) => { @@ -306,11 +308,11 @@ describe('SqsPermissionsConsumerMultiSchema', () => { describe('prehandlers', () => { let diContainer: AwilixContainer - let publisher: SqsPermissionPublisherMultiSchema + let publisher: SqsPermissionPublisher beforeEach(async () => { diContainer = await registerDependencies() - await diContainer.cradle.permissionConsumerMultiSchema.close() - publisher = diContainer.cradle.permissionPublisherMultiSchema + await diContainer.cradle.permissionConsumer.close() + publisher = diContainer.cradle.permissionPublisher }) afterEach(async () => { @@ -321,10 +323,10 @@ describe('SqsPermissionsConsumerMultiSchema', () => { it('processes one prehandler', async () => { expect.assertions(1) - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { - QueueName: publisher.queueName, + QueueName: publisher.queueProps.name, }, }, removeHandlerOverride: async (message, _context, prehandlerOutputs) => { @@ -357,10 +359,10 @@ describe('SqsPermissionsConsumerMultiSchema', () => { it('processes two prehandlers', async () => { expect.assertions(1) - const newConsumer = new SqsPermissionConsumerMultiSchema(diContainer.cradle, { + const newConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { - QueueName: publisher.queueName, + QueueName: publisher.queueProps.name, }, }, removeHandlerOverride: async (message, _context, prehandlerOutputs) => { @@ -400,26 +402,28 @@ describe('SqsPermissionsConsumerMultiSchema', () => { describe('consume', () => { let diContainer: AwilixContainer - let publisher: SqsPermissionPublisherMultiSchema - let consumer: SqsPermissionConsumerMultiSchema let sqsClient: SQSClient + + let publisher: SqsPermissionPublisher + let consumer: SqsPermissionConsumer + let errorResolver: FakeConsumerErrorResolver + beforeEach(async () => { diContainer = await registerDependencies({ consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), }) sqsClient = diContainer.cradle.sqsClient - publisher = diContainer.cradle.permissionPublisherMultiSchema - consumer = diContainer.cradle.permissionConsumerMultiSchema + publisher = diContainer.cradle.permissionPublisher + consumer = diContainer.cradle.permissionConsumer const command = new ReceiveMessageCommand({ - QueueUrl: publisher.queueUrl, + QueueUrl: publisher.queueProps.url, }) const reply = await sqsClient.send(command) expect(reply.Messages).toBeUndefined() - const fakeErrorResolver = diContainer.cradle - .consumerErrorResolver as FakeConsumerErrorResolver - fakeErrorResolver.clear() + errorResolver = diContainer.cradle.consumerErrorResolver as FakeConsumerErrorResolver + errorResolver.clear() }) afterEach(async () => { @@ -427,28 +431,48 @@ describe('SqsPermissionsConsumerMultiSchema', () => { await diContainer.dispose() }) - describe('happy path', () => { - it('Processes messages', async () => { - await publisher.publish({ - id: '10', - messageType: 'add', - }) - await publisher.publish({ - id: '20', - messageType: 'remove', - }) - await publisher.publish({ - id: '30', - messageType: 'remove', - }) - - await consumer.handlerSpy.waitForMessageWithId('10', 'consumed') - await consumer.handlerSpy.waitForMessageWithId('20', 'consumed') - await consumer.handlerSpy.waitForMessageWithId('30', 'consumed') - - expect(consumer.addCounter).toBe(1) - expect(consumer.removeCounter).toBe(2) + it('bad event', async () => { + const message = { + messageType: 'add', + } + + // not using publisher to avoid publisher validation + const input = { + QueueUrl: consumer.queueProps.url, + MessageBody: JSON.stringify(message), + } satisfies SendMessageCommandInput + const command = new SendMessageCommand(input) + await sqsClient.send(command) + + await waitAndRetry(() => errorResolver.errors.length > 0, 100, 5) + + expect(errorResolver.errors).toHaveLength(1) + expect(errorResolver.errors[0] instanceof ZodError).toBe(true) + + expect(consumer.addCounter).toBe(0) + expect(consumer.removeCounter).toBe(0) + }) + + it('Processes messages', async () => { + await publisher.publish({ + id: '10', + messageType: 'add', }) + await publisher.publish({ + id: '20', + messageType: 'remove', + }) + await publisher.publish({ + id: '30', + messageType: 'remove', + }) + + await consumer.handlerSpy.waitForMessageWithId('10', 'consumed') + await consumer.handlerSpy.waitForMessageWithId('20', 'consumed') + await consumer.handlerSpy.waitForMessageWithId('30', 'consumed') + + expect(consumer.addCounter).toBe(1) + expect(consumer.removeCounter).toBe(2) }) }) }) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.ts similarity index 71% rename from packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts rename to packages/sqs/test/consumers/SqsPermissionConsumer.ts index bbe823ca..1527c192 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumerMultiSchema.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.ts @@ -2,12 +2,8 @@ import type { Either } from '@lokalise/node-core' import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' import type { BarrierResult, Prehandler, PrehandlingOutputs } from '@message-queue-toolkit/core' -import type { SQSCreationConfig } from '../../lib/sqs/AbstractSqsConsumer' -import type { - ExistingSQSConsumerOptionsMultiSchema, - NewSQSConsumerOptionsMultiSchema, -} from '../../lib/sqs/AbstractSqsConsumerMultiSchema' -import { AbstractSqsConsumerMultiSchema } from '../../lib/sqs/AbstractSqsConsumerMultiSchema' +import type { SQSConsumerOptions } from '../../lib/sqs/AbstractSqsConsumer' +import { AbstractSqsConsumer } from '../../lib/sqs/AbstractSqsConsumer' import type { SQSConsumerDependencies } from '../../lib/sqs/AbstractSqsService' import type { @@ -19,21 +15,12 @@ import { PERMISSIONS_REMOVE_MESSAGE_SCHEMA, } from './userConsumerSchemas' -type SqsPermissionConsumerMultiSchemaOptions = ( - | Pick< - NewSQSConsumerOptionsMultiSchema< - SupportedMessages, - ExecutionContext, - PrehandlerOutput, - SQSCreationConfig - >, - 'creationConfig' | 'logMessages' | 'deletionConfig' - > - | Pick< - ExistingSQSConsumerOptionsMultiSchema, - 'locatorConfig' | 'logMessages' - > -) & { +type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE + +type SqsPermissionConsumerOptions = Pick< + SQSConsumerOptions, + 'creationConfig' | 'locatorConfig' | 'logMessages' | 'deletionConfig' +> & { addPreHandlerBarrier?: ( message: SupportedMessages, _executionContext: ExecutionContext, @@ -47,7 +34,6 @@ type SqsPermissionConsumerMultiSchemaOptions = ( removePreHandlers?: Prehandler[] } -type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE type ExecutionContext = { incrementAmount: number } @@ -55,21 +41,21 @@ type PrehandlerOutput = { messageId: string } -export class SqsPermissionConsumerMultiSchema extends AbstractSqsConsumerMultiSchema< +export class SqsPermissionConsumer extends AbstractSqsConsumer< SupportedMessages, ExecutionContext, PrehandlerOutput > { public addCounter = 0 public removeCounter = 0 - public static QUEUE_NAME = 'user_permissions_multi' + public static readonly QUEUE_NAME = 'user_permissions_multi' constructor( dependencies: SQSConsumerDependencies, - options: SqsPermissionConsumerMultiSchemaOptions = { + options: SqsPermissionConsumerOptions = { creationConfig: { queue: { - QueueName: SqsPermissionConsumerMultiSchema.QUEUE_NAME, + QueueName: SqsPermissionConsumer.QUEUE_NAME, }, }, }, @@ -88,24 +74,22 @@ export class SqsPermissionConsumerMultiSchema extends AbstractSqsConsumerMultiSc super( dependencies, { - messageTypeField: 'messageType', - handlerSpy: true, - deletionConfig: { + ...(options.locatorConfig + ? { locatorConfig: options.locatorConfig } + : { + creationConfig: options.creationConfig ?? { + queue: { QueueName: SqsPermissionConsumer.QUEUE_NAME }, + }, + }), + logMessages: options.logMessages, + deletionConfig: options.deletionConfig ?? { deleteIfExists: true, }, + messageTypeField: 'messageType', + handlerSpy: true, consumerOverrides: { terminateVisibilityTimeout: true, // this allows to retry failed messages immediately }, - // FixMe this casting shouldn't be necessary - ...(options as Pick< - NewSQSConsumerOptionsMultiSchema< - SupportedMessages, - ExecutionContext, - PrehandlerOutput, - SQSCreationConfig - >, - 'creationConfig' | 'logMessages' - >), handlers: new MessageHandlerConfigBuilder< SupportedMessages, ExecutionContext, @@ -148,4 +132,12 @@ export class SqsPermissionConsumerMultiSchema extends AbstractSqsConsumerMultiSc }, ) } + + public get queueProps() { + return { + name: this.queueName, + url: this.queueUrl, + arn: this.queueArn, + } + } } diff --git a/packages/sqs/test/consumers/SqsPermissionConsumerMonoSchema.ts b/packages/sqs/test/consumers/SqsPermissionConsumerMonoSchema.ts deleted file mode 100644 index 0accce6c..00000000 --- a/packages/sqs/test/consumers/SqsPermissionConsumerMonoSchema.ts +++ /dev/null @@ -1,89 +0,0 @@ -import type { Either } from '@lokalise/node-core' -import type { BarrierResult, PrehandlingOutputs } from '@message-queue-toolkit/core' - -import type { - ExistingSQSConsumerOptions, - NewSQSConsumerOptions, - SQSCreationConfig, -} from '../../lib/sqs/AbstractSqsConsumer' -import { AbstractSqsConsumerMonoSchema } from '../../lib/sqs/AbstractSqsConsumerMonoSchema' -import type { SQSConsumerDependencies } from '../../lib/sqs/AbstractSqsService' -import { userPermissionMap } from '../repositories/PermissionRepository' - -import type { PERMISSIONS_MESSAGE_TYPE } from './userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from './userConsumerSchemas' - -export class SqsPermissionConsumerMonoSchema extends AbstractSqsConsumerMonoSchema< - PERMISSIONS_MESSAGE_TYPE, - undefined, - undefined, - string[][] -> { - public static QUEUE_NAME = 'user_permissions' - - constructor( - dependencies: SQSConsumerDependencies, - options: - | Pick, 'creationConfig' | 'logMessages'> - | Pick = { - creationConfig: { - queue: { - QueueName: SqsPermissionConsumerMonoSchema.QUEUE_NAME, - }, - }, - }, - ) { - super(dependencies, { - messageSchema: PERMISSIONS_MESSAGE_SCHEMA, - messageTypeField: 'messageType', - handlerSpy: true, - deletionConfig: { - deleteIfExists: true, - }, - consumerOverrides: { - terminateVisibilityTimeout: true, // this allows to retry failed messages immediately - }, - ...options, - }) - } - - protected override async preHandlerBarrier( - message: PERMISSIONS_MESSAGE_TYPE, - _messageType: string, - ): Promise> { - const matchedUserPermissions = message.userIds.reduce((acc, userId) => { - if (userPermissionMap[userId]) { - acc.push(userPermissionMap[userId]) - } - return acc - }, [] as string[][]) - - if (matchedUserPermissions && matchedUserPermissions.length == message.userIds.length) { - return { - isPassing: true, - output: matchedUserPermissions, - } - } - - // not all users were already created, we need to wait to be able to set permissions - return { - isPassing: false, - } - } - - override async processMessage( - message: PERMISSIONS_MESSAGE_TYPE, - _messageType: string, - prehandlingOutputs: PrehandlingOutputs, - ): Promise> { - const matchedUserPermissions = prehandlingOutputs.barrierOutput - // Do not do this in production, some kind of bulk insertion is needed here - for (const userPermissions of matchedUserPermissions) { - userPermissions.push(...message.permissions) - } - - return { - result: 'success', - } - } -} diff --git a/packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.errors.spec.ts b/packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.errors.spec.ts deleted file mode 100644 index ac444729..00000000 --- a/packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.errors.spec.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { SQSClient } from '@aws-sdk/client-sqs' -import { ReceiveMessageCommand } from '@aws-sdk/client-sqs' -import { waitAndRetry } from '@message-queue-toolkit/core' -import type { AwilixContainer } from 'awilix' -import { asClass } from 'awilix' -import { describe, beforeEach, afterEach, expect, it } from 'vitest' -import z from 'zod' - -import { FakeConsumerErrorResolver } from '../../lib/fakes/FakeConsumerErrorResolver' -import type { SqsPermissionPublisherMonoSchema } from '../publishers/SqsPermissionPublisherMonoSchema' -import { userPermissionMap } from '../repositories/PermissionRepository' -import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' -import type { Dependencies } from '../utils/testContext' - -import type { SqsPermissionConsumerMonoSchema } from './SqsPermissionConsumerMonoSchema' - -const perms: [string, ...string[]] = ['perm1', 'perm2'] - -describe('SqsPermissionsConsumerMonoSchema', () => { - describe('error handling', () => { - let diContainer: AwilixContainer - let publisher: SqsPermissionPublisherMonoSchema - let consumer: SqsPermissionConsumerMonoSchema - let sqsClient: SQSClient - - beforeEach(async () => { - diContainer = await registerDependencies({ - consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), - }) - sqsClient = diContainer.cradle.sqsClient - publisher = diContainer.cradle.permissionPublisher - consumer = diContainer.cradle.permissionConsumer - - delete userPermissionMap[100] - delete userPermissionMap[200] - delete userPermissionMap[300] - - const command = new ReceiveMessageCommand({ - QueueUrl: diContainer.cradle.permissionPublisher.queueUrl, - }) - const reply = await sqsClient.send(command) - expect(reply.Messages).toBeUndefined() - - const fakeErrorResolver = diContainer.cradle - .consumerErrorResolver as FakeConsumerErrorResolver - fakeErrorResolver.clear() - }) - - afterEach(async () => { - await diContainer.cradle.awilixManager.executeDispose() - await diContainer.dispose() - }) - - // Flaky test, mono consumer is going to be removed soon anyway - it.skip('Invalid message in the queue', async () => { - const { consumerErrorResolver } = diContainer.cradle - - // @ts-ignore - publisher['messageSchema'] = z.any() - await publisher.publish({ - id: 'abc', - messageType: 'add', - permissions: perms, - } as any) - - const fakeResolver = consumerErrorResolver as FakeConsumerErrorResolver - const messageResult = await consumer.handlerSpy.waitForMessageWithId('abc') - expect(messageResult.processingResult).toBe('invalid_message') - - expect(fakeResolver.handleErrorCallsCount).toBe(1) - expect(fakeResolver.errors[0].message).toContain('"received": "undefined"') - }) - - it('Non-JSON message in the queue', async () => { - const { consumerErrorResolver } = diContainer.cradle - - // @ts-ignore - publisher['messageSchema'] = z.any() - await publisher.publish('dummy' as any) - - const fakeResolver = consumerErrorResolver as FakeConsumerErrorResolver - const errorCount = await waitAndRetry(() => { - return fakeResolver.handleErrorCallsCount - }) - - expect(errorCount).toBe(1) - expect(fakeResolver.errors[0].message).toContain('Expected object, received string') - }) - }) -}) diff --git a/packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.spec.ts b/packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.spec.ts deleted file mode 100644 index 85bb9792..00000000 --- a/packages/sqs/test/consumers/SqsPermissionsConsumerMonoSchema.spec.ts +++ /dev/null @@ -1,208 +0,0 @@ -import type { SQSClient } from '@aws-sdk/client-sqs' -import { ReceiveMessageCommand } from '@aws-sdk/client-sqs' -import type { AwilixContainer } from 'awilix' -import { asClass } from 'awilix' -import { describe, beforeEach, afterEach, expect, it, beforeAll } from 'vitest' - -import { FakeConsumerErrorResolver } from '../../lib/fakes/FakeConsumerErrorResolver' -import { assertQueue, deleteQueue } from '../../lib/utils/sqsUtils' -import type { SqsPermissionPublisherMonoSchema } from '../publishers/SqsPermissionPublisherMonoSchema' -import { userPermissionMap } from '../repositories/PermissionRepository' -import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' -import type { Dependencies } from '../utils/testContext' - -import { SqsPermissionConsumerMonoSchema } from './SqsPermissionConsumerMonoSchema' - -const userIds = [100, 200, 300] -const perms: [string, ...string[]] = ['perm1', 'perm2'] - -async function retrievePermissions(userIds: number[]) { - const usersPerms = userIds.reduce((acc, userId) => { - if (userPermissionMap[userId]) { - acc.push(userPermissionMap[userId]) - } - return acc - }, [] as string[][]) - - if (usersPerms && usersPerms.length !== userIds.length) { - return null - } - - for (const userPerms of usersPerms) - if (userPerms.length !== perms.length) { - return null - } - - return usersPerms -} - -describe('SqsPermissionsConsumerMonoSchema', () => { - describe('init', () => { - let diContainer: AwilixContainer - let sqsClient: SQSClient - beforeAll(async () => { - diContainer = await registerDependencies() - sqsClient = diContainer.cradle.sqsClient - await deleteQueue(sqsClient, 'existingQueue') - }) - - it('throws an error when invalid queue locator is passed', async () => { - const newConsumer = new SqsPermissionConsumerMonoSchema(diContainer.cradle, { - locatorConfig: { - queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', - }, - }) - - await expect(() => newConsumer.init()).rejects.toThrow(/does not exist/) - - await newConsumer.close() - }) - - it('does not create a new queue when queue locator is passed', async () => { - await assertQueue(sqsClient, { - QueueName: 'existingQueue', - }) - - const newConsumer = new SqsPermissionConsumerMonoSchema(diContainer.cradle, { - locatorConfig: { - queueUrl: 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', - }, - }) - - await newConsumer.init() - expect(newConsumer.queueUrl).toBe( - 'http://s3.localhost.localstack.cloud:4566/000000000000/existingQueue', - ) - - await newConsumer.close() - }) - }) - - describe('consume', () => { - let diContainer: AwilixContainer - let publisher: SqsPermissionPublisherMonoSchema - let sqsClient: SQSClient - let consumer: SqsPermissionConsumerMonoSchema - - beforeEach(async () => { - diContainer = await registerDependencies({ - consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), - }) - sqsClient = diContainer.cradle.sqsClient - publisher = diContainer.cradle.permissionPublisher - consumer = diContainer.cradle.permissionConsumer - - delete userPermissionMap[100] - delete userPermissionMap[200] - delete userPermissionMap[300] - - const command = new ReceiveMessageCommand({ - QueueUrl: diContainer.cradle.permissionPublisher.queueUrl, - }) - const reply = await sqsClient.send(command) - expect(reply.Messages).toBeUndefined() - - const fakeErrorResolver = diContainer.cradle - .consumerErrorResolver as FakeConsumerErrorResolver - fakeErrorResolver.clear() - }) - - afterEach(async () => { - await diContainer.cradle.awilixManager.executeDispose() - await diContainer.dispose() - }) - - describe('happy path', () => { - it('Creates permissions', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - await publisher.publish({ - id: 'abcd', - messageType: 'add', - userIds, - permissions: perms, - }) - - await consumer.handlerSpy.waitForMessageWithId('abcd', 'consumed') - const updatedUsersPermissions = await retrievePermissions(userIds) - - if (null === updatedUsersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(updatedUsersPermissions).toBeDefined() - expect(updatedUsersPermissions[0]).toHaveLength(2) - }) - - it('Wait for users to be created and then create permissions', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - await publisher.publish({ - id: '123', - messageType: 'add', - userIds, - permissions: perms, - }) - - // no users in the database, so message will go back to the queue - await consumer.handlerSpy.waitForMessageWithId('123', 'retryLater') - - const usersFromDb = await retrievePermissions(userIds) - expect(usersFromDb).toBeNull() - - userPermissionMap[100] = [] - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - await consumer.handlerSpy.waitForMessageWithId('123', 'consumed') - const usersPermissions = await retrievePermissions(userIds) - - if (null === usersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(usersPermissions).toBeDefined() - expect(usersPermissions[0]).toHaveLength(2) - }) - - it('Not all users exist, no permissions were created initially', async () => { - const users = Object.values(userPermissionMap) - expect(users).toHaveLength(0) - - userPermissionMap[100] = [] - - await publisher.publish({ - id: 'abc', - messageType: 'add', - userIds, - permissions: perms, - }) - - // not all users are in the database, so message will go back to the queue - await consumer.handlerSpy.waitForMessageWithId('abc', 'retryLater') - - const usersFromDb = await retrievePermissions(userIds) - expect(usersFromDb).toBeNull() - - userPermissionMap[200] = [] - userPermissionMap[300] = [] - - await consumer.handlerSpy.waitForMessageWithId('abc', 'consumed') - const usersPermissions = await retrievePermissions(userIds) - - if (null === usersPermissions) { - throw new Error('Users permissions unexpectedly null') - } - - expect(usersPermissions).toBeDefined() - expect(usersPermissions[0]).toHaveLength(2) - }) - }) - }) -}) diff --git a/packages/sqs/test/consumers/userConsumerSchemas.ts b/packages/sqs/test/consumers/userConsumerSchemas.ts index 163901b4..8f13a090 100644 --- a/packages/sqs/test/consumers/userConsumerSchemas.ts +++ b/packages/sqs/test/consumers/userConsumerSchemas.ts @@ -17,12 +17,7 @@ export const PERMISSIONS_REMOVE_MESSAGE_SCHEMA = z.object({ messageType: z.literal('remove'), }) -export const OTHER_MESSAGE_SCHEMA = z.object({ - dummy: z.literal('dummy'), -}) - export type PERMISSIONS_MESSAGE_TYPE = z.infer export type PERMISSIONS_ADD_MESSAGE_TYPE = z.infer export type PERMISSIONS_REMOVE_MESSAGE_TYPE = z.infer -export type OTHER_MESSAGE_SCHEMA_TYPE = z.infer diff --git a/packages/sqs/test/publishers/SqsPermissionPublisher.spec.ts b/packages/sqs/test/publishers/SqsPermissionPublisher.spec.ts new file mode 100644 index 00000000..a09cf2e2 --- /dev/null +++ b/packages/sqs/test/publishers/SqsPermissionPublisher.spec.ts @@ -0,0 +1,206 @@ +import type { SQSClient } from '@aws-sdk/client-sqs' +import type { AwilixContainer } from 'awilix' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { assertQueue, deleteQueue, getQueueAttributes } from '../../lib/utils/sqsUtils' +import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' +import { registerDependencies } from '../utils/testContext' +import type { Dependencies } from '../utils/testContext' + +import { SqsPermissionPublisher } from './SqsPermissionPublisher' + +describe('SqsPermissionPublisher', () => { + describe('init', () => { + const queueName = 'someQueue' + + let diContainer: AwilixContainer + let sqsClient: SQSClient + beforeEach(async () => { + diContainer = await registerDependencies() + sqsClient = diContainer.cradle.sqsClient + await deleteQueue(sqsClient, queueName) + }) + + afterEach(async () => { + await diContainer.cradle.awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('throws an error when invalid queue locator is passed', async () => { + const newPublisher = new SqsPermissionPublisher(diContainer.cradle, { + locatorConfig: { + queueUrl: `http://s3.localhost.localstack.cloud:4566/000000000000/${queueName}`, + }, + }) + + await expect(() => newPublisher.init()).rejects.toThrow(/does not exist/) + }) + + it('does not create a new queue when queue locator is passed', async () => { + await assertQueue(sqsClient, { + QueueName: queueName, + }) + + const newPublisher = new SqsPermissionPublisher(diContainer.cradle, { + locatorConfig: { + queueUrl: `http://s3.localhost.localstack.cloud:4566/000000000000/${queueName}`, + }, + }) + + await newPublisher.init() + expect(newPublisher.queueProps.url).toBe( + `http://s3.localhost.localstack.cloud:4566/000000000000/${queueName}`, + ) + }) + + it('updates existing queue when one with different attributes exist', async () => { + await assertQueue(sqsClient, { + QueueName: queueName, + Attributes: { + KmsMasterKeyId: 'somevalue', + }, + }) + + const newPublisher = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { + queue: { + QueueName: queueName, + Attributes: { + KmsMasterKeyId: 'othervalue', + }, + }, + updateAttributesIfExists: true, + }, + deletionConfig: { + deleteIfExists: false, + }, + logMessages: true, + }) + + const sqsSpy = vi.spyOn(sqsClient, 'send') + + await newPublisher.init() + expect(newPublisher.queueProps.url).toBe( + `http://sqs.eu-west-1.localstack:4566/000000000000/${queueName}`, + ) + + const updateCall = sqsSpy.mock.calls.find((entry) => { + return entry[0].constructor.name === 'SetQueueAttributesCommand' + }) + expect(updateCall).toBeDefined() + + const attributes = await getQueueAttributes(sqsClient, { + queueUrl: newPublisher.queueProps.url, + }) + + expect(attributes.result?.attributes!.KmsMasterKeyId).toBe('othervalue') + }) + + it('does not update existing queue when attributes did not change', async () => { + await assertQueue(sqsClient, { + QueueName: queueName, + Attributes: { + KmsMasterKeyId: 'somevalue', + }, + }) + + const newPublisher = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { + queue: { + QueueName: queueName, + Attributes: { + KmsMasterKeyId: 'somevalue', + }, + }, + updateAttributesIfExists: true, + }, + deletionConfig: { + deleteIfExists: false, + }, + logMessages: true, + }) + + const sqsSpy = vi.spyOn(sqsClient, 'send') + + await newPublisher.init() + expect(newPublisher.queueProps.url).toBe( + `http://sqs.eu-west-1.localstack:4566/000000000000/${queueName}`, + ) + + const updateCall = sqsSpy.mock.calls.find((entry) => { + return entry[0].constructor.name === 'SetQueueAttributesCommand' + }) + expect(updateCall).toBeUndefined() + + const attributes = await getQueueAttributes(sqsClient, { + queueUrl: newPublisher.queueProps.url, + }) + + expect(attributes.result?.attributes!.KmsMasterKeyId).toBe('somevalue') + }) + }) + + describe('publish', () => { + let diContainer: AwilixContainer + let sqsClient: SQSClient + let permissionPublisher: SqsPermissionPublisher + + beforeEach(async () => { + diContainer = await registerDependencies() + sqsClient = diContainer.cradle.sqsClient + await diContainer.cradle.permissionConsumer.close() + permissionPublisher = diContainer.cradle.permissionPublisher + + await deleteQueue(sqsClient, SqsPermissionPublisher.QUEUE_NAME) + }) + + afterEach(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('publish inalid message', async () => { + await expect( + permissionPublisher.publish({ + id: '10', + messageType: 'bad' as any, + }), + ).rejects.toThrow(/Unsupported message type: bad/) + }) + + it('publishes a message', async () => { + const { permissionPublisher } = diContainer.cradle + + const message = { + id: '1', + userIds: [100, 200, 300], + messageType: 'add', + permissions: ['perm1', 'perm2'], + } satisfies PERMISSIONS_MESSAGE_TYPE + + await permissionPublisher.publish(message) + + const spy = await permissionPublisher.handlerSpy.waitForMessageWithId('1', 'published') + expect(spy.message).toEqual(message) + expect(spy.processingResult).toEqual('published') + }) + + it('publish message with lazy loading', async () => { + const newPublisher = new SqsPermissionPublisher(diContainer.cradle) + + const message = { + id: '1', + userIds: [100, 200, 300], + messageType: 'add', + permissions: ['perm1', 'perm2'], + } satisfies PERMISSIONS_MESSAGE_TYPE + + await newPublisher.publish(message) + + const spy = await newPublisher.handlerSpy.waitForMessageWithId('1', 'published') + expect(spy.message).toEqual(message) + expect(spy.processingResult).toEqual('published') + }) + }) +}) diff --git a/packages/sqs/test/publishers/SqsPermissionPublisher.ts b/packages/sqs/test/publishers/SqsPermissionPublisher.ts new file mode 100644 index 00000000..f373fe90 --- /dev/null +++ b/packages/sqs/test/publishers/SqsPermissionPublisher.ts @@ -0,0 +1,54 @@ +import type { QueuePublisherOptions } from '@message-queue-toolkit/core' + +import type { SQSCreationConfig } from '../../lib/sqs/AbstractSqsConsumer' +import { AbstractSqsPublisher } from '../../lib/sqs/AbstractSqsPublisher' +import type { SQSDependencies, SQSQueueLocatorType } from '../../lib/sqs/AbstractSqsService' +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from '../consumers/userConsumerSchemas' +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from '../consumers/userConsumerSchemas' + +type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE + +export class SqsPermissionPublisher extends AbstractSqsPublisher { + public static readonly QUEUE_NAME = 'user_permissions_multi' + + constructor( + dependencies: SQSDependencies, + options?: Pick< + QueuePublisherOptions, + 'creationConfig' | 'locatorConfig' | 'deletionConfig' | 'logMessages' + >, + ) { + super(dependencies, { + ...(options?.locatorConfig + ? { locatorConfig: options.locatorConfig } + : { + creationConfig: options?.creationConfig ?? { + queue: { + QueueName: SqsPermissionPublisher.QUEUE_NAME, + }, + }, + }), + logMessages: options?.logMessages, + deletionConfig: options?.deletionConfig ?? { + deleteIfExists: false, + }, + handlerSpy: true, + messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], + messageTypeField: 'messageType', + }) + } + + public get queueProps() { + return { + name: this.queueName, + url: this.queueUrl, + arn: this.queueArn, + } + } +} diff --git a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts deleted file mode 100644 index 8f7c6789..00000000 --- a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.spec.ts +++ /dev/null @@ -1,157 +0,0 @@ -import type { SQSClient } from '@aws-sdk/client-sqs' -import { waitAndRetry } from '@message-queue-toolkit/core' -import type { AwilixContainer } from 'awilix' -import { asClass, asFunction } from 'awilix' -import { Consumer } from 'sqs-consumer' -import { describe, beforeEach, expect, it, afterEach } from 'vitest' - -import { FakeConsumerErrorResolver } from '../../lib/fakes/FakeConsumerErrorResolver' -import type { SQSMessage } from '../../lib/types/MessageTypes' -import { deserializeSQSMessage } from '../../lib/utils/sqsMessageDeserializer' -import { assertQueue, deleteQueue } from '../../lib/utils/sqsUtils' -import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' -import { FakeLogger } from '../fakes/FakeLogger' -import { userPermissionMap } from '../repositories/PermissionRepository' -import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext' -import type { Dependencies } from '../utils/testContext' - -import { SqsPermissionPublisherMonoSchema } from './SqsPermissionPublisherMonoSchema' - -const perms: [string, ...string[]] = ['perm1', 'perm2'] -const userIds = [100, 200, 300] - -describe('SqsPermissionPublisher', () => { - let diContainer: AwilixContainer - let publisher: SqsPermissionPublisherMonoSchema - let logger: FakeLogger - let sqsClient: SQSClient - let consumer: Consumer - - beforeEach(async () => { - logger = new FakeLogger() - diContainer = await registerDependencies({ - consumerErrorResolver: asClass(FakeConsumerErrorResolver, SINGLETON_CONFIG), - logger: asFunction(() => logger), - }) - publisher = diContainer.cradle.permissionPublisher - sqsClient = diContainer.cradle.sqsClient - }) - - afterEach(async () => { - await diContainer.cradle.awilixManager.executeDispose() - await diContainer.dispose() - }) - - describe('logging', () => { - it('logs a message when logging is enabled', async () => { - const message = { - id: '1', - userIds, - messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE - - await publisher.publish(message) - - await publisher.handlerSpy.waitForMessageWithId('1') - - expect(logger.loggedMessages.length).toBe(2) - expect(logger.loggedMessages).toMatchInlineSnapshot(` - [ - { - "id": "1", - "messageType": "add", - "permissions": [ - "perm1", - "perm2", - ], - "userIds": [ - 100, - 200, - 300, - ], - }, - { - "messageId": "1", - "processingResult": "published", - }, - ] - `) - }) - }) - - describe('publish', () => { - beforeEach(async () => { - delete userPermissionMap[100] - delete userPermissionMap[200] - delete userPermissionMap[300] - - await deleteQueue(sqsClient, diContainer.cradle.permissionPublisher.queueName) - // @ts-ignore - await assertQueue(sqsClient, diContainer.cradle.permissionPublisher.creationConfig!.queue) - }) - - it('publishes a message', async () => { - const { permissionPublisher } = diContainer.cradle - - const message = { - id: '2', - userIds, - messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE - - let receivedMessage: PERMISSIONS_MESSAGE_TYPE | null = null - consumer = Consumer.create({ - queueUrl: diContainer.cradle.permissionPublisher.queueUrl, - handleMessage: async (message: SQSMessage) => { - if (message === null) { - return - } - const decodedMessage = deserializeSQSMessage( - message as any, - PERMISSIONS_MESSAGE_SCHEMA, - new FakeConsumerErrorResolver(), - ) - receivedMessage = decodedMessage.result! - }, - sqs: diContainer.cradle.sqsClient, - }) - consumer.start() - - consumer.on('error', () => {}) - - await permissionPublisher.publish(message) - - await waitAndRetry(() => { - return receivedMessage !== null - }) - - expect(receivedMessage).toEqual({ - id: '2', - messageType: 'add', - permissions: ['perm1', 'perm2'], - userIds: [100, 200, 300], - }) - - consumer.stop() - }) - - it('publish message with lazy loading', async () => { - const newPublisher = new SqsPermissionPublisherMonoSchema(diContainer.cradle) - - const message = { - id: '1', - userIds, - messageType: 'add', - permissions: perms, - } satisfies PERMISSIONS_MESSAGE_TYPE - - await newPublisher.publish(message) - - const res = await newPublisher.handlerSpy.waitForMessageWithId('1', 'published') - expect(res.message).toEqual(message) - }) - }) -}) diff --git a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts b/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts deleted file mode 100644 index 40684b4a..00000000 --- a/packages/sqs/test/publishers/SqsPermissionPublisherMonoSchema.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { AbstractSqsPublisherMonoSchema } from '../../lib/sqs/AbstractSqsPublisherMonoSchema' -import type { SQSDependencies } from '../../lib/sqs/AbstractSqsService' -import type { PERMISSIONS_MESSAGE_TYPE } from '../consumers/userConsumerSchemas' -import { PERMISSIONS_MESSAGE_SCHEMA } from '../consumers/userConsumerSchemas' - -export class SqsPermissionPublisherMonoSchema extends AbstractSqsPublisherMonoSchema { - public static QUEUE_NAME = 'user_permissions' - - constructor(dependencies: SQSDependencies) { - super(dependencies, { - creationConfig: { - queue: { - QueueName: SqsPermissionPublisherMonoSchema.QUEUE_NAME, - }, - }, - handlerSpy: true, - deletionConfig: { - deleteIfExists: false, - }, - logMessages: true, - messageSchema: PERMISSIONS_MESSAGE_SCHEMA, - messageTypeField: 'messageType', - }) - } -} diff --git a/packages/sqs/test/publishers/SqsPermissionPublisherMultiSchema.ts b/packages/sqs/test/publishers/SqsPermissionPublisherMultiSchema.ts deleted file mode 100644 index 0448ecc4..00000000 --- a/packages/sqs/test/publishers/SqsPermissionPublisherMultiSchema.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { AbstractSqsPublisherMultiSchema } from '../../lib/sqs/AbstractSqsPublisherMultiSchema' -import type { SQSDependencies } from '../../lib/sqs/AbstractSqsService' -import type { - PERMISSIONS_ADD_MESSAGE_TYPE, - PERMISSIONS_REMOVE_MESSAGE_TYPE, -} from '../consumers/userConsumerSchemas' -import { - PERMISSIONS_ADD_MESSAGE_SCHEMA, - PERMISSIONS_REMOVE_MESSAGE_SCHEMA, -} from '../consumers/userConsumerSchemas' - -type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE - -export class SqsPermissionPublisherMultiSchema extends AbstractSqsPublisherMultiSchema { - public static QUEUE_NAME = 'user_permissions_multi' - - constructor(dependencies: SQSDependencies) { - super(dependencies, { - creationConfig: { - queue: { - QueueName: SqsPermissionPublisherMultiSchema.QUEUE_NAME, - }, - }, - deletionConfig: { - deleteIfExists: false, - }, - messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], - messageTypeField: 'messageType', - }) - } -} diff --git a/packages/sqs/test/repositories/PermissionRepository.ts b/packages/sqs/test/repositories/PermissionRepository.ts deleted file mode 100644 index 89b09f10..00000000 --- a/packages/sqs/test/repositories/PermissionRepository.ts +++ /dev/null @@ -1 +0,0 @@ -export const userPermissionMap: Record = {} diff --git a/packages/sqs/test/utils/testContext.ts b/packages/sqs/test/utils/testContext.ts index 70ae71b4..6f67a485 100644 --- a/packages/sqs/test/utils/testContext.ts +++ b/packages/sqs/test/utils/testContext.ts @@ -6,10 +6,8 @@ import { asClass, asFunction, createContainer, Lifetime } from 'awilix' import { AwilixManager } from 'awilix-manager' import { SqsConsumerErrorResolver } from '../../lib/errors/SqsConsumerErrorResolver' -import { SqsPermissionConsumerMonoSchema } from '../consumers/SqsPermissionConsumerMonoSchema' -import { SqsPermissionConsumerMultiSchema } from '../consumers/SqsPermissionConsumerMultiSchema' -import { SqsPermissionPublisherMonoSchema } from '../publishers/SqsPermissionPublisherMonoSchema' -import { SqsPermissionPublisherMultiSchema } from '../publishers/SqsPermissionPublisherMultiSchema' +import { SqsPermissionConsumer } from '../consumers/SqsPermissionConsumer' +import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher' import { TEST_SQS_CONFIG } from './testSqsConfig' @@ -52,26 +50,13 @@ export async function registerDependencies(dependencyOverrides: DependencyOverri return new SqsConsumerErrorResolver() }), - permissionConsumer: asClass(SqsPermissionConsumerMonoSchema, { + permissionConsumer: asClass(SqsPermissionConsumer, { lifetime: Lifetime.SINGLETON, asyncInit: 'start', asyncDispose: 'close', asyncDisposePriority: 10, }), - permissionPublisher: asClass(SqsPermissionPublisherMonoSchema, { - lifetime: Lifetime.SINGLETON, - asyncInit: 'init', - asyncDispose: 'close', - asyncDisposePriority: 20, - }), - - permissionConsumerMultiSchema: asClass(SqsPermissionConsumerMultiSchema, { - lifetime: Lifetime.SINGLETON, - asyncInit: 'start', - asyncDispose: 'close', - asyncDisposePriority: 10, - }), - permissionPublisherMultiSchema: asClass(SqsPermissionPublisherMultiSchema, { + permissionPublisher: asClass(SqsPermissionPublisher, { lifetime: Lifetime.SINGLETON, asyncInit: 'init', asyncDispose: 'close', @@ -112,8 +97,6 @@ export interface Dependencies { errorReporter: ErrorReporter consumerErrorResolver: ErrorResolver - permissionConsumer: SqsPermissionConsumerMonoSchema - permissionPublisher: SqsPermissionPublisherMonoSchema - permissionConsumerMultiSchema: SqsPermissionConsumerMultiSchema - permissionPublisherMultiSchema: SqsPermissionPublisherMultiSchema + permissionConsumer: SqsPermissionConsumer + permissionPublisher: SqsPermissionPublisher }