From 466a0a0d08a5afd6367a14303f84d408eb0dc421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 29 Nov 2024 16:40:04 +0100 Subject: [PATCH 1/8] fix(handler): import types --- packages/handler/src/types.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/handler/src/types.ts b/packages/handler/src/types.ts index de5381d83d3..94e4b0dfff4 100644 --- a/packages/handler/src/types.ts +++ b/packages/handler/src/types.ts @@ -1,7 +1,8 @@ import "@fastify/cookie"; -import { FastifyRequest, FastifyReply, HTTPMethods, RouteHandlerMethod } from "fastify"; -export { FastifyInstance, HTTPMethods } from "fastify"; -import { ClientContext } from "@webiny/handler-client/types"; +import type { FastifyReply, FastifyRequest, HTTPMethods, RouteHandlerMethod } from "fastify"; +import type { ClientContext } from "@webiny/handler-client/types"; + +export type { FastifyInstance, HTTPMethods } from "fastify"; export interface RouteMethodOptions { override?: boolean; From 7f94bca3ec28e0bbd837b3f1039ec6c008ef3602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 29 Nov 2024 16:44:10 +0100 Subject: [PATCH 2/8] feat(aws-sdk): create sqs client --- packages/aws-sdk/src/client-sqs/index.ts | 44 ++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/aws-sdk/src/client-sqs/index.ts b/packages/aws-sdk/src/client-sqs/index.ts index 1af930ff4e7..966de98aceb 100644 --- a/packages/aws-sdk/src/client-sqs/index.ts +++ b/packages/aws-sdk/src/client-sqs/index.ts @@ -1,5 +1,43 @@ -export { - SQSClient, +import { SQSClient, SQSClientConfig as BaseSQSClientConfig } from "@aws-sdk/client-sqs"; +import { createCacheKey } from "@webiny/utils"; + +export { SQSClient, SendMessageCommand, SendMessageBatchCommand } from "@aws-sdk/client-sqs"; +export type { + SendMessageCommandInput, + SendMessageCommandOutput, SendMessageBatchRequestEntry, - SendMessageBatchCommand + MessageAttributeValue } from "@aws-sdk/client-sqs"; + +export interface SQSClientConfig extends BaseSQSClientConfig { + cache?: boolean; +} + +const sqsClientsCache = new Map(); + +export const createSqsClient = (input: Partial) => { + const options: SQSClientConfig = { + region: process.env.AWS_REGION, + ...input + }; + + const skipCache = options.cache === false; + delete options.cache; + if (skipCache) { + return new SQSClient({ + ...options + }); + } + + const key = createCacheKey(options); + if (sqsClientsCache.has(key)) { + return sqsClientsCache.get(key) as SQSClient; + } + + const instance = new SQSClient({ + ...options + }); + sqsClientsCache.set(key, instance); + + return instance; +}; From 1336ffba2339e9135cce01b9f9717e64f54cd63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 29 Nov 2024 16:44:29 +0100 Subject: [PATCH 3/8] fix(api): import types --- packages/api/src/ServiceDiscovery.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/api/src/ServiceDiscovery.ts b/packages/api/src/ServiceDiscovery.ts index 4c5231d721f..d0cad2c97e7 100644 --- a/packages/api/src/ServiceDiscovery.ts +++ b/packages/api/src/ServiceDiscovery.ts @@ -1,10 +1,6 @@ -import { - getDocumentClient, - DynamoDBDocument, - QueryCommand, - unmarshall -} from "@webiny/aws-sdk/client-dynamodb"; -import { GenericRecord } from "~/types"; +import { getDocumentClient, QueryCommand, unmarshall } from "@webiny/aws-sdk/client-dynamodb"; +import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; +import type { GenericRecord } from "~/types"; interface ServiceManifest { name: string; From 613a2925db416b432b8d5ba6505c9752a058211e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 29 Nov 2024 16:45:14 +0100 Subject: [PATCH 4/8] wip(api-serverless-cms): request cloning --- packages/api-serverless-cms/package.json | 6 +- .../src/enterprise/index.ts | 1 + .../src/enterprise/requestCloning/index.ts | 1 + .../src/enterprise/requestCloning/options.ts | 49 +++++++++++++ .../requestCloning/request/RequestTransfer.ts | 71 +++++++++++++++++++ .../requestCloning/request/index.ts | 2 + .../requestCloning/request/types.ts | 12 ++++ .../requestCloning/requestCloning.ts | 68 ++++++++++++++++++ .../requestCloning/s3/S3Transfer.ts | 59 +++++++++++++++ .../src/enterprise/requestCloning/s3/index.ts | 2 + .../src/enterprise/requestCloning/s3/types.ts | 12 ++++ .../requestCloning/sqs/SQSTransfer.ts | 61 ++++++++++++++++ .../enterprise/requestCloning/sqs/index.ts | 2 + .../enterprise/requestCloning/sqs/types.ts | 15 ++++ .../api-serverless-cms/tsconfig.build.json | 4 +- packages/api-serverless-cms/tsconfig.json | 12 +++- 16 files changed, 371 insertions(+), 6 deletions(-) create mode 100644 packages/api-serverless-cms/src/enterprise/index.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/index.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/options.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/request/index.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/request/types.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/s3/index.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/sqs/SQSTransfer.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/sqs/index.ts create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/sqs/types.ts diff --git a/packages/api-serverless-cms/package.json b/packages/api-serverless-cms/package.json index 27d7f488e6f..0c8200edaed 100644 --- a/packages/api-serverless-cms/package.json +++ b/packages/api-serverless-cms/package.json @@ -21,9 +21,12 @@ "@webiny/api-prerendering-service": "0.0.0", "@webiny/api-security": "0.0.0", "@webiny/api-tenancy": "0.0.0", + "@webiny/aws-sdk": "0.0.0", "@webiny/error": "0.0.0", + "@webiny/handler": "0.0.0", "@webiny/handler-client": "0.0.0", - "@webiny/handler-graphql": "0.0.0" + "@webiny/handler-graphql": "0.0.0", + "@webiny/utils": "0.0.0" }, "devDependencies": { "@webiny/api-admin-users": "0.0.0", @@ -38,7 +41,6 @@ "@webiny/api-wcp": "0.0.0", "@webiny/api-websockets": "0.0.0", "@webiny/cli": "0.0.0", - "@webiny/handler": "0.0.0", "@webiny/handler-aws": "0.0.0", "@webiny/plugins": "0.0.0", "@webiny/project-utils": "0.0.0", diff --git a/packages/api-serverless-cms/src/enterprise/index.ts b/packages/api-serverless-cms/src/enterprise/index.ts new file mode 100644 index 00000000000..fa22bb3e015 --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/index.ts @@ -0,0 +1 @@ +export * from "./requestCloning"; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/index.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/index.ts new file mode 100644 index 00000000000..fa22bb3e015 --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/index.ts @@ -0,0 +1 @@ +export * from "./requestCloning"; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts new file mode 100644 index 00000000000..b5909e98a28 --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts @@ -0,0 +1,49 @@ +import { ServiceDiscovery } from "@webiny/api"; + +export interface IOptions { + sqsRegion: string; + sqsUrl: string; + s3Region: string; + s3Bucket: string; +} + +interface IManifestApiTwoPhasedDeployment { + isPrimary: boolean; + s3Region: string; + s3Bucket: string; + sqsRegion: string; + sqsUrl: string; +} + +interface IManifestApi { + twoPhasedDeployment: IManifestApiTwoPhasedDeployment; +} + +interface IManifest { + api: IManifestApi; +} + +export const getOptions = async (): Promise => { + const manifest = (await ServiceDiscovery.load()) as IManifest | undefined; + if (!manifest) { + console.error("Service manifest not found."); + return null; + } + + const { twoPhasedDeployment } = manifest.api; + + if (!twoPhasedDeployment.isPrimary) { + return null; + } + + const result: Partial = {}; + const keys = ["sqsRegion", "sqsUrl", "s3Region", "s3Bucket"] as const; + for (const key of keys) { + if (!twoPhasedDeployment[key]) { + console.log(`${key} is not set.`); + return null; + } + result[key] = twoPhasedDeployment[key]; + } + return result as IOptions; +}; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts new file mode 100644 index 00000000000..06bc1c1266e --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts @@ -0,0 +1,71 @@ +import { compress } from "@webiny/utils/compression/gzip"; +import type { ISQSTransfer } from "../sqs/types"; +import type { IS3Transfer, IS3TransferSendResult } from "../s3/types"; +import type { + IRequestTransfer, + IRequestTransferSendParams, + IRequestTransferSendResult +} from "./types"; + +/** + * There are few steps we must go through: + * 1. compress the request body + * 2. store the compressed request body in S3 + * 3. send a message to SQS with the location of the compressed request body in S3 + */ + +export interface ITransferRequestParams { + s3: IS3Transfer; + sqs: ISQSTransfer; +} + +export class RequestTransfer implements IRequestTransfer { + private readonly s3: IS3Transfer; + private readonly sqs: ISQSTransfer; + + public constructor(params: ITransferRequestParams) { + this.s3 = params.s3; + this.sqs = params.sqs; + } + + public async send(params: IRequestTransferSendParams): Promise { + const { key, event } = params; + const body = await compress(JSON.stringify(event)); + const s3Key = `requests/${Date.now()}.${key}.json.gz`; + + let result: IS3TransferSendResult; + try { + result = await this.s3.send({ + key: s3Key, + body + }); + } catch (ex) { + console.error("Failed to store the request in S3."); + console.log(ex); + return; + } + + try { + await this.sqs.send({ + body: JSON.stringify({ + type: "s3", + value: result.key + }), + attributes: [ + { + name: "ETag", + value: result.eTag + } + ], + groupId: "systemAToSystemB" + }); + } catch (ex) { + console.error("Failed to send a message to SQS."); + console.log(ex); + } + } +} + +export const createRequestTransfer = (params: ITransferRequestParams): IRequestTransfer => { + return new RequestTransfer(params); +}; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/request/index.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/request/index.ts new file mode 100644 index 00000000000..0dd006460f2 --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/request/index.ts @@ -0,0 +1,2 @@ +export * from "./RequestTransfer"; +export * from "./types"; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/request/types.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/request/types.ts new file mode 100644 index 00000000000..dc1f11e5f74 --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/request/types.ts @@ -0,0 +1,12 @@ +import type { APIGatewayEvent } from "@webiny/handler-aws/types"; + +export interface IRequestTransferSendParams { + key: string; + event: APIGatewayEvent; +} + +export type IRequestTransferSendResult = void; + +export interface IRequestTransfer { + send(params: IRequestTransferSendParams): Promise; +} diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts new file mode 100644 index 00000000000..384c97e0c0a --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts @@ -0,0 +1,68 @@ +import { createModifyFastifyPlugin } from "@webiny/handler"; +import { createSqsClient } from "@webiny/aws-sdk/client-sqs"; +import { createS3Client } from "@webiny/aws-sdk/client-s3"; +import { getOptions } from "./options"; +import { createCacheKey } from "@webiny/utils"; +import { createS3Transfer } from "./s3"; +import { createSQSTransfer } from "./sqs"; +import { createRequestTransfer } from "./request"; + +export const requestCloning = () => { + return createModifyFastifyPlugin(instance => { + instance.addHook("onRequest", async request => { + /** + * We will not transfer OPTIONS request. Everything else can go into another system. + */ + if (request.method.toLowerCase() === "options") { + return; + } + const event = request.awsLambda?.event; + if (!event) { + console.error(`There is no event to be transferred into another system.`); + return; + } + const lambdaContext = request.awsLambda?.context; + if (!lambdaContext) { + console.error(`There is no context to be transferred into another system.`); + return; + } + const options = await getOptions(); + if (!options) { + return; + } + + const s3Transfer = createS3Transfer({ + client: createS3Client({ + region: options.s3Region + }), + bucket: options.s3Bucket + }); + const sqsTransfer = createSQSTransfer({ + client: createSqsClient({ + region: options.sqsRegion + }), + url: options.sqsUrl + }); + + const transfer = createRequestTransfer({ + s3: s3Transfer, + sqs: sqsTransfer + }); + + const key = createCacheKey(event, { + algorithm: "sha512" + }); + + try { + await transfer.send({ + key, + event + }); + } catch (ex) { + console.error("Failed to transfer the request to another system."); + console.log(ex); + throw ex; + } + }); + }); +}; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts new file mode 100644 index 00000000000..0aa072dd11e --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts @@ -0,0 +1,59 @@ +import { IS3Transfer, IS3TransferSendParams, IS3TransferSendResult } from "./types"; +import { + PutObjectCommand, + PutObjectCommandInput, + PutObjectCommandOutput, + S3Client +} from "@webiny/aws-sdk/client-s3"; + +export interface IS3TransferParams { + client: S3Client; + bucket: string; +} + +export class S3Transfer implements IS3Transfer { + private readonly client: S3Client; + private readonly bucket: string; + + public constructor(params: IS3TransferParams) { + this.client = params.client; + this.bucket = params.bucket; + } + + public async send(params: IS3TransferSendParams): Promise { + let s3Result: PutObjectCommandOutput; + try { + const input: PutObjectCommandInput = { + ACL: "private", + Bucket: this.bucket, + Key: params.key, + Body: params.body + }; + const cmd = new PutObjectCommand(input); + s3Result = await this.client.send(cmd); + } catch (ex) { + console.error("Failed to store the request in S3."); + console.log(ex); + throw ex; + } + + const statusCode = s3Result.$metadata?.httpStatusCode; + const eTag = s3Result.ETag; + + if (statusCode !== 200 || !eTag) { + const message = `Failed to store the request in S3. Key: ${params.key}`; + console.error(message); + throw new Error(message); + } + + return { + key: params.key, + eTag, + statusCode + }; + } +} + +export const createS3Transfer = (params: IS3TransferParams): IS3Transfer => { + return new S3Transfer(params); +}; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/index.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/index.ts new file mode 100644 index 00000000000..b4a8252c05c --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/index.ts @@ -0,0 +1,2 @@ +export * from "./S3Transfer"; +export * from "./types"; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts new file mode 100644 index 00000000000..f3192432cca --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts @@ -0,0 +1,12 @@ +export interface IS3TransferSendParams { + key: string; + body: string | Buffer; +} +export interface IS3TransferSendResult { + key: string; + eTag: string; + statusCode: 200 | unknown; +} +export interface IS3Transfer { + send(params: IS3TransferSendParams): Promise; +} diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/SQSTransfer.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/SQSTransfer.ts new file mode 100644 index 00000000000..6748c29961c --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/SQSTransfer.ts @@ -0,0 +1,61 @@ +import type { + MessageAttributeValue, + SendMessageCommandInput, + SQSClient +} from "@webiny/aws-sdk/client-sqs"; +import { SendMessageCommand } from "@webiny/aws-sdk/client-sqs"; +import type { ISQSTransfer, ISQSTransferSendParams } from "./types"; + +export interface ISQSTransferParams { + client: SQSClient; + url: string; +} + +export class SQSTransfer implements ISQSTransfer { + private readonly client: SQSClient; + private readonly url: string; + + public constructor(params: ISQSTransferParams) { + this.client = params.client; + this.url = params.url; + } + + public async send(params: ISQSTransferSendParams): Promise { + try { + const input: SendMessageCommandInput = { + QueueUrl: this.url, + MessageBody: params.body, + DelaySeconds: 0, + MessageAttributes: this.getMessageAttributes(params.attributes), + MessageGroupId: params.groupId, + MessageDeduplicationId: params.deduplicationId + }; + const cmd = new SendMessageCommand(input); + await this.client.send(cmd); + } catch (ex) { + console.error("Failed to send a message to SQS."); + console.log(ex); + throw ex; + } + } + + private getMessageAttributes( + attributes: ISQSTransferSendParams["attributes"] + ): Record | undefined { + if (!attributes?.length) { + return undefined; + } + return attributes.reduce>((attributes, item) => { + attributes[item.name] = { + DataType: "String", + StringValue: item.value + }; + + return attributes; + }, {}); + } +} + +export const createSQSTransfer = (params: ISQSTransferParams): ISQSTransfer => { + return new SQSTransfer(params); +}; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/index.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/index.ts new file mode 100644 index 00000000000..75938ca1edc --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/index.ts @@ -0,0 +1,2 @@ +export * from "./SQSTransfer"; +export * from "./types"; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/types.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/types.ts new file mode 100644 index 00000000000..42e479cc538 --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/sqs/types.ts @@ -0,0 +1,15 @@ +export interface ISQSTransfer { + send(params: ISQSTransferSendParams): Promise; +} + +export interface ISQSTransferSendParamsAttribute { + name: string; + value: string; +} + +export interface ISQSTransferSendParams { + body: string; + attributes?: ISQSTransferSendParamsAttribute[]; + groupId: string; + deduplicationId?: string; +} diff --git a/packages/api-serverless-cms/tsconfig.build.json b/packages/api-serverless-cms/tsconfig.build.json index 8546fb63978..789cb156794 100644 --- a/packages/api-serverless-cms/tsconfig.build.json +++ b/packages/api-serverless-cms/tsconfig.build.json @@ -14,9 +14,12 @@ { "path": "../api-prerendering-service/tsconfig.build.json" }, { "path": "../api-security/tsconfig.build.json" }, { "path": "../api-tenancy/tsconfig.build.json" }, + { "path": "../aws-sdk/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, + { "path": "../handler/tsconfig.build.json" }, { "path": "../handler-client/tsconfig.build.json" }, { "path": "../handler-graphql/tsconfig.build.json" }, + { "path": "../utils/tsconfig.build.json" }, { "path": "../api-admin-users/tsconfig.build.json" }, { "path": "../api-apw/tsconfig.build.json" }, { "path": "../api-audit-logs/tsconfig.build.json" }, @@ -28,7 +31,6 @@ { "path": "../api-record-locking/tsconfig.build.json" }, { "path": "../api-wcp/tsconfig.build.json" }, { "path": "../api-websockets/tsconfig.build.json" }, - { "path": "../handler/tsconfig.build.json" }, { "path": "../handler-aws/tsconfig.build.json" }, { "path": "../plugins/tsconfig.build.json" }, { "path": "../tasks/tsconfig.build.json" } diff --git a/packages/api-serverless-cms/tsconfig.json b/packages/api-serverless-cms/tsconfig.json index 7ef989ac80f..7722e0f8a9a 100644 --- a/packages/api-serverless-cms/tsconfig.json +++ b/packages/api-serverless-cms/tsconfig.json @@ -14,9 +14,12 @@ { "path": "../api-prerendering-service" }, { "path": "../api-security" }, { "path": "../api-tenancy" }, + { "path": "../aws-sdk" }, { "path": "../error" }, + { "path": "../handler" }, { "path": "../handler-client" }, { "path": "../handler-graphql" }, + { "path": "../utils" }, { "path": "../api-admin-users" }, { "path": "../api-apw" }, { "path": "../api-audit-logs" }, @@ -28,7 +31,6 @@ { "path": "../api-record-locking" }, { "path": "../api-wcp" }, { "path": "../api-websockets" }, - { "path": "../handler" }, { "path": "../handler-aws" }, { "path": "../plugins" }, { "path": "../tasks" } @@ -64,12 +66,18 @@ "@webiny/api-security": ["../api-security/src"], "@webiny/api-tenancy/*": ["../api-tenancy/src/*"], "@webiny/api-tenancy": ["../api-tenancy/src"], + "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], + "@webiny/aws-sdk": ["../aws-sdk/src"], "@webiny/error/*": ["../error/src/*"], "@webiny/error": ["../error/src"], + "@webiny/handler/*": ["../handler/src/*"], + "@webiny/handler": ["../handler/src"], "@webiny/handler-client/*": ["../handler-client/src/*"], "@webiny/handler-client": ["../handler-client/src"], "@webiny/handler-graphql/*": ["../handler-graphql/src/*"], "@webiny/handler-graphql": ["../handler-graphql/src"], + "@webiny/utils/*": ["../utils/src/*"], + "@webiny/utils": ["../utils/src"], "@webiny/api-admin-users/*": ["../api-admin-users/src/*"], "@webiny/api-admin-users": ["../api-admin-users/src"], "@webiny/api-apw/*": ["../api-apw/src/*"], @@ -96,8 +104,6 @@ "@webiny/api-wcp": ["../api-wcp/src"], "@webiny/api-websockets/*": ["../api-websockets/src/*"], "@webiny/api-websockets": ["../api-websockets/src"], - "@webiny/handler/*": ["../handler/src/*"], - "@webiny/handler": ["../handler/src"], "@webiny/handler-aws/*": ["../handler-aws/src/*"], "@webiny/handler-aws": ["../handler-aws/src"], "@webiny/plugins/*": ["../plugins/src/*"], From d45a8e76fc642d0dffe3ca35ab6143d7ccc3bba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 29 Nov 2024 22:39:52 +0100 Subject: [PATCH 5/8] chore: yarn lock --- yarn.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yarn.lock b/yarn.lock index e3260cd34ba..02bea9df5d3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13339,6 +13339,7 @@ __metadata: "@webiny/api-tenancy": 0.0.0 "@webiny/api-wcp": 0.0.0 "@webiny/api-websockets": 0.0.0 + "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 "@webiny/error": 0.0.0 "@webiny/handler": 0.0.0 @@ -13348,6 +13349,7 @@ __metadata: "@webiny/plugins": 0.0.0 "@webiny/project-utils": 0.0.0 "@webiny/tasks": 0.0.0 + "@webiny/utils": 0.0.0 graphql: ^15.8.0 rimraf: ^5.0.5 ttypescript: ^1.5.13 From c3c44f6a18021fa6ae11853826a7e45d3115e4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 2 Dec 2024 10:40:57 +0100 Subject: [PATCH 6/8] refactor(api): add generic to service discovery manifest loading --- .../src/enterprise/requestCloning/options.ts | 14 +++++--------- packages/api/src/ServiceDiscovery.ts | 16 ++++++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts index b5909e98a28..455234661dd 100644 --- a/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts @@ -7,7 +7,7 @@ export interface IOptions { s3Bucket: string; } -interface IManifestApiTwoPhasedDeployment { +interface IManifestTwoPhasedDeployment { isPrimary: boolean; s3Region: string; s3Bucket: string; @@ -15,24 +15,20 @@ interface IManifestApiTwoPhasedDeployment { sqsUrl: string; } -interface IManifestApi { - twoPhasedDeployment: IManifestApiTwoPhasedDeployment; -} - interface IManifest { - api: IManifestApi; + twoPhasedDeployment: IManifestTwoPhasedDeployment; } export const getOptions = async (): Promise => { - const manifest = (await ServiceDiscovery.load()) as IManifest | undefined; + const manifest = await ServiceDiscovery.load(); if (!manifest) { console.error("Service manifest not found."); return null; } - const { twoPhasedDeployment } = manifest.api; + const { twoPhasedDeployment } = manifest; - if (!twoPhasedDeployment.isPrimary) { + if (!twoPhasedDeployment?.isPrimary) { return null; } diff --git a/packages/api/src/ServiceDiscovery.ts b/packages/api/src/ServiceDiscovery.ts index d0cad2c97e7..1f2bb667fa4 100644 --- a/packages/api/src/ServiceDiscovery.ts +++ b/packages/api/src/ServiceDiscovery.ts @@ -1,21 +1,21 @@ -import { getDocumentClient, QueryCommand, unmarshall } from "@webiny/aws-sdk/client-dynamodb"; import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; +import { getDocumentClient, QueryCommand, unmarshall } from "@webiny/aws-sdk/client-dynamodb"; import type { GenericRecord } from "~/types"; -interface ServiceManifest { +export interface ServiceManifest { name: string; manifest: Manifest; } -type Manifest = GenericRecord; +export type Manifest = GenericRecord; class ServiceManifestLoader { private client: DynamoDBDocument | undefined; private manifest: Manifest | undefined = undefined; - async load() { + async load(): Promise { if (this.manifest) { - return this.manifest; + return this.manifest as T; } const manifests = await this.loadManifests(); @@ -32,7 +32,7 @@ class ServiceManifestLoader { return { ...acc, [manifest.name]: manifest.manifest }; }, {}); - return this.manifest; + return this.manifest as T; } setDocumentClient(client: DynamoDBDocument) { @@ -68,7 +68,7 @@ export class ServiceDiscovery { serviceManifestLoader.setDocumentClient(client); } - static async load() { - return serviceManifestLoader.load(); + static async load() { + return serviceManifestLoader.load(); } } From 5f9ef50d38bb71533e3b9a3c500e3d3391dd3708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 2 Dec 2024 11:59:26 +0100 Subject: [PATCH 7/8] refactor(api-serverless-cms): request cloning --- packages/api-log/package.json | 2 +- packages/api-serverless-cms/package.json | 4 +- .../src/enterprise/requestCloning/options.ts | 32 ++++-- .../requestCloning/request/RequestTransfer.ts | 16 ++- .../requestCloning/requestCloning.ts | 105 ++++++++++-------- .../requestCloning/s3/S3Transfer.ts | 2 + .../enterprise/requestCloning/s3/constants.ts | 1 + .../src/enterprise/requestCloning/s3/types.ts | 1 + packages/handler/src/fastify.ts | 14 ++- yarn.lock | 5 +- 10 files changed, 112 insertions(+), 70 deletions(-) create mode 100644 packages/api-serverless-cms/src/enterprise/requestCloning/s3/constants.ts diff --git a/packages/api-log/package.json b/packages/api-log/package.json index 2f171869d7d..350dda6fa58 100644 --- a/packages/api-log/package.json +++ b/packages/api-log/package.json @@ -25,7 +25,7 @@ "@webiny/plugins": "0.0.0", "@webiny/tasks": "0.0.0", "@webiny/utils": "0.0.0", - "zod": "^3.22.4" + "zod": "^3.23.8" }, "devDependencies": { "@webiny/cli": "0.0.0", diff --git a/packages/api-serverless-cms/package.json b/packages/api-serverless-cms/package.json index 0c8200edaed..b03570f199f 100644 --- a/packages/api-serverless-cms/package.json +++ b/packages/api-serverless-cms/package.json @@ -14,6 +14,7 @@ "@webiny/api-file-manager": "0.0.0", "@webiny/api-form-builder": "0.0.0", "@webiny/api-headless-cms": "0.0.0", + "date-fns": "^2.22.1", "@webiny/api-i18n": "0.0.0", "@webiny/api-i18n-content": "0.0.0", "@webiny/api-page-builder": "0.0.0", @@ -26,7 +27,8 @@ "@webiny/handler": "0.0.0", "@webiny/handler-client": "0.0.0", "@webiny/handler-graphql": "0.0.0", - "@webiny/utils": "0.0.0" + "@webiny/utils": "0.0.0", + "zod": "^3.23.8" }, "devDependencies": { "@webiny/api-admin-users": "0.0.0", diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts index 455234661dd..b668e78940d 100644 --- a/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/options.ts @@ -1,4 +1,5 @@ import { ServiceDiscovery } from "@webiny/api"; +import zod from "zod"; export interface IOptions { sqsRegion: string; @@ -19,6 +20,16 @@ interface IManifest { twoPhasedDeployment: IManifestTwoPhasedDeployment; } +const optionsValidation = zod.object({ + twoPhasedDeployment: zod.object({ + isPrimary: zod.boolean(), + s3Region: zod.string(), + s3Bucket: zod.string(), + sqsRegion: zod.string(), + sqsUrl: zod.string() + }) +}); + export const getOptions = async (): Promise => { const manifest = await ServiceDiscovery.load(); if (!manifest) { @@ -26,20 +37,17 @@ export const getOptions = async (): Promise => { return null; } - const { twoPhasedDeployment } = manifest; - - if (!twoPhasedDeployment?.isPrimary) { + const validated = await optionsValidation.safeParseAsync(manifest); + if (!validated.success || !validated.data) { + console.error("Service manifest is not valid."); + console.log(validated.error); return null; } - const result: Partial = {}; - const keys = ["sqsRegion", "sqsUrl", "s3Region", "s3Bucket"] as const; - for (const key of keys) { - if (!twoPhasedDeployment[key]) { - console.log(`${key} is not set.`); - return null; - } - result[key] = twoPhasedDeployment[key]; + const { twoPhasedDeployment } = validated.data; + + if (!twoPhasedDeployment?.isPrimary) { + return null; } - return result as IOptions; + return twoPhasedDeployment; }; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts index 06bc1c1266e..c997cbf86cb 100644 --- a/packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/request/RequestTransfer.ts @@ -6,6 +6,7 @@ import type { IRequestTransferSendParams, IRequestTransferSendResult } from "./types"; +import { format as formatDate } from "date-fns"; /** * There are few steps we must go through: @@ -29,14 +30,15 @@ export class RequestTransfer implements IRequestTransfer { } public async send(params: IRequestTransferSendParams): Promise { - const { key, event } = params; + const { event } = params; const body = await compress(JSON.stringify(event)); - const s3Key = `requests/${Date.now()}.${key}.json.gz`; + + const key = this.createRequestTransferKey(params); let result: IS3TransferSendResult; try { result = await this.s3.send({ - key: s3Key, + key, body }); } catch (ex) { @@ -48,7 +50,7 @@ export class RequestTransfer implements IRequestTransfer { try { await this.sqs.send({ body: JSON.stringify({ - type: "s3", + type: result.type, value: result.key }), attributes: [ @@ -64,6 +66,12 @@ export class RequestTransfer implements IRequestTransfer { console.log(ex); } } + + private createRequestTransferKey(params: Pick): string { + const formattedDate = formatDate(new Date(), "yyyy/MM/dd"); + const formattedTime = formatDate(new Date(), "HH:mm:ss"); + return `requests/${formattedDate}/${formattedTime}/${params.key}.json.gz`; + } } export const createRequestTransfer = (params: ITransferRequestParams): IRequestTransfer => { diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts index 384c97e0c0a..b60b1344aab 100644 --- a/packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/requestCloning.ts @@ -10,59 +10,66 @@ import { createRequestTransfer } from "./request"; export const requestCloning = () => { return createModifyFastifyPlugin(instance => { instance.addHook("onRequest", async request => { - /** - * We will not transfer OPTIONS request. Everything else can go into another system. - */ - if (request.method.toLowerCase() === "options") { - return; - } - const event = request.awsLambda?.event; - if (!event) { - console.error(`There is no event to be transferred into another system.`); - return; - } - const lambdaContext = request.awsLambda?.context; - if (!lambdaContext) { - console.error(`There is no context to be transferred into another system.`); - return; - } - const options = await getOptions(); - if (!options) { - return; - } + // @ts-expect-error + request.cloning = (async () => { + /** + * We will not transfer OPTIONS request. Everything else can go into another system. + */ + if (request.method.toLowerCase() === "options") { + return; + } + const event = request.awsLambda?.event; + if (!event) { + console.error(`There is no event to be transferred into another system.`); + return; + } + const lambdaContext = request.awsLambda?.context; + if (!lambdaContext) { + console.error(`There is no context to be transferred into another system.`); + return; + } + const options = await getOptions(); + if (!options) { + /** + * If no options, either there is some error in the options validation or this system is not primary. + * In both cases, we will skip the transfer. + */ + return; + } - const s3Transfer = createS3Transfer({ - client: createS3Client({ - region: options.s3Region - }), - bucket: options.s3Bucket - }); - const sqsTransfer = createSQSTransfer({ - client: createSqsClient({ - region: options.sqsRegion - }), - url: options.sqsUrl - }); - - const transfer = createRequestTransfer({ - s3: s3Transfer, - sqs: sqsTransfer - }); + const s3Transfer = createS3Transfer({ + client: createS3Client({ + region: options.s3Region + }), + bucket: options.s3Bucket + }); + const sqsTransfer = createSQSTransfer({ + client: createSqsClient({ + region: options.sqsRegion + }), + url: options.sqsUrl + }); - const key = createCacheKey(event, { - algorithm: "sha512" - }); + const requestTransfer = createRequestTransfer({ + s3: s3Transfer, + sqs: sqsTransfer + }); - try { - await transfer.send({ - key, - event + const key = createCacheKey(event, { + algorithm: "sha512" }); - } catch (ex) { - console.error("Failed to transfer the request to another system."); - console.log(ex); - throw ex; - } + + try { + await requestTransfer.send({ + key, + event + }); + } catch (ex) { + console.error("Failed to transfer the request to another system."); + console.log(ex); + throw ex; + } + })(); }); }); }; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts index 0aa072dd11e..854c4ba9143 100644 --- a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/S3Transfer.ts @@ -5,6 +5,7 @@ import { PutObjectCommandOutput, S3Client } from "@webiny/aws-sdk/client-s3"; +import { TRANSFER_TYPE_S3 } from "./constants"; export interface IS3TransferParams { client: S3Client; @@ -47,6 +48,7 @@ export class S3Transfer implements IS3Transfer { } return { + type: TRANSFER_TYPE_S3, key: params.key, eTag, statusCode diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/constants.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/constants.ts new file mode 100644 index 00000000000..51543f3e340 --- /dev/null +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/constants.ts @@ -0,0 +1 @@ +export const TRANSFER_TYPE_S3 = "s3"; diff --git a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts index f3192432cca..11d715bb757 100644 --- a/packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts +++ b/packages/api-serverless-cms/src/enterprise/requestCloning/s3/types.ts @@ -3,6 +3,7 @@ export interface IS3TransferSendParams { body: string | Buffer; } export interface IS3TransferSendResult { + type: string; key: string; eTag: string; statusCode: 200 | unknown; diff --git a/packages/handler/src/fastify.ts b/packages/handler/src/fastify.ts index de1c2fac3e3..0b4cf06a0e3 100644 --- a/packages/handler/src/fastify.ts +++ b/packages/handler/src/fastify.ts @@ -500,12 +500,24 @@ export const createHandler = (params: CreateHandlerParams) => { /** * We need to output the benchmark results at the end of the request in both response and timeout cases */ - app.addHook("onResponse", async () => { + app.addHook("onResponse", async request => { await context.benchmark.output(); + // @ts-expect-error + if (!request.cloning) { + return; + } + // @ts-expect-error + await request.cloning; }); app.addHook("onTimeout", async () => { await context.benchmark.output(); + // @ts-expect-error + if (!request.cloning) { + return; + } + // @ts-expect-error + await request.cloning; }); /** diff --git a/yarn.lock b/yarn.lock index 02bea9df5d3..ee85f08509f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12828,7 +12828,7 @@ __metadata: rimraf: ^5.0.5 ttypescript: ^1.5.12 typescript: 4.9.5 - zod: ^3.22.4 + zod: ^3.23.8 languageName: unknown linkType: soft @@ -13354,6 +13354,7 @@ __metadata: rimraf: ^5.0.5 ttypescript: ^1.5.13 typescript: 4.9.5 + zod: ^3.23.8 languageName: unknown linkType: soft @@ -39289,7 +39290,7 @@ __metadata: languageName: node linkType: hard -"zod@npm:^3.22.4, zod@npm:^3.23.8": +"zod@npm:^3.23.8": version: 3.23.8 resolution: "zod@npm:3.23.8" checksum: 15949ff82118f59c893dacd9d3c766d02b6fa2e71cf474d5aa888570c469dbf5446ac5ad562bb035bf7ac9650da94f290655c194f4a6de3e766f43febd432c5c From 5952435aa60d842c450bf7ca8071ddad55d2e97b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 2 Dec 2024 12:22:45 +0100 Subject: [PATCH 8/8] refactor(api-serverless-cms): add date as request key --- packages/api-serverless-cms/package.json | 2 +- yarn.lock | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/api-serverless-cms/package.json b/packages/api-serverless-cms/package.json index b03570f199f..7fe817a7799 100644 --- a/packages/api-serverless-cms/package.json +++ b/packages/api-serverless-cms/package.json @@ -14,7 +14,6 @@ "@webiny/api-file-manager": "0.0.0", "@webiny/api-form-builder": "0.0.0", "@webiny/api-headless-cms": "0.0.0", - "date-fns": "^2.22.1", "@webiny/api-i18n": "0.0.0", "@webiny/api-i18n-content": "0.0.0", "@webiny/api-page-builder": "0.0.0", @@ -28,6 +27,7 @@ "@webiny/handler-client": "0.0.0", "@webiny/handler-graphql": "0.0.0", "@webiny/utils": "0.0.0", + "date-fns": "^2.22.1", "zod": "^3.23.8" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index ee85f08509f..c62bef3be8d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13350,6 +13350,7 @@ __metadata: "@webiny/project-utils": 0.0.0 "@webiny/tasks": 0.0.0 "@webiny/utils": 0.0.0 + date-fns: ^2.22.1 graphql: ^15.8.0 rimraf: ^5.0.5 ttypescript: ^1.5.13