From 22fac8e3891bef7d64cf229374eac36200edebc8 Mon Sep 17 00:00:00 2001 From: reebhub Date: Mon, 23 Dec 2024 23:50:25 +0200 Subject: [PATCH] AWS SQS --> Amazon SQS, added a "Queue methods" panel with FIFO queue usage sample and deduplication message description, clarified loadTo format in relation to FIFO usage --- .../add-connection-string.dotnet.markdown | 4 +- .../maintenance/etl/add-etl.dotnet.markdown | 12 +- .../server/ongoing-tasks/etl/basics.markdown | 2 +- .../ongoing-tasks/etl/queue-etl/.docs.json | 2 +- .../{aws-sqs.markdown => amazon-sqs.markdown} | 154 +++++++++++------- .../etl/queue-etl/overview.markdown | 8 +- .../database/tasks/ongoing-tasks/.docs.json | 2 +- ...s-etl.markdown => amazon-sqs-etl.markdown} | 16 +- .../{AWS-SQS_Etl.cs => amazon-sqs_etl.cs} | 47 +++++- 9 files changed, 166 insertions(+), 81 deletions(-) rename Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/{aws-sqs.markdown => amazon-sqs.markdown} (65%) rename Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/{aws-sqs-etl.markdown => amazon-sqs-etl.markdown} (93%) rename Documentation/7.0/Samples/csharp/Raven.Documentation.Samples/Server/OngoingTasks/ETL/Queue/{AWS-SQS_Etl.cs => amazon-sqs_etl.cs} (84%) diff --git a/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/connection-strings/add-connection-string.dotnet.markdown b/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/connection-strings/add-connection-string.dotnet.markdown index a77a54a30..308083d67 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/connection-strings/add-connection-string.dotnet.markdown +++ b/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/connection-strings/add-connection-string.dotnet.markdown @@ -123,8 +123,8 @@ Learn to add an Azure Queue Storage connection string in the [Add an Azure Queue {PANEL: Add an Amazon SQS connection string} -Amazon SQS connection strings are used by RavenDB [Amazon SQS ETL Tasks](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs). -Learn to add an SQS connection string in [this section](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#add-an-aws-sqs-connection-string). +Amazon SQS connection strings are used by RavenDB [Amazon SQS ETL Tasks](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs). +Learn to add an SQS connection string in [this section](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#add-an-amazon-sqs-connection-string). {PANEL/} diff --git a/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/etl/add-etl.dotnet.markdown b/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/etl/add-etl.dotnet.markdown index 2e764552b..8f3bacaab 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/etl/add-etl.dotnet.markdown +++ b/Documentation/7.0/Raven.Documentation.Pages/client-api/operations/maintenance/etl/add-etl.dotnet.markdown @@ -18,7 +18,7 @@ * [Add Kafka ETL task](../../../../client-api/operations/maintenance/etl/add-etl#add-kafka-etl-task) * [Add RabbitMQ ETL task](../../../../client-api/operations/maintenance/etl/add-etl#add-rabbitmq-etl-task) * [Add Azure Queue Storage ETL task](../../../../client-api/operations/maintenance/etl/add-etl#add-azure-queue-storage-etl-task) - * [Add AWS SQS ETL task](../../../../client-api/operations/maintenance/etl/add-etl#add-aws-sqs-etl-task) + * [Add Amazon SQS ETL task](../../../../client-api/operations/maintenance/etl/add-etl#add-amazon-sqs-etl-task) * [Syntax](../../../../client-api/operations/maintenance/etl/add-etl#syntax) {NOTE/} @@ -130,14 +130,14 @@ The following example adds an Elasticsearch ETL task: {PANEL/} -{PANEL: Add AWS SQS ETL task} +{PANEL: Add Amazon SQS ETL task} -* Learn about the AWS SQS ETL task in the **[AWS SQS ETL task](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs)** article. - * [This section](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#add-an-aws-sqs-connection-string) +* Learn about the AWS SQS ETL task in the **[Amazon SQS ETL task](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs)** article. + * [This section](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#add-an-amazon-sqs-connection-string) shows how to define a connection string to the SQS destination. - * [This section](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#add-an-aws-sqs-etl-task) + * [This section](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#add-an-amazon-sqs-etl-task) shows how to run an ETL task that uses the defined connection string. -* To learn how to manage the task from Studio, see **[Studio: AWS SQS ETL Task](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl)**. +* To learn how to manage the task from Studio, see **[Studio: Amazon SQS ETL Task](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl)**. {PANEL/} diff --git a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/basics.markdown b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/basics.markdown index 69ca30fed..f186f5cf4 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/basics.markdown +++ b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/basics.markdown @@ -63,7 +63,7 @@ * [Kafka ETL](../../../server/ongoing-tasks/etl/queue-etl/kafka) - send data to a _Kafka message broker_ * [RabbitMQ ETL](../../../server/ongoing-tasks/etl/queue-etl/rabbit-mq) - send data to an _RabbitMQ exchange_ * [Azure Queue Storage ETL](../../../server/ongoing-tasks/etl/queue-etl/azure-queue) - send data to an _Azure Queue Storage message queue_ - * [AWS SQS ETL](../../../server/ongoing-tasks/etl/queue-etl/aws-sqs) - send data to an _AWS SQS message queue_ + * [Amazon SQS ETL](../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs) - send data to an _Amazon SQS message queue_ * All ETL tasks can be defined from the Client API or from the [Studio](../../../studio/database/tasks/ongoing-tasks/general-info). diff --git a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/.docs.json b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/.docs.json index 484b9c3c8..1ad4654b8 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/.docs.json +++ b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/.docs.json @@ -24,7 +24,7 @@ "Mappings": [] }, { - "Path": "aws-sqs.markdown", + "Path": "amazon-sqs.markdown", "Name": "Amazon SQS ETL", "DiscussionId": "1b5b21e5-5e51-4717-a467-4a31694271cf", "Mappings": [] diff --git a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/aws-sqs.markdown b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/amazon-sqs.markdown similarity index 65% rename from Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/aws-sqs.markdown rename to Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/amazon-sqs.markdown index e30ac50b9..d050f7b57 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/aws-sqs.markdown +++ b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/amazon-sqs.markdown @@ -12,28 +12,31 @@ * Create an **Amazon SQS ETL Task** to: * **Extract** data from a RavenDB database, * **Transform** the data using one or more custom scripts, - * and **Load** the resulting JSON object to an SQS destination in - [CloudEvents]([CloudEvents messages](https://cloudevents.io) format. + * and **Load** the resulting JSON object to an SQS destination + in [CloudEvents messages](https://cloudevents.io) format. {INFO: } This article focuses on the creation of an Amazon SQS ETL task using the Client API. -To define an Amazon SQS ETL task from Studio, see [Studio: Amazon SQS ETL Task](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl). +To define an Amazon SQS ETL task from Studio, see [Studio: Amazon SQS ETL Task](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl). For an **overview of Queue ETL tasks**, see [Queue ETL tasks overview](../../../../server/ongoing-tasks/etl/queue-etl/overview). {INFO/} * In this page: - * [RavenDB ETL and Amazon SQS](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#ravendb-etl-and-amazon-sqs) - * [Queue methods](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#queue-methods) - * [Add an Amazon SQS connection string](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#add-an-amazon-sqs-connection-string) - * [Authentication methods](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#authentication-methods) - * [Example](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#example) - * [Syntax](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#syntax) - * [Add an Amazon SQS ETL task](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#add-an-amazon-sqs-etl-task) - * [Example: Add SQS ETL task](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#example-add-sqs-etl-task) - * [Delete processed documents](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#delete-processed-documents) - * [Syntax](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#syntax-1) - * [The transformation script](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#the-transformation-script) - * [The loadTo method](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#the-loadto-method) + * [RavenDB ETL and Amazon SQS](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#ravendb-etl-and-amazon-sqs) + * [Queue methods](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#queue-methods) + * [Standard queueing](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#standard-queueing) + * [FIFO queueing](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#fifo-queueing) + * [Caution: ETL message size -vs- Queue message size](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#caution-etl-message-size--vs--queue-message-size) + * [Add an Amazon SQS connection string](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#add-an-amazon-sqs-connection-string) + * [Authentication methods](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#authentication-methods) + * [Example](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#example) + * [Syntax](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#syntax) + * [Add an Amazon SQS ETL task](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#add-an-amazon-sqs-etl-task) + * [Example: Add SQS ETL task](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#example-add-sqs-etl-task) + * [Delete processed documents](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#delete-processed-documents) + * [Syntax](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#syntax-1) + * [The transformation script](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#the-transformation-script) + * [The loadTo method](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#the-loadto-method) {NOTE/} @@ -56,40 +59,60 @@ For an **overview of Queue ETL tasks**, see [Queue ETL tasks overview](../../../ Read more about Amazon SQS in the platform's [official documentation](https://docs.aws.amazon.com/sqs/). {INFO/} ---- +{PANEL/} -#### Queue methods +{PANEL: Queue methods} The data that ETL tasks handle is carefully selected and tailored for specific user needs. Selecting which Queue Type Amazon SQS would use should also take into account the specific nature of the transferred data. -* **Standard queueing** offers an extremely high transfer rate but lacks the ability to ensure - that messages would arrive in the same order they were sent or prevent their duplication. - - {INFO: } - Use this method when quick delivery takes precedence over messages order and distinctness - or the recepient can make up for them. - {INFO/} - -* **FIFO queueing** controls delivery order using a First-In-First-Out queue and ensures - the delivery of each message exactly once, in exchange for a much slower transfer rate - than that of the Standard Queueing method. - * Deduplication: - FIFO queues automatically prevent duplicate messages within a _deduplication interval_. - Interval default: 5 minutes - Deduplication is achieved using a unique _Message Deduplication ID_ that is given to - each message. - * Message Grouping: - Messages with the same _Message Group ID_ are processed in order. - Each group is handled independently, allowing parallel processing across different message groups. - - {INFO: } - Use this method when throughput is not as important as the order and uniqueness of - arriving messages. - {INFO/} - -{WARNING: ETL message size -vs- Queue message size} +--- + +#### Standard queueing + +Standard queueing offers an extremely high transfer rate but lacks the ability to ensure +that messages would arrive in the same order they were sent or prevent their duplication. + +{INFO: } +Use standard queueing when quick delivery takes precedence over messages order and +distinctness or the recepient can make up for them. +{INFO/} + +--- + +#### FIFO queueing + +FIFO queueing controls delivery order using a First-In-First-Out queue and ensures +the delivery of each message exactly once, in exchange for a much slower transfer rate +than that of the Standard Queueing method. + +To load messages to a FIFO queue, add `.fifo` to the queue name while calling the +transformation script's [loadTo method](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#the-loadto-method): +{CODE sqs_fifo@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} + +* **Deduplication**: + FIFO queues automatically prevent duplicate messages within a _deduplication interval_. + Interval default: 5 minutes + Deduplication is achieved by giving each message a _Message Deduplication ID_ + with a unique **Change Vector**. + If the change vector is longer than 128 characters (Amazon's restriction for + a message deduplication ID) we will truncate the CV and add a `-{9-digit hash}`. + +* **Message Grouping**: + Messages with the same _Message Group ID_ are processed in order. + Each group is handled independently, allowing parallel processing across different message groups. + + {INFO: } + Use this method when throughput is not as important as the order and uniqueness of + arriving messages. + {INFO/} + +--- + +#### Caution: ETL message size -vs- Queue message size + +{WARNING: } Please **be aware** that the maximum size of an SQS queue message is `64 KB`, while the maximum size of an ETL message to the queue is `256 KB`. The significance of this difference is that when a maximum-size ETL message arrives @@ -111,7 +134,7 @@ by properties of the connection string it uses, as shown in the example below. #### Example -{CODE add_sqs_connection_string@Server\OngoingTasks\ETL\Queue\AWS-SQS_Etl.cs /} +{CODE add_sqs_connection_string@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} * **Passwordless** Defines whether to use a password or not. @@ -128,9 +151,9 @@ by properties of the connection string it uses, as shown in the example below. #### Syntax -{CODE amazon_sqs_con_str_settings@Server\OngoingTasks\ETL\Queue\AWS-SQS_Etl.cs /} -{CODE queue_connection_string@Server\OngoingTasks\ETL\Queue\AWS-SQS_Etl.cs /} -{CODE queue_broker_type@Server\OngoingTasks\ETL\Queue\AWS-SQS_Etl.cs /} +{CODE amazon_sqs_con_str_settings@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} +{CODE queue_connection_string@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} +{CODE queue_broker_type@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} {PANEL/} @@ -144,7 +167,7 @@ by properties of the connection string it uses, as shown in the example below. * Load the `orderData` object to the "OrdersQueue" queue on an SQS destination. * For more details about the script and the `loadTo` method, see the transromation script section below. -{CODE add_amazon_sqs_etl_task@Server\OngoingTasks\ETL\Queue\AWS-SQS_Etl.cs /} +{CODE add_amazon_sqs_etl_task@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} --- @@ -154,13 +177,13 @@ by properties of the connection string it uses, as shown in the example below. * To do this, set the optional `Queues` property in the ETL configuration with the list of SQS queues for which processed documents are to be deleted. -{CODE sqs_delete_documents@Server\OngoingTasks\ETL\Queue\AWS-SQS_Etl.cs /} +{CODE sqs_delete_documents@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} --- #### Syntax -{CODE etl_configuration@Server\OngoingTasks\ETL\Queue\AWS-SQS_Etl.cs /} +{CODE etl_configuration@Server\OngoingTasks\ETL\Queue\amazon-sqs_etl.cs /} {PANEL/} @@ -178,15 +201,34 @@ and which SQS Queue to **load** the data to. To specify which SQS queue to load the data to, use either of the following methods in your script. The two methods are equivalent, offering alternative syntax: -* **`loadTo(obj, {attributes})`** +* **`loadTo(obj, {attributes})`** + {CODE-BLOCK:csharp} +loadToOrdersQueue(orderData, { + Id: id(this), + Type: 'com.example.promotions', + Source: '/promotion-campaigns/summer-sale' + } + {CODE-BLOCK/} * Here the target is specified as part of the function name. * The target _<QueueName>_ in this syntax is Not a variable and cannot be used as one, it is simply a string literal of the target's name. -* **`loadTo('QueueName', obj, {attributes})`** - * Here the target is passed as an argument to the method. - * Separating the target name from the `loadTo` command makes it possible to include symbols like `'-'` and `'.'` in target names. - This is not possible when the `loadTo` syntax is used because including special characters in the name of a JavaScript function makes it invalid. +* **`loadTo('QueueName', obj, {attributes})`** +{CODE-BLOCK:csharp} + loadTo('OrdersQueue', orderData, { + Id: id(this), + Type: 'com.example.promotions', + Source: '/promotion-campaigns/summer-sale' + } + {CODE-BLOCK/} + * Here the target is passed as an argument to the method. + * Separating the target name from the `loadTo` command makes it possible to include symbols + like `'-'` and `'.'` in target names. + This is not possible when the `loadTo` syntax is used because including special + characters in the name of a JavaScript function makes it invalid. + * To deliver messages to a [FIFO queue](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#fifo-queueing), + use this format and add `.fifo` to the queue name. + `loadTo('QueueName.fifo', obj, {attributes})` | Parameter | Type | Description | |----------------|--------|----------------------------------------------------------------------------------------------------------------------------------| @@ -242,7 +284,7 @@ loadToOrdersQueue(orderData, { ### Studio -- [Studio: Amazon SQS ETL Task](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl) +- [Studio: Amazon SQS ETL Task](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl) - [Studio: Kafka ETL Task](../../../../studio/database/tasks/ongoing-tasks/kafka-etl-task) - [Studio: RabbitMQ ETL Task](../../../../studio/database/tasks/ongoing-tasks/rabbitmq-etl-task) - [Studio: Azure Queue Storage ETL Task](../../../../studio/database/tasks/ongoing-tasks/azure-queue-storage-etl) diff --git a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/overview.markdown b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/overview.markdown index 74d05fda7..308e53612 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/overview.markdown +++ b/Documentation/7.0/Raven.Documentation.Pages/server/ongoing-tasks/etl/queue-etl/overview.markdown @@ -10,7 +10,7 @@ * **Apache Kafka** * **RabbitMQ** * **Azure Queue Storage** - * **AWS SQS** + * **Amazon SQS** * This functionality is achieved by defining [Queue ETL tasks](../../../../server/ongoing-tasks/etl/queue-etl/overview#queue-etl-tasks) within a RavenDB database. @@ -43,9 +43,9 @@ RavenDB produces messages to broker queues via the following Queue ETL tasks: * **Azure Queue Storage ETL Task** You can define an Azure Queue Storage ETL Task from [Studio](../../../../studio/database/tasks/ongoing-tasks/azure-queue-storage-etl) or using the [Client API](../../../../server/ongoing-tasks/etl/queue-etl/azure-queue). -* **AWS SQS ETL Task** - You can define an AWS SQS ETL Task from [Studio](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl) - or using the [Client API](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs). +* **Amazon SQS ETL Task** + You can define an Amazon SQS ETL Task from [Studio](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl) + or using the [Client API](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs). --- diff --git a/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/.docs.json b/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/.docs.json index 849b1c857..903e8c602 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/.docs.json +++ b/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/.docs.json @@ -54,7 +54,7 @@ "Mappings": [] }, { - "Path": "aws-sqs-etl.markdown", + "Path": "amazon-sqs-etl.markdown", "Name": "Amazon SQS ETL Task", "DiscussionId": "89c14b90-9ddd-426d-b0f4-60a49d5865c0", "Mappings": [] diff --git a/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/aws-sqs-etl.markdown b/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/amazon-sqs-etl.markdown similarity index 93% rename from Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/aws-sqs-etl.markdown rename to Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/amazon-sqs-etl.markdown index 00254f6ee..be7287fa7 100644 --- a/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/aws-sqs-etl.markdown +++ b/Documentation/7.0/Raven.Documentation.Pages/studio/database/tasks/ongoing-tasks/amazon-sqs-etl.markdown @@ -12,15 +12,15 @@ Document extensions like attachments, counters, time series, and revisions are not sent. * This page explains how to create an Amazon SQS ETL task using Studio. - [Learn here](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs) how to define + [Learn here](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs) how to define an Amazon SQS ETL task using the Client API. . * In this page: - * [Ongoing Tasks View](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl#ongoing-tasks-view) - * [Define the Amazon SQS ETL task](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl#define-amazon-sqs-etl-task) - * [Add or Edit Transformation Script](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl#add-or-edit-transformation-script) - * [Comments](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl#comments) - * [ETL message size -vs- Queue message size](../../../../studio/database/tasks/ongoing-tasks/aws-sqs-etl#etl-message-size--vs--queue-message-size) + * [Ongoing Tasks View](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl#ongoing-tasks-view) + * [Define the Amazon SQS ETL task](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl#define-amazon-sqs-etl-task) + * [Add or Edit Transformation Script](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl#add-or-edit-transformation-script) + * [Comments](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl#comments) + * [ETL message size -vs- Queue message size](../../../../studio/database/tasks/ongoing-tasks/amazon-sqs-etl#etl-message-size--vs--queue-message-size) {NOTE/} --- @@ -69,7 +69,7 @@ **Passwordless** requires the machine to be pre-authorized and can only be used in self-hosted mode. * `C.` **Advanced** Click to open per-queue advanced options. - Use this option to determine whether to [delete documents](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs#delete-processed-documents) + Use this option to determine whether to [delete documents](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs#delete-processed-documents) from the database after they are processed. ![Delete processed documents](images/sqs_delete-processed-docs.png "Delete processed documents") @@ -153,7 +153,7 @@ at its destination queue it may be charged for not 1 but 4 queue messages. ### Server - [ETL Basics](../../../../server/ongoing-tasks/etl/basics) -- [Amazon SQS ETL](../../../../server/ongoing-tasks/etl/queue-etl/aws-sqs) +- [Amazon SQS ETL](../../../../server/ongoing-tasks/etl/queue-etl/amazon-sqs) - [Queue ETL Overview](../../../../server/ongoing-tasks/etl/queue-etl/overview) - [Kafka ETL](../../../../server/ongoing-tasks/etl/queue-etl/kafka) - [RabbitMQ ETL](../../../../server/ongoing-tasks/etl/queue-etl/rabbit-mq) diff --git a/Documentation/7.0/Samples/csharp/Raven.Documentation.Samples/Server/OngoingTasks/ETL/Queue/AWS-SQS_Etl.cs b/Documentation/7.0/Samples/csharp/Raven.Documentation.Samples/Server/OngoingTasks/ETL/Queue/amazon-sqs_etl.cs similarity index 84% rename from Documentation/7.0/Samples/csharp/Raven.Documentation.Samples/Server/OngoingTasks/ETL/Queue/AWS-SQS_Etl.cs rename to Documentation/7.0/Samples/csharp/Raven.Documentation.Samples/Server/OngoingTasks/ETL/Queue/amazon-sqs_etl.cs index 900997116..46446a76f 100644 --- a/Documentation/7.0/Samples/csharp/Raven.Documentation.Samples/Server/OngoingTasks/ETL/Queue/AWS-SQS_Etl.cs +++ b/Documentation/7.0/Samples/csharp/Raven.Documentation.Samples/Server/OngoingTasks/ETL/Queue/amazon-sqs_etl.cs @@ -8,7 +8,7 @@ using Raven.Client.Documents.Operations.ETL.Queue; namespace Raven.Documentation.Samples.Server.OngoingTasks.ETL.Queue { - public class AwsSqsEtl + public class AmazonSqsEtl { public void AddConnectionString() { @@ -116,6 +116,49 @@ public void AddAmazonSqsEtlTask() } } + public void AmazonSqsUseFifo() + { + using (var store = new DocumentStore()) + { + Transformation transformation = new Transformation + { + // Input collections + Collections = { "Orders" }, + + ApplyToAllDocuments = false, + + // Transformation script + Name = "scriptName", + + #region sqs_fifo + Script = @"// Create an orderData object + // ========================== + var orderData = { + Id: id(this), // property with RavenDB document ID + OrderLinesCount: this.Lines.length, + TotalCost: 0 + }; + + for (var i = 0; i < this.Lines.length; i++) { + var line = this.Lines[i]; + var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); + orderData.TotalCost += cost; + } + + // Load the object to the FIFO 'Orders' ququq on the SQS destination + // ================================================================= + loadTo('orders.fifo', orderData, { + Id: id(this), + Type: 'com.github.users', + Source: '/registrations/direct-signup' + });" + #endregion + + }; + } + } + + public void DeleteProcessedDocuments() { using (var store = new DocumentStore()) @@ -184,7 +227,7 @@ public class QueueConnectionString : ConnectionString // Configure this when setting a connection string for Azure Queue Storage public AzureQueueStorageConnectionSettings AzureQueueStorageConnectionSettings { get; set; } - // Configure this when setting a connection string for AWS SQS + // Configure this when setting a connection string for Amazon SQS public AmazonSqsConnectionSettings AmazonSqsConnectionSettings { get; set; } } #endregion