From 911a498e68c5be0f5f76b3a0fce2616ee05798c2 Mon Sep 17 00:00:00 2001 From: Jinser Kafka Date: Wed, 5 Jun 2024 00:35:34 +0800 Subject: [PATCH 1/2] test: add dynamodb integration test --- integration_tests/dynamodb/create_sink.sql | 13 +++++ integration_tests/dynamodb/create_source.sql | 6 +++ .../dynamodb/create_table_request.json | 15 ++++++ integration_tests/dynamodb/data_check | 1 + integration_tests/dynamodb/docker-compose.yml | 46 ++++++++++++++++++ integration_tests/dynamodb/prepare.sh | 8 ++++ integration_tests/dynamodb/query.sql | 42 ++++++++++++++++ integration_tests/dynamodb/sink_check.py | 48 +++++++++++++++++++ 8 files changed, 179 insertions(+) create mode 100644 integration_tests/dynamodb/create_sink.sql create mode 100644 integration_tests/dynamodb/create_source.sql create mode 100644 integration_tests/dynamodb/create_table_request.json create mode 100644 integration_tests/dynamodb/data_check create mode 100644 integration_tests/dynamodb/docker-compose.yml create mode 100644 integration_tests/dynamodb/prepare.sh create mode 100644 integration_tests/dynamodb/query.sql create mode 100644 integration_tests/dynamodb/sink_check.py diff --git a/integration_tests/dynamodb/create_sink.sql b/integration_tests/dynamodb/create_sink.sql new file mode 100644 index 0000000000000..6de71404a9da1 --- /dev/null +++ b/integration_tests/dynamodb/create_sink.sql @@ -0,0 +1,13 @@ +CREATE SINK dyn_sink +FROM + movies +WITH +( + connector = 'dynamodb', + table = 'Movies', + primary_key = 'year,title', + endpoint = 'http://dynamodb:8000', + region = 'us', + access_key = 'ac', + secret_key = 'sk' +); diff --git a/integration_tests/dynamodb/create_source.sql b/integration_tests/dynamodb/create_source.sql new file mode 100644 index 0000000000000..24cd3ed52e3fd --- /dev/null +++ b/integration_tests/dynamodb/create_source.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS movies ( + year integer, + title varchar, + description varchar, + primary key (year, title) +); diff --git a/integration_tests/dynamodb/create_table_request.json b/integration_tests/dynamodb/create_table_request.json new file mode 100644 index 0000000000000..b10747e1116d5 --- /dev/null +++ b/integration_tests/dynamodb/create_table_request.json @@ -0,0 +1,15 @@ +{ + "TableName": "Movies", + "KeySchema": [ + { "AttributeName": "year", "KeyType": "HASH" }, + { "AttributeName": "title", "KeyType": "RANGE" } + ], + "AttributeDefinitions": [ + { "AttributeName": "year", "AttributeType": "N" }, + { "AttributeName": "title", "AttributeType": "S" } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5 + } +} diff --git a/integration_tests/dynamodb/data_check b/integration_tests/dynamodb/data_check new file mode 100644 index 0000000000000..991948c9a7971 --- /dev/null +++ b/integration_tests/dynamodb/data_check @@ -0,0 +1 @@ +movies diff --git a/integration_tests/dynamodb/docker-compose.yml b/integration_tests/dynamodb/docker-compose.yml new file mode 100644 index 0000000000000..0bc6a50849976 --- /dev/null +++ b/integration_tests/dynamodb/docker-compose.yml @@ -0,0 +1,46 @@ +--- +version: "3" +services: + dynamodb: + image: amazon/dynamodb-local + ports: + - "8000:8000" + command: "-jar DynamoDBLocal.jar -sharedDb -inMemory -port 8000" + risingwave-standalone: + extends: + file: ../../docker/docker-compose.yml + service: risingwave-standalone + postgres-0: + extends: + file: ../../docker/docker-compose.yml + service: postgres-0 + grafana-0: + extends: + file: ../../docker/docker-compose.yml + service: grafana-0 + minio-0: + extends: + file: ../../docker/docker-compose.yml + service: minio-0 + prometheus-0: + extends: + file: ../../docker/docker-compose.yml + service: prometheus-0 + message_queue: + extends: + file: ../../docker/docker-compose.yml + service: message_queue +volumes: + risingwave-standalone: + external: false + postgres-0: + external: false + grafana-0: + external: false + minio-0: + external: false + prometheus-0: + external: false + message_queue: + external: false +name: risingwave-compose diff --git a/integration_tests/dynamodb/prepare.sh b/integration_tests/dynamodb/prepare.sh new file mode 100644 index 0000000000000..29a8009b1635a --- /dev/null +++ b/integration_tests/dynamodb/prepare.sh @@ -0,0 +1,8 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +curl -X POST http://localhost:8000/ \ + -H "Accept-Encoding: identity" \ + -H "X-Amz-Target: DynamoDB_20120810.CreateTable" \ + -H "Content-Type: application/x-amz-json-1.0" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=DUMMY/20240601/us/dynamodb/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=DUMMY" \ + --data "@$SCRIPT_DIR/create_table_request.json" diff --git a/integration_tests/dynamodb/query.sql b/integration_tests/dynamodb/query.sql new file mode 100644 index 0000000000000..8ae514743ae97 --- /dev/null +++ b/integration_tests/dynamodb/query.sql @@ -0,0 +1,42 @@ +INSERT INTO + movies (year, title, description) +VALUES + (2020, 'The Emoji Movie', 'a'), + (2019, 'Avengers: Endgame', 'b'), + (2018, 'Black Panther', 'c'), + (2017, 'Wonder Woman', 'd'); + +INSERT INTO + movies (year, title, description) +VALUES + (2023, 'Beautiful beauty', 'ABC DUMMY'); + +FLUSH; + +DELETE FROM + movies +WHERE + title = 'Beautiful beauty'; + +DELETE FROM + movies +WHERE + year = 2017; + +FLUSH; + +INSERT INTO + movies (year, title, description) +VALUES + (2017, 'Beautiful beauty', 'ABC DUMMY'); + +FLUSH; + +UPDATE + movies +SET + description = 'ABC' +WHERE + year = 2017; + +FLUSH; diff --git a/integration_tests/dynamodb/sink_check.py b/integration_tests/dynamodb/sink_check.py new file mode 100644 index 0000000000000..2bfb9b6fe1352 --- /dev/null +++ b/integration_tests/dynamodb/sink_check.py @@ -0,0 +1,48 @@ +import sys +import subprocess +import json + +curl = """ +curl -X POST http://localhost:8000/ \ + -H "Accept-Encoding: identity" \ + -H "X-Amz-Target: DynamoDB_20120810.Scan" \ + -H "Content-Type: application/x-amz-json-1.0" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=DUMMY/20240601/us/dynamodb/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=DUMMY" \ + --data '{"TableName": "Movies"}' +""" + +output = subprocess.Popen( + ["bash", "-c", curl,], + stdout=subprocess.PIPE, +) +msgs = json.loads(output.stdout.read()) +print(msgs) + +item_count = msgs["Count"] +items = msgs["Items"] +print("item count: ", item_count) +print("items: ", items) + +output.stdout.close() +output.wait() + +if item_count != 4: + print("Data check failed") + sys.exit(1) + +for item in items: + title = item["title"]["S"] + year = item["year"]["N"] + description = item["description"]["S"] + + if title not in ["Black Panther", "Avengers: Endgame", "Beautiful beauty", "The Emoji Movie"]: + print(f"Data check failed: unexcept title: {title}") + sys.exit(1) + + if int(year) not in range(2017, 2021): + print(f"Data check failed: unexcept year: {year}") + sys.exit(1) + + if description not in ["a", "b", "c", "ABC"]: + print(f"Data check failed: unexcept description: {description}") + sys.exit(1) From 8907f11141d5543477f585f11f68e9e22c01346d Mon Sep 17 00:00:00 2001 From: Jinser Kafka Date: Wed, 5 Jun 2024 00:37:28 +0800 Subject: [PATCH 2/2] ci: add dynamodb integration test to the pipeline --- ci/scripts/gen-integration-test-yaml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/gen-integration-test-yaml.py b/ci/scripts/gen-integration-test-yaml.py index 3d1dec7f66914..c778205cfbb3e 100644 --- a/ci/scripts/gen-integration-test-yaml.py +++ b/ci/scripts/gen-integration-test-yaml.py @@ -45,6 +45,7 @@ 'client-library': ['none'], 'kafka-cdc': ['json'], 'pubsub': ['json'], + 'dynamodb': ['json'], } def gen_pipeline_steps():