From 7ca8136f8d718c8e10799076884b21952da2e490 Mon Sep 17 00:00:00 2001 From: xfz Date: Wed, 22 Nov 2023 19:45:20 +0800 Subject: [PATCH 01/17] test: add elasticsearch-sink test into the workflow --- .../elasticsearch-sink/README.md | 21 +++------- .../elasticsearch-sink/create_es7_sink.sql | 9 ---- .../elasticsearch-sink/create_es8_sink.sql | 9 ---- .../elasticsearch-sink/create_mv.sql | 6 +-- .../elasticsearch-sink/create_sink.sql | 19 +++++++++ .../elasticsearch-sink/docker-compose.yml | 8 ++-- .../elasticsearch-sink/sink_check | 1 + integration_tests/scripts/check_data.py | 2 +- integration_tests/scripts/run_demos.py | 42 +++++++++++++++++++ 9 files changed, 74 insertions(+), 43 deletions(-) delete mode 100644 integration_tests/elasticsearch-sink/create_es7_sink.sql delete mode 100644 integration_tests/elasticsearch-sink/create_es8_sink.sql create mode 100644 integration_tests/elasticsearch-sink/create_sink.sql create mode 100644 integration_tests/elasticsearch-sink/sink_check diff --git a/integration_tests/elasticsearch-sink/README.md b/integration_tests/elasticsearch-sink/README.md index b114e8132024a..53c2335d371ed 100644 --- a/integration_tests/elasticsearch-sink/README.md +++ b/integration_tests/elasticsearch-sink/README.md @@ -2,18 +2,7 @@ In this demo, we want to showcase how RisingWave is able to sink data to ElasticSearch. -1. Set the compose profile accordingly: -Demo with elasticsearch 7: -``` -export COMPOSE_PROFILES=es7 -``` - -Demo with elasticsearch 8 -``` -export COMPOSE_PROFILES=es8 -``` - -2. Launch the cluster: +1. Launch the cluster: ```sh docker-compose up -d @@ -21,13 +10,13 @@ docker-compose up -d The cluster contains a RisingWave cluster and its necessary dependencies, a datagen that generates the data, a single-node elasticsearch for sink. -3. Execute the SQL queries in sequence: +2. Execute the SQL queries in sequence: - create_source.sql - create_mv.sql -- create_es[7/8]_sink.sql +- create_sink.sql -4. Check the contents in ES: +3. Check the contents in ES: ```sh # Check the document counts @@ -38,4 +27,4 @@ curl -XGET -u elastic:risingwave "http://localhost:9200/test/_search" -H 'Conten # Get the first 10 documents sort by user_id curl -XGET -u elastic:risingwave "http://localhost:9200/test/_search?size=10" -H 'Content-Type: application/json' -d'{"query":{"match_all":{}}, "sort": ["user_id"]}' | jq -``` \ No newline at end of file +``` diff --git a/integration_tests/elasticsearch-sink/create_es7_sink.sql b/integration_tests/elasticsearch-sink/create_es7_sink.sql deleted file mode 100644 index 997c238b90344..0000000000000 --- a/integration_tests/elasticsearch-sink/create_es7_sink.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE SINK bhv_es_sink -FROM - bhv_mv WITH ( - connector = 'elasticsearch', - index = 'test', - url = 'http://elasticsearch8:9200', - username = 'elastic', - password = 'risingwave' -); \ No newline at end of file diff --git a/integration_tests/elasticsearch-sink/create_es8_sink.sql b/integration_tests/elasticsearch-sink/create_es8_sink.sql deleted file mode 100644 index 997c238b90344..0000000000000 --- a/integration_tests/elasticsearch-sink/create_es8_sink.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE SINK bhv_es_sink -FROM - bhv_mv WITH ( - connector = 'elasticsearch', - index = 'test', - url = 'http://elasticsearch8:9200', - username = 'elastic', - password = 'risingwave' -); \ No newline at end of file diff --git a/integration_tests/elasticsearch-sink/create_mv.sql b/integration_tests/elasticsearch-sink/create_mv.sql index 0a803f8a2762d..d7d2c0bb8fb64 100644 --- a/integration_tests/elasticsearch-sink/create_mv.sql +++ b/integration_tests/elasticsearch-sink/create_mv.sql @@ -1,7 +1,7 @@ CREATE MATERIALIZED VIEW bhv_mv AS SELECT user_id, - target_id, - event_timestamp + target_id + -- event_timestamp FROM - user_behaviors; \ No newline at end of file + user_behaviors; diff --git a/integration_tests/elasticsearch-sink/create_sink.sql b/integration_tests/elasticsearch-sink/create_sink.sql new file mode 100644 index 0000000000000..e318baac989b1 --- /dev/null +++ b/integration_tests/elasticsearch-sink/create_sink.sql @@ -0,0 +1,19 @@ +CREATE SINK bhv_es7_sink +FROM + bhv_mv WITH ( + connector = 'elasticsearch', + index = 'test', + url = 'http://elasticsearch7:9200', + username = 'elastic', + password = 'risingwave' +); + +CREATE SINK bhv_es8_sink +FROM + bhv_mv WITH ( + connector = 'elasticsearch', + index = 'test', + url = 'http://elasticsearch8:9200', + username = 'elastic', + password = 'risingwave' +); diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index ed1063837843a..db76f02a0cc70 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -2,6 +2,7 @@ version: "3" services: elasticsearch7: + container_name: elasticsearch7 image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 environment: - xpack.security.enabled=true @@ -9,18 +10,15 @@ services: - ELASTIC_PASSWORD=risingwave ports: - 9200:9200 - profiles: - - es7 elasticsearch8: + container_name: elasticsearch8 image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 environment: - xpack.security.enabled=true - discovery.type=single-node - ELASTIC_PASSWORD=risingwave ports: - - 9200:9200 - profiles: - - es8 + - 9201:9200 compactor-0: extends: file: ../../docker/docker-compose.yml diff --git a/integration_tests/elasticsearch-sink/sink_check b/integration_tests/elasticsearch-sink/sink_check new file mode 100644 index 0000000000000..9daeafb9864cf --- /dev/null +++ b/integration_tests/elasticsearch-sink/sink_check @@ -0,0 +1 @@ +test diff --git a/integration_tests/scripts/check_data.py b/integration_tests/scripts/check_data.py index 4888deaab0618..a4a35a7e03414 100644 --- a/integration_tests/scripts/check_data.py +++ b/integration_tests/scripts/check_data.py @@ -54,7 +54,7 @@ def run_psql(sql): demo = sys.argv[1] upstream = sys.argv[2] # mysql, postgres, etc. see scripts/integration_tests.sh -if demo in ['docker', 'iceberg-sink','clickhouse-sink', 'iceberg-cdc', 'kafka-cdc-sink', 'cassandra-and-scylladb-sink']: +if demo in ['docker', 'iceberg-sink','clickhouse-sink', 'iceberg-cdc', 'kafka-cdc-sink', 'cassandra-and-scylladb-sink', 'elasticsearch-sink']: print('Skip for running test for `%s`' % demo) sys.exit(0) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index 5487ec6b40009..8768d9160c3db 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -6,6 +6,7 @@ import subprocess from time import sleep import argparse +import json def run_sql_file(f: str, dir: str): @@ -231,6 +232,45 @@ def run_cassandra_and_scylladb_sink_demo(): if len(failed_cases) != 0: raise Exception("Data check failed for case {}".format(failed_cases)) +def run_elasticsearch_sink_demo(): + demo = "elasticsearch-sink" + file_dir = dirname(abspath(__file__)) + project_dir = dirname(file_dir) + demo_dir = os.path.join(project_dir, demo) + print("Running demo: {}".format(demo)) + + subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) + sleep(40) + + sql_files = ['create_source.sql', 'create_mv.sql', 'create_sink.sql'] + for fname in sql_files: + sql_file = os.path.join(demo_dir, fname) + print("executing sql: ", open(sql_file).read()) + run_sql_file(sql_file, demo_dir) + + print("sink created. Wait for half min time for ingestion") + + # wait for half min ingestion + sleep(30) + + versions = ['7', '8'] + sink_check_file = os.path.join(demo_dir, 'sink_check') + with open(sink_check_file) as f: + relations = f.read().strip().split(",") + failed_cases = [] + for rel in relations: + query = 'curl -XGET -u elastic:risingwave "http://localhost:9200/test/_count" -H "Content-Type: application/json"' + for v in versions: + es = 'elasticsearch{}'.format(v) + print("Running Query: {} on {}".format(query, es)) + counts = subprocess.check_output(["docker", "compose", "exec", es, "bash", "-c", query], cwd=demo_dir) + counts = json.loads(counts)['count'] + print("{} counts in {}_{}".format(counts, es, rel)) + if counts < 1: + failed_cases.append(es + '_' + rel) + if len(failed_cases) != 0: + raise Exception("Data check failed for case {}".format(failed_cases)) + arg_parser = argparse.ArgumentParser(description='Run the demo') arg_parser.add_argument('--format', metavar='format', @@ -259,5 +299,7 @@ def run_cassandra_and_scylladb_sink_demo(): run_kafka_cdc_demo() elif args.case == "cassandra-and-scylladb-sink": run_cassandra_and_scylladb_sink_demo() +elif args.case == "elasticsearch-sink": + run_elasticsearch_sink_demo() else: run_demo(args.case, args.format) From bac1d3f9e88255b80fab8842fa95de868edc151c Mon Sep 17 00:00:00 2001 From: xfz Date: Wed, 22 Nov 2023 19:54:04 +0800 Subject: [PATCH 02/17] fix --- integration_tests/scripts/run_demos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index 8768d9160c3db..7e4dd0cac4edc 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -259,7 +259,7 @@ def run_elasticsearch_sink_demo(): relations = f.read().strip().split(",") failed_cases = [] for rel in relations: - query = 'curl -XGET -u elastic:risingwave "http://localhost:9200/test/_count" -H "Content-Type: application/json"' + query = 'curl -XGET -u elastic:risingwave "http://localhost:9200/{}/_count" -H "Content-Type: application/json"'.format(rel) for v in versions: es = 'elasticsearch{}'.format(v) print("Running Query: {} on {}".format(query, es)) From 1fd533b9f6c2b5dfd3254af64ae82e5e72bde3d6 Mon Sep 17 00:00:00 2001 From: xfz Date: Wed, 22 Nov 2023 19:59:10 +0800 Subject: [PATCH 03/17] update workflow yml --- ci/workflows/integration-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/workflows/integration-tests.yml b/ci/workflows/integration-tests.yml index cff7767812e29..19c5af18dc863 100644 --- a/ci/workflows/integration-tests.yml +++ b/ci/workflows/integration-tests.yml @@ -116,6 +116,7 @@ steps: - "cockroach-sink" - "kafka-cdc-sink" - "cassandra-and-scylladb-sink" + - "elasticsearch-sink" format: - "json" - "protobuf" @@ -164,3 +165,7 @@ steps: testcase: "cassandra-and-scylladb-sink" format: "protobuf" skip: true + - with: + testcase: "elasticsearch-sink" + format: "protobuf" + skip: true From fdaae3ab764ac1a8c0137e6194272ff14ceb868c Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 10:02:08 +0800 Subject: [PATCH 04/17] update --- integration_tests/scripts/run_demos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index 7e4dd0cac4edc..b2726e7fb5174 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -240,7 +240,7 @@ def run_elasticsearch_sink_demo(): print("Running demo: {}".format(demo)) subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) - sleep(40) + sleep(60) sql_files = ['create_source.sql', 'create_mv.sql', 'create_sink.sql'] for fname in sql_files: From 2c682fafd7605883b69a9b131cf3484360434d71 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 11:48:53 +0800 Subject: [PATCH 05/17] update --- integration_tests/elasticsearch-sink/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index db76f02a0cc70..6e2b5df476026 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -2,7 +2,6 @@ version: "3" services: elasticsearch7: - container_name: elasticsearch7 image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 environment: - xpack.security.enabled=true @@ -11,7 +10,6 @@ services: ports: - 9200:9200 elasticsearch8: - container_name: elasticsearch8 image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 environment: - xpack.security.enabled=true From 92519cdb4a270c1a69f46cbbd843e1a4f67defd2 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 13:03:58 +0800 Subject: [PATCH 06/17] update --- integration_tests/scripts/run_demos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index b2726e7fb5174..de4d95c830118 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -253,7 +253,7 @@ def run_elasticsearch_sink_demo(): # wait for half min ingestion sleep(30) - versions = ['7', '8'] + versions = ['8'] sink_check_file = os.path.join(demo_dir, 'sink_check') with open(sink_check_file) as f: relations = f.read().strip().split(",") From e64b6055913ba87420c5ff1693cc866788443a50 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 13:05:22 +0800 Subject: [PATCH 07/17] update --- .../elasticsearch-sink/create_sink.sql | 18 +++++++++--------- .../elasticsearch-sink/docker-compose.yml | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/integration_tests/elasticsearch-sink/create_sink.sql b/integration_tests/elasticsearch-sink/create_sink.sql index e318baac989b1..39d9711338d9a 100644 --- a/integration_tests/elasticsearch-sink/create_sink.sql +++ b/integration_tests/elasticsearch-sink/create_sink.sql @@ -1,12 +1,12 @@ -CREATE SINK bhv_es7_sink -FROM - bhv_mv WITH ( - connector = 'elasticsearch', - index = 'test', - url = 'http://elasticsearch7:9200', - username = 'elastic', - password = 'risingwave' -); +-- CREATE SINK bhv_es7_sink +-- FROM + -- bhv_mv WITH ( + -- connector = 'elasticsearch', + -- index = 'test', + -- url = 'http://elasticsearch7:9200', + -- username = 'elastic', + -- password = 'risingwave' +-- ); CREATE SINK bhv_es8_sink FROM diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index 6e2b5df476026..393538a88aa32 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -1,14 +1,14 @@ --- version: "3" services: - elasticsearch7: - image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 - environment: - - xpack.security.enabled=true - - discovery.type=single-node - - ELASTIC_PASSWORD=risingwave - ports: - - 9200:9200 + # elasticsearch7: + # image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 + # environment: + # - xpack.security.enabled=true + # - discovery.type=single-node + # - ELASTIC_PASSWORD=risingwave + # ports: + # - 9200:9200 elasticsearch8: image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 environment: From fa5f39e5da77d638eb2e17e5d66624348e791985 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 13:11:39 +0800 Subject: [PATCH 08/17] update --- .../elasticsearch-sink/docker-compose.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index 393538a88aa32..6e2b5df476026 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -1,14 +1,14 @@ --- version: "3" services: - # elasticsearch7: - # image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 - # environment: - # - xpack.security.enabled=true - # - discovery.type=single-node - # - ELASTIC_PASSWORD=risingwave - # ports: - # - 9200:9200 + elasticsearch7: + image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 + environment: + - xpack.security.enabled=true + - discovery.type=single-node + - ELASTIC_PASSWORD=risingwave + ports: + - 9200:9200 elasticsearch8: image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 environment: From b61a48632a2a26929096af0c87a025c47bc6c50b Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 13:20:50 +0800 Subject: [PATCH 09/17] update --- .../elasticsearch-sink/create_sink.sql | 18 +++++++++--------- integration_tests/scripts/run_demos.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/integration_tests/elasticsearch-sink/create_sink.sql b/integration_tests/elasticsearch-sink/create_sink.sql index 39d9711338d9a..e318baac989b1 100644 --- a/integration_tests/elasticsearch-sink/create_sink.sql +++ b/integration_tests/elasticsearch-sink/create_sink.sql @@ -1,12 +1,12 @@ --- CREATE SINK bhv_es7_sink --- FROM - -- bhv_mv WITH ( - -- connector = 'elasticsearch', - -- index = 'test', - -- url = 'http://elasticsearch7:9200', - -- username = 'elastic', - -- password = 'risingwave' --- ); +CREATE SINK bhv_es7_sink +FROM + bhv_mv WITH ( + connector = 'elasticsearch', + index = 'test', + url = 'http://elasticsearch7:9200', + username = 'elastic', + password = 'risingwave' +); CREATE SINK bhv_es8_sink FROM diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index de4d95c830118..aa0a0f6575477 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -240,7 +240,7 @@ def run_elasticsearch_sink_demo(): print("Running demo: {}".format(demo)) subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) - sleep(60) + sleep(120) sql_files = ['create_source.sql', 'create_mv.sql', 'create_sink.sql'] for fname in sql_files: @@ -253,7 +253,7 @@ def run_elasticsearch_sink_demo(): # wait for half min ingestion sleep(30) - versions = ['8'] + versions = ['7', '8'] sink_check_file = os.path.join(demo_dir, 'sink_check') with open(sink_check_file) as f: relations = f.read().strip().split(",") From 05009e007d2360f66b155c223f65b9095f752b0a Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 13:46:51 +0800 Subject: [PATCH 10/17] check es8 log --- integration_tests/scripts/run_demos.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index aa0a0f6575477..4f7bc2ae76b7f 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -242,6 +242,8 @@ def run_elasticsearch_sink_demo(): subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) sleep(120) + subprocess.run(["docker", "compose", "logs", "elasticsearch8"], cwd=demo_dir, check=True) + sql_files = ['create_source.sql', 'create_mv.sql', 'create_sink.sql'] for fname in sql_files: sql_file = os.path.join(demo_dir, fname) From 2a79b02a9f472c716d4b85adfa941233681f7c39 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 14:14:48 +0800 Subject: [PATCH 11/17] update es8 docker version --- integration_tests/elasticsearch-sink/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index 6e2b5df476026..1861a41896215 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -10,7 +10,7 @@ services: ports: - 9200:9200 elasticsearch8: - image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1 environment: - xpack.security.enabled=true - discovery.type=single-node From 85c5d90c7ec2cf402dc4bc75aa7596e4d5d791a7 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 14:27:06 +0800 Subject: [PATCH 12/17] ignore por --- integration_tests/elasticsearch-sink/docker-compose.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index 1861a41896215..b30e7f75abc6f 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -7,16 +7,12 @@ services: - xpack.security.enabled=true - discovery.type=single-node - ELASTIC_PASSWORD=risingwave - ports: - - 9200:9200 elasticsearch8: - image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1 + image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 environment: - xpack.security.enabled=true - discovery.type=single-node - ELASTIC_PASSWORD=risingwave - ports: - - 9201:9200 compactor-0: extends: file: ../../docker/docker-compose.yml From 8e7aabce30bccdadc78f1e517e59ecfe498b32ce Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 14:33:06 +0800 Subject: [PATCH 13/17] update --- integration_tests/elasticsearch-sink/docker-compose.yml | 2 ++ integration_tests/scripts/run_demos.py | 1 + 2 files changed, 3 insertions(+) diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index b30e7f75abc6f..d389f12de274e 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -7,6 +7,8 @@ services: - xpack.security.enabled=true - discovery.type=single-node - ELASTIC_PASSWORD=risingwave + ports: + - 9200:9200 elasticsearch8: image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 environment: diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index 4f7bc2ae76b7f..511b9b0fea28c 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -242,6 +242,7 @@ def run_elasticsearch_sink_demo(): subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) sleep(120) + subprocess.run(["docker", "compose", "logs", "elasticsearch7"], cwd=demo_dir, check=True) subprocess.run(["docker", "compose", "logs", "elasticsearch8"], cwd=demo_dir, check=True) sql_files = ['create_source.sql', 'create_mv.sql', 'create_sink.sql'] From fd47fdaf376de6a9ca98658f76051731146b5067 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 14:38:07 +0800 Subject: [PATCH 14/17] update --- integration_tests/scripts/run_demos.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index 511b9b0fea28c..a68a8058c2ae1 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -240,7 +240,9 @@ def run_elasticsearch_sink_demo(): print("Running demo: {}".format(demo)) subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) - sleep(120) + subprocess.run(["docker", "compose", "logs", "elasticsearch7"], cwd=demo_dir, check=True) + subprocess.run(["docker", "compose", "logs", "elasticsearch8"], cwd=demo_dir, check=True) + sleep(180) subprocess.run(["docker", "compose", "logs", "elasticsearch7"], cwd=demo_dir, check=True) subprocess.run(["docker", "compose", "logs", "elasticsearch8"], cwd=demo_dir, check=True) From fcb4ab7f28a07861464d417dcdd4d9dbada7da8e Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 14:52:22 +0800 Subject: [PATCH 15/17] update --- ci/workflows/integration-tests.yml | 258 +++++++++--------- .../elasticsearch-sink/docker-compose.yml | 10 + integration_tests/scripts/run_demos.py | 4 +- 3 files changed, 140 insertions(+), 132 deletions(-) diff --git a/ci/workflows/integration-tests.yml b/ci/workflows/integration-tests.yml index 19c5af18dc863..f2016a03fe17a 100644 --- a/ci/workflows/integration-tests.yml +++ b/ci/workflows/integration-tests.yml @@ -5,89 +5,89 @@ auto-retry: &auto-retry limit: 3 steps: - - label: "Run Demos {{matrix.testcase}} {{matrix.format}}" - command: "ci/scripts/integration-tests.sh -c {{matrix.testcase}} -f {{matrix.format}}" - timeout_in_minutes: 30 - retry: *auto-retry - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - GHCR_USERNAME: ghcr-username - GHCR_TOKEN: ghcr-token - - ./ci/plugins/docker-compose-logs - matrix: - setup: - testcase: - - "ad-click" - - "ad-ctr" - - "cdn-metrics" - - "clickstream" - - "livestream" - - "prometheus" - - "schema-registry" - - "mysql-cdc" - - "postgres-cdc" - - "mysql-sink" - - "postgres-sink" - - "iceberg-cdc" + # - label: "Run Demos {{matrix.testcase}} {{matrix.format}}" + # command: "ci/scripts/integration-tests.sh -c {{matrix.testcase}} -f {{matrix.format}}" + # timeout_in_minutes: 30 + # retry: *auto-retry + # plugins: + # - seek-oss/aws-sm#v2.3.1: + # env: + # GHCR_USERNAME: ghcr-username + # GHCR_TOKEN: ghcr-token + # - ./ci/plugins/docker-compose-logs + # matrix: + # setup: + # testcase: + # - "ad-click" + # - "ad-ctr" + # - "cdn-metrics" + # - "clickstream" + # - "livestream" + # - "prometheus" + # - "schema-registry" + # - "mysql-cdc" + # - "postgres-cdc" + # - "mysql-sink" + # - "postgres-sink" + # - "iceberg-cdc" # - "iceberg-sink" - - "debezium-mysql" - format: - - "json" - - "protobuf" - adjustments: - - with: - testcase: "ad-click" - format: "protobuf" - skip: true - - with: - testcase: "ad-ctr" - format: "protobuf" - skip: true - - with: - testcase: "cdn-metrics" - format: "protobuf" - skip: true - - with: - testcase: "clickstream" - format: "protobuf" - skip: true - - with: - testcase: "prometheus" - format: "protobuf" - skip: true - - with: - testcase: "schema-registry" - format: "protobuf" - skip: true - - with: - testcase: "mysql-cdc" - format: "protobuf" - skip: true - - with: - testcase: "postgres-cdc" - format: "protobuf" - skip: true - - with: - testcase: "mysql-sink" - format: "protobuf" - skip: true - - with: - testcase: "postgres-sink" - format: "protobuf" - skip: true + # - "debezium-mysql" + # format: + # - "json" + # - "protobuf" + # adjustments: + # - with: + # testcase: "ad-click" + # format: "protobuf" + # skip: true + # - with: + # testcase: "ad-ctr" + # format: "protobuf" + # skip: true + # - with: + # testcase: "cdn-metrics" + # format: "protobuf" + # skip: true + # - with: + # testcase: "clickstream" + # format: "protobuf" + # skip: true + # - with: + # testcase: "prometheus" + # format: "protobuf" + # skip: true + # - with: + # testcase: "schema-registry" + # format: "protobuf" + # skip: true + # - with: + # testcase: "mysql-cdc" + # format: "protobuf" + # skip: true + # - with: + # testcase: "postgres-cdc" + # format: "protobuf" + # skip: true + # - with: + # testcase: "mysql-sink" + # format: "protobuf" + # skip: true + # - with: + # testcase: "postgres-sink" + # format: "protobuf" + # skip: true # - with: # testcase: "iceberg-sink" # format: "protobuf" # skip: true - - with: - testcase: "iceberg-cdc" - format: "protobuf" - skip: true - - with: - testcase: "debezium-mysql" - format: "protobuf" - skip: true + # - with: + # testcase: "iceberg-cdc" + # format: "protobuf" + # skip: true + # - with: + # testcase: "debezium-mysql" + # format: "protobuf" + # skip: true # NOTE: buildkite matrix-limits # Each build matrix has a limit of 6 dimensions, 20 elements in each dimension and a total of 12 adjustments. @@ -104,67 +104,67 @@ steps: matrix: setup: testcase: - - "twitter" - - "twitter-pulsar" + # - "twitter" + # - "twitter-pulsar" # - "debezium-mongo" - - "debezium-postgres" - - "tidb-cdc-sink" - - "debezium-sqlserver" - - "citus-cdc" - - "kinesis-s3-source" - - "clickhouse-sink" - - "cockroach-sink" - - "kafka-cdc-sink" - - "cassandra-and-scylladb-sink" + # - "debezium-postgres" + # - "tidb-cdc-sink" + # - "debezium-sqlserver" + # - "citus-cdc" + # - "kinesis-s3-source" + # - "clickhouse-sink" + # - "cockroach-sink" + # - "kafka-cdc-sink" + # - "cassandra-and-scylladb-sink" - "elasticsearch-sink" format: - "json" - "protobuf" adjustments: - - with: - testcase: "twitter-pulsar" - format: "protobuf" - skip: true + # - with: + # testcase: "twitter-pulsar" + # format: "protobuf" + # skip: true # - with: # testcase: "debezium-mongo" # format: "protobuf" # skip: true - - with: - testcase: "debezium-postgres" - format: "protobuf" - skip: true - - with: - testcase: "tidb-cdc-sink" - format: "protobuf" - skip: true - - with: - testcase: "debezium-sqlserver" - format: "protobuf" - skip: true - - with: - testcase: "citus-cdc" - format: "protobuf" - skip: true - - with: - testcase: "kinesis-s3-source" - format: "protobuf" - skip: true - - with: - testcase: "clickhouse-sink" - format: "protobuf" - skip: true - - with: - testcase: "cockroach-sink" - format: "protobuf" - skip: true - - with: - testcase: "kafka-cdc-sink" - format: "protobuf" - skip: true - - with: - testcase: "cassandra-and-scylladb-sink" - format: "protobuf" - skip: true + # - with: + # testcase: "debezium-postgres" + # format: "protobuf" + # skip: true + # - with: + # testcase: "tidb-cdc-sink" + # format: "protobuf" + # skip: true + # - with: + # testcase: "debezium-sqlserver" + # format: "protobuf" + # skip: true + # - with: + # testcase: "citus-cdc" + # format: "protobuf" + # skip: true + # - with: + # testcase: "kinesis-s3-source" + # format: "protobuf" + # skip: true + # - with: + # testcase: "clickhouse-sink" + # format: "protobuf" + # skip: true + # - with: + # testcase: "cockroach-sink" + # format: "protobuf" + # skip: true + # - with: + # testcase: "kafka-cdc-sink" + # format: "protobuf" + # skip: true + # - with: + # testcase: "cassandra-and-scylladb-sink" + # format: "protobuf" + # skip: true - with: testcase: "elasticsearch-sink" format: "protobuf" diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml index d389f12de274e..593d19850fa14 100644 --- a/integration_tests/elasticsearch-sink/docker-compose.yml +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -7,6 +7,10 @@ services: - xpack.security.enabled=true - discovery.type=single-node - ELASTIC_PASSWORD=risingwave + deploy: + resources: + limits: + memory: 1G ports: - 9200:9200 elasticsearch8: @@ -15,6 +19,12 @@ services: - xpack.security.enabled=true - discovery.type=single-node - ELASTIC_PASSWORD=risingwave + deploy: + resources: + limits: + memory: 1G + ports: + - 9201:9200 compactor-0: extends: file: ../../docker/docker-compose.yml diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index a68a8058c2ae1..75878dcb2b2f0 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -240,9 +240,7 @@ def run_elasticsearch_sink_demo(): print("Running demo: {}".format(demo)) subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) - subprocess.run(["docker", "compose", "logs", "elasticsearch7"], cwd=demo_dir, check=True) - subprocess.run(["docker", "compose", "logs", "elasticsearch8"], cwd=demo_dir, check=True) - sleep(180) + sleep(60) subprocess.run(["docker", "compose", "logs", "elasticsearch7"], cwd=demo_dir, check=True) subprocess.run(["docker", "compose", "logs", "elasticsearch8"], cwd=demo_dir, check=True) From 371008cc232d06b390bb8957cfa374be87adf889 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 15:03:55 +0800 Subject: [PATCH 16/17] update --- ci/workflows/integration-tests.yml | 258 ++++++++++++++--------------- 1 file changed, 129 insertions(+), 129 deletions(-) diff --git a/ci/workflows/integration-tests.yml b/ci/workflows/integration-tests.yml index f2016a03fe17a..19c5af18dc863 100644 --- a/ci/workflows/integration-tests.yml +++ b/ci/workflows/integration-tests.yml @@ -5,89 +5,89 @@ auto-retry: &auto-retry limit: 3 steps: - # - label: "Run Demos {{matrix.testcase}} {{matrix.format}}" - # command: "ci/scripts/integration-tests.sh -c {{matrix.testcase}} -f {{matrix.format}}" - # timeout_in_minutes: 30 - # retry: *auto-retry - # plugins: - # - seek-oss/aws-sm#v2.3.1: - # env: - # GHCR_USERNAME: ghcr-username - # GHCR_TOKEN: ghcr-token - # - ./ci/plugins/docker-compose-logs - # matrix: - # setup: - # testcase: - # - "ad-click" - # - "ad-ctr" - # - "cdn-metrics" - # - "clickstream" - # - "livestream" - # - "prometheus" - # - "schema-registry" - # - "mysql-cdc" - # - "postgres-cdc" - # - "mysql-sink" - # - "postgres-sink" - # - "iceberg-cdc" + - label: "Run Demos {{matrix.testcase}} {{matrix.format}}" + command: "ci/scripts/integration-tests.sh -c {{matrix.testcase}} -f {{matrix.format}}" + timeout_in_minutes: 30 + retry: *auto-retry + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + GHCR_USERNAME: ghcr-username + GHCR_TOKEN: ghcr-token + - ./ci/plugins/docker-compose-logs + matrix: + setup: + testcase: + - "ad-click" + - "ad-ctr" + - "cdn-metrics" + - "clickstream" + - "livestream" + - "prometheus" + - "schema-registry" + - "mysql-cdc" + - "postgres-cdc" + - "mysql-sink" + - "postgres-sink" + - "iceberg-cdc" # - "iceberg-sink" - # - "debezium-mysql" - # format: - # - "json" - # - "protobuf" - # adjustments: - # - with: - # testcase: "ad-click" - # format: "protobuf" - # skip: true - # - with: - # testcase: "ad-ctr" - # format: "protobuf" - # skip: true - # - with: - # testcase: "cdn-metrics" - # format: "protobuf" - # skip: true - # - with: - # testcase: "clickstream" - # format: "protobuf" - # skip: true - # - with: - # testcase: "prometheus" - # format: "protobuf" - # skip: true - # - with: - # testcase: "schema-registry" - # format: "protobuf" - # skip: true - # - with: - # testcase: "mysql-cdc" - # format: "protobuf" - # skip: true - # - with: - # testcase: "postgres-cdc" - # format: "protobuf" - # skip: true - # - with: - # testcase: "mysql-sink" - # format: "protobuf" - # skip: true - # - with: - # testcase: "postgres-sink" - # format: "protobuf" - # skip: true + - "debezium-mysql" + format: + - "json" + - "protobuf" + adjustments: + - with: + testcase: "ad-click" + format: "protobuf" + skip: true + - with: + testcase: "ad-ctr" + format: "protobuf" + skip: true + - with: + testcase: "cdn-metrics" + format: "protobuf" + skip: true + - with: + testcase: "clickstream" + format: "protobuf" + skip: true + - with: + testcase: "prometheus" + format: "protobuf" + skip: true + - with: + testcase: "schema-registry" + format: "protobuf" + skip: true + - with: + testcase: "mysql-cdc" + format: "protobuf" + skip: true + - with: + testcase: "postgres-cdc" + format: "protobuf" + skip: true + - with: + testcase: "mysql-sink" + format: "protobuf" + skip: true + - with: + testcase: "postgres-sink" + format: "protobuf" + skip: true # - with: # testcase: "iceberg-sink" # format: "protobuf" # skip: true - # - with: - # testcase: "iceberg-cdc" - # format: "protobuf" - # skip: true - # - with: - # testcase: "debezium-mysql" - # format: "protobuf" - # skip: true + - with: + testcase: "iceberg-cdc" + format: "protobuf" + skip: true + - with: + testcase: "debezium-mysql" + format: "protobuf" + skip: true # NOTE: buildkite matrix-limits # Each build matrix has a limit of 6 dimensions, 20 elements in each dimension and a total of 12 adjustments. @@ -104,67 +104,67 @@ steps: matrix: setup: testcase: - # - "twitter" - # - "twitter-pulsar" + - "twitter" + - "twitter-pulsar" # - "debezium-mongo" - # - "debezium-postgres" - # - "tidb-cdc-sink" - # - "debezium-sqlserver" - # - "citus-cdc" - # - "kinesis-s3-source" - # - "clickhouse-sink" - # - "cockroach-sink" - # - "kafka-cdc-sink" - # - "cassandra-and-scylladb-sink" + - "debezium-postgres" + - "tidb-cdc-sink" + - "debezium-sqlserver" + - "citus-cdc" + - "kinesis-s3-source" + - "clickhouse-sink" + - "cockroach-sink" + - "kafka-cdc-sink" + - "cassandra-and-scylladb-sink" - "elasticsearch-sink" format: - "json" - "protobuf" adjustments: - # - with: - # testcase: "twitter-pulsar" - # format: "protobuf" - # skip: true + - with: + testcase: "twitter-pulsar" + format: "protobuf" + skip: true # - with: # testcase: "debezium-mongo" # format: "protobuf" # skip: true - # - with: - # testcase: "debezium-postgres" - # format: "protobuf" - # skip: true - # - with: - # testcase: "tidb-cdc-sink" - # format: "protobuf" - # skip: true - # - with: - # testcase: "debezium-sqlserver" - # format: "protobuf" - # skip: true - # - with: - # testcase: "citus-cdc" - # format: "protobuf" - # skip: true - # - with: - # testcase: "kinesis-s3-source" - # format: "protobuf" - # skip: true - # - with: - # testcase: "clickhouse-sink" - # format: "protobuf" - # skip: true - # - with: - # testcase: "cockroach-sink" - # format: "protobuf" - # skip: true - # - with: - # testcase: "kafka-cdc-sink" - # format: "protobuf" - # skip: true - # - with: - # testcase: "cassandra-and-scylladb-sink" - # format: "protobuf" - # skip: true + - with: + testcase: "debezium-postgres" + format: "protobuf" + skip: true + - with: + testcase: "tidb-cdc-sink" + format: "protobuf" + skip: true + - with: + testcase: "debezium-sqlserver" + format: "protobuf" + skip: true + - with: + testcase: "citus-cdc" + format: "protobuf" + skip: true + - with: + testcase: "kinesis-s3-source" + format: "protobuf" + skip: true + - with: + testcase: "clickhouse-sink" + format: "protobuf" + skip: true + - with: + testcase: "cockroach-sink" + format: "protobuf" + skip: true + - with: + testcase: "kafka-cdc-sink" + format: "protobuf" + skip: true + - with: + testcase: "cassandra-and-scylladb-sink" + format: "protobuf" + skip: true - with: testcase: "elasticsearch-sink" format: "protobuf" From 9a59e2d8f7fa6c9952f7aab40a51a001d3d0ad18 Mon Sep 17 00:00:00 2001 From: xfz Date: Thu, 23 Nov 2023 15:11:06 +0800 Subject: [PATCH 17/17] remove --- integration_tests/scripts/run_demos.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index 75878dcb2b2f0..b2726e7fb5174 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -242,9 +242,6 @@ def run_elasticsearch_sink_demo(): subprocess.run(["docker", "compose", "up", "-d", "--build"], cwd=demo_dir, check=True) sleep(60) - subprocess.run(["docker", "compose", "logs", "elasticsearch7"], cwd=demo_dir, check=True) - subprocess.run(["docker", "compose", "logs", "elasticsearch8"], cwd=demo_dir, check=True) - sql_files = ['create_source.sql', 'create_mv.sql', 'create_sink.sql'] for fname in sql_files: sql_file = os.path.join(demo_dir, fname)