-
Notifications
You must be signed in to change notification settings - Fork 590
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(test): Add starrocks redis doris cassandra e2e test (#14142)
- Loading branch information
Showing
17 changed files
with
660 additions
and
39 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Exits as soon as any line fails. | ||
| set -euo pipefail | ||
|
|
||
| source ci/scripts/common.sh | ||
|
|
||
| # prepare environment | ||
| export CONNECTOR_LIBS_PATH="./connector-node/libs" | ||
|
|
||
| while getopts 'p:' opt; do | ||
| case ${opt} in | ||
| p ) | ||
| profile=$OPTARG | ||
| ;; | ||
| \? ) | ||
| echo "Invalid Option: -$OPTARG" 1>&2 | ||
| exit 1 | ||
| ;; | ||
| : ) | ||
| echo "Invalid option: $OPTARG requires an argument" 1>&2 | ||
| ;; | ||
| esac | ||
| done | ||
| shift $((OPTIND -1)) | ||
|
|
||
| download_and_prepare_rw "$profile" source | ||
|
|
||
| echo "--- Download connector node package" | ||
| buildkite-agent artifact download risingwave-connector.tar.gz ./ | ||
| mkdir ./connector-node | ||
| tar xf ./risingwave-connector.tar.gz -C ./connector-node | ||
|
|
||
| echo "--- starting risingwave cluster" | ||
| cargo make ci-start ci-sink-test | ||
| sleep 1 | ||
|
|
||
| echo "--- create cassandra table" | ||
| curl https://downloads.apache.org/cassandra/4.1.3/apache-cassandra-4.1.3-bin.tar.gz --output apache-cassandra-4.1.3-bin.tar.gz | ||
| tar xfvz apache-cassandra-4.1.3-bin.tar.gz | ||
| cd apache-cassandra-4.1.3/bin | ||
| export CQLSH_HOST=cassandra-server | ||
| export CQLSH_PORT=9042 | ||
| ./cqlsh -e "CREATE KEYSPACE demo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};use demo; | ||
| CREATE table demo_bhv_table(v1 int primary key,v2 smallint,v3 bigint,v4 float,v5 double,v6 text,v7 date,v8 timestamp,v9 boolean);" | ||
|
|
||
| echo "--- testing sinks" | ||
| cd ../../ | ||
| sqllogictest -p 4566 -d dev './e2e_test/sink/cassandra_sink.slt' | ||
| sleep 1 | ||
| cd apache-cassandra-4.1.3/bin | ||
| ./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false AND ENCODING = 'UTF-8';" | ||
|
|
||
| if cat ./query_result.csv | awk -F "," '{ | ||
| exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01.000+0000" && $9 == "False\r"); }'; then | ||
| echo "Cassandra sink check passed" | ||
| else | ||
| cat ./query_result.csv | ||
| echo "The output is not as expected." | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "--- Kill cluster" | ||
| cd ../../ | ||
| cargo make ci-kill |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Exits as soon as any line fails. | ||
| set -euo pipefail | ||
|
|
||
| source ci/scripts/common.sh | ||
|
|
||
| while getopts 'p:' opt; do | ||
| case ${opt} in | ||
| p ) | ||
| profile=$OPTARG | ||
| ;; | ||
| \? ) | ||
| echo "Invalid Option: -$OPTARG" 1>&2 | ||
| exit 1 | ||
| ;; | ||
| : ) | ||
| echo "Invalid option: $OPTARG requires an argument" 1>&2 | ||
| ;; | ||
| esac | ||
| done | ||
| shift $((OPTIND -1)) | ||
|
|
||
| download_and_prepare_rw "$profile" source | ||
|
|
||
| echo "--- starting risingwave cluster" | ||
| cargo make ci-start ci-sink-test | ||
| sleep 1 | ||
|
|
||
| echo "--- create doris table" | ||
| apt-get update -y && apt-get install -y mysql-client | ||
| sleep 2 | ||
| mysql -uroot -P 9030 -h doris-fe-server -e "CREATE database demo;use demo; | ||
| CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean) UNIQUE KEY(\`v1\`) | ||
| DISTRIBUTED BY HASH(\`v1\`) BUCKETS 1 | ||
| PROPERTIES ( | ||
| \"replication_allocation\" = \"tag.location.default: 1\" | ||
| ); | ||
| CREATE USER 'users'@'%' IDENTIFIED BY '123456'; | ||
| GRANT ALL ON *.* TO 'users'@'%';" | ||
| sleep 2 | ||
|
|
||
| echo "--- testing sinks" | ||
| sqllogictest -p 4566 -d dev './e2e_test/sink/doris_sink.slt' | ||
| sleep 1 | ||
| mysql -uroot -P 9030 -h doris-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv | ||
|
|
||
|
|
||
| if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ | ||
| exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0); }'; then | ||
| echo "Doris sink check passed" | ||
| else | ||
| cat ./query_result.csv | ||
| echo "The output is not as expected." | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "--- Kill cluster" | ||
| cargo make ci-kill |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Exits as soon as any line fails. | ||
| set -euo pipefail | ||
|
|
||
| source ci/scripts/common.sh | ||
|
|
||
| while getopts 'p:' opt; do | ||
| case ${opt} in | ||
| p ) | ||
| profile=$OPTARG | ||
| ;; | ||
| \? ) | ||
| echo "Invalid Option: -$OPTARG" 1>&2 | ||
| exit 1 | ||
| ;; | ||
| : ) | ||
| echo "Invalid option: $OPTARG requires an argument" 1>&2 | ||
| ;; | ||
| esac | ||
| done | ||
| shift $((OPTIND -1)) | ||
|
|
||
| download_and_prepare_rw "$profile" source | ||
|
|
||
| echo "--- starting risingwave cluster" | ||
| cargo make ci-start ci-sink-test | ||
| apt-get update -y && apt-get install -y redis-server | ||
| sleep 1 | ||
|
|
||
| echo "--- testing sinks" | ||
| sqllogictest -p 4566 -d dev './e2e_test/sink/redis_sink.slt' | ||
| sleep 1 | ||
|
|
||
| redis-cli -h redis-server -p 6379 get {\"v1\":1} >> ./query_result.txt | ||
| redis-cli -h redis-server -p 6379 get V1:1 >> ./query_result.txt | ||
|
|
||
| # check sink destination using shell | ||
| if cat ./query_result.txt | tr '\n' '\0' | xargs -0 -n1 bash -c '[[ "$0" == "{\"v1\":1,\"v2\":1,\"v3\":1,\"v4\":1.100000023841858,\"v5\":1.2,\"v6\":\"test\",\"v7\":734869,\"v8\":\"2013-01-01T01:01:01.000000Z\",\"v9\":false}" || "$0" == "V2:1,V3:1" ]]'; then | ||
| echo "Redis sink check passed" | ||
| else | ||
| cat ./query_result.txt | ||
| echo "The output is not as expected." | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "--- Kill cluster" | ||
| cargo make ci-kill |
Oops, something went wrong.