-
Notifications
You must be signed in to change notification settings - Fork 594
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add doris redis strarrocks cassdran sink e2e
- Loading branch information
Showing
13 changed files
with
521 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
#!/usr/bin/env bash | ||
|
||
# Exits as soon as any line fails. | ||
set -euo pipefail | ||
|
||
source ci/scripts/common.sh | ||
|
||
# prepare environment | ||
export CONNECTOR_LIBS_PATH="./connector-node/libs" | ||
|
||
while getopts 'p:' opt; do | ||
case ${opt} in | ||
p ) | ||
profile=$OPTARG | ||
;; | ||
\? ) | ||
echo "Invalid Option: -$OPTARG" 1>&2 | ||
exit 1 | ||
;; | ||
: ) | ||
echo "Invalid option: $OPTARG requires an argument" 1>&2 | ||
;; | ||
esac | ||
done | ||
shift $((OPTIND -1)) | ||
|
||
download_and_prepare_rw "$profile" source | ||
|
||
echo "--- Download connector node package" | ||
buildkite-agent artifact download risingwave-connector.tar.gz ./ | ||
mkdir ./connector-node | ||
tar xf ./risingwave-connector.tar.gz -C ./connector-node | ||
|
||
echo "--- starting risingwave cluster" | ||
cargo make ci-start ci-sink-test | ||
sleep 1 | ||
|
||
echo "--- create cassandra table" | ||
curl https://downloads.apache.org/cassandra/4.1.3/apache-cassandra-4.1.3-bin.tar.gz --output apache-cassandra-4.1.3-bin.tar.gz | ||
tar xfvz apache-cassandra-4.1.3-bin.tar.gz | ||
./cqlsh -e "CREATE KEYSPACE demo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};use demo; | ||
CREATE table demo_bhv_table(v1 int primary key,v2 smallint,v3 bigint,v4 float,v5 double,v6 text,v7 date,v8 timestamp,v9 boolean);" | ||
|
||
echo "--- testing sinks" | ||
sqllogictest -p 4566 -d dev './e2e_test/sink/cassandra_sink.slt' | ||
sleep 1 | ||
./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false;" | ||
|
||
AND ENCODING = 'UTF-8'; | ||
if cat ./query_result.csv | awk -F "," '{ | ||
exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01.000+0000" && $9 == "False\r"); }'; then | ||
echo "Cassandra sink check passed" | ||
else | ||
cat ./query_result.csv | ||
echo "The output is not as expected." | ||
fi | ||
|
||
echo "--- Kill cluster" | ||
cargo make ci-kill |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
#!/usr/bin/env bash | ||
|
||
# Exits as soon as any line fails. | ||
set -euo pipefail | ||
|
||
source ci/scripts/common.sh | ||
|
||
while getopts 'p:' opt; do | ||
case ${opt} in | ||
p ) | ||
profile=$OPTARG | ||
;; | ||
\? ) | ||
echo "Invalid Option: -$OPTARG" 1>&2 | ||
exit 1 | ||
;; | ||
: ) | ||
echo "Invalid option: $OPTARG requires an argument" 1>&2 | ||
;; | ||
esac | ||
done | ||
shift $((OPTIND -1)) | ||
|
||
download_and_prepare_rw "$profile" source | ||
|
||
echo "--- starting risingwave cluster" | ||
cargo make ci-start ci-sink-test | ||
sleep 1 | ||
|
||
|
||
echo "--- create doris table" | ||
apt-get install -y mysql-client | ||
sleep 2 | ||
mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo; | ||
CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean) UNIQUE KEY(\`v1\`) | ||
DISTRIBUTED BY HASH(\`v1\`) BUCKETS 1 | ||
PROPERTIES ( | ||
\"replication_allocation\" = \"tag.location.default: 1\" | ||
); | ||
CREATE USER 'users'@'%' IDENTIFIED BY '123456'; | ||
GRANT ALL ON *.* TO 'users'@'%';" | ||
sleep 2 | ||
|
||
echo "--- testing sinks" | ||
sqllogictest -p 4566 -d dev './e2e_test/sink/doris_sink.slt' | ||
sleep 1 | ||
mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv | ||
|
||
|
||
if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ ─╯ | ||
exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0); }'; then | ||
echo "Doris sink check passed" | ||
else | ||
cat ./query_result.csv | ||
echo "The output is not as expected." | ||
fi | ||
|
||
echo "--- Kill cluster" | ||
cargo make ci-kill |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
#!/usr/bin/env bash | ||
|
||
# Exits as soon as any line fails. | ||
set -euo pipefail | ||
|
||
source ci/scripts/common.sh | ||
|
||
while getopts 'p:' opt; do | ||
case ${opt} in | ||
p ) | ||
profile=$OPTARG | ||
;; | ||
\? ) | ||
echo "Invalid Option: -$OPTARG" 1>&2 | ||
exit 1 | ||
;; | ||
: ) | ||
echo "Invalid option: $OPTARG requires an argument" 1>&2 | ||
;; | ||
esac | ||
done | ||
shift $((OPTIND -1)) | ||
|
||
download_and_prepare_rw "$profile" source | ||
|
||
echo "--- starting risingwave cluster" | ||
mkdir -p .risingwave/log | ||
cargo make ci-start ci-sink-test | ||
sleep 1 | ||
|
||
echo "--- testing sinks" | ||
sqllogictest -p 4566 -d dev './e2e_test/sink/redis_sink.slt' | ||
sleep 1 | ||
|
||
redis-cli -p 6378 get {\"v1\":1} >> ./query_result.txt | ||
redis-cli -p 6378 get V1:1 >> ./query_result.txt | ||
|
||
# check sink destination using shell | ||
if cat ./query_result.txt | tr '\n' '\0' | xargs -0 -n1 bash -c '[[ "$0" == "{\"v1\":1,\"v2\":1,\"v3\":1,\"v4\":1.100000023841858,\"v5\":1.2,\"v6\":\"test\",\"v7\":734869,\"v8\":\"2013-01-01T01:01:01.000000Z\",\"v9\":false}" || "$0" == "V2:1,V3:1" ]]'; then | ||
echo "Redis sink check passed" | ||
else | ||
cat ./query_result.txt | ||
echo "The output is not as expected." | ||
exit 1 | ||
fi | ||
|
||
echo "--- Kill cluster" | ||
cargo make ci-kill |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
#!/usr/bin/env bash | ||
|
||
# Exits as soon as any line fails. | ||
set -euo pipefail | ||
|
||
source ci/scripts/common.sh | ||
|
||
while getopts 'p:' opt; do | ||
case ${opt} in | ||
p ) | ||
profile=$OPTARG | ||
;; | ||
\? ) | ||
echo "Invalid Option: -$OPTARG" 1>&2 | ||
exit 1 | ||
;; | ||
: ) | ||
echo "Invalid option: $OPTARG requires an argument" 1>&2 | ||
;; | ||
esac | ||
done | ||
shift $((OPTIND -1)) | ||
|
||
download_and_prepare_rw "$profile" source | ||
|
||
echo "--- starting risingwave cluster" | ||
cargo make ci-start ci-sink-test | ||
sleep 1 | ||
|
||
|
||
echo "--- create starrocks table" | ||
apt-get install -y mysql-client | ||
sleep 2 | ||
mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo; | ||
CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 date,v8 datetime,v9 boolean) ENGINE=OLAP | ||
PRIMARY KEY(\`v1\`) | ||
DISTRIBUTED BY HASH(\`v1\`) properties(\"replication_num\" = \"1\"); | ||
CREATE USER 'users'@'%' IDENTIFIED BY '123456'; | ||
GRANT ALL ON *.* TO 'users'@'%';" | ||
sleep 2 | ||
|
||
echo "--- testing sinks" | ||
sqllogictest -p 4566 -d dev './e2e_test/sink/starrocks_sink.slt' | ||
sleep 1 | ||
mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv | ||
|
||
|
||
if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ | ||
exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0); }'; then | ||
echo "Starrocks sink check passed" | ||
else | ||
cat ./query_result.csv | ||
echo "The output is not as expected." | ||
fi | ||
|
||
echo "--- Kill cluster" | ||
cargo make ci-kill |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.