Skip to content

Commit

Permalink
add doris redis strarrocks cassdran sink e2e
Browse files Browse the repository at this point in the history
  • Loading branch information
xxhZs committed Dec 22, 2023
1 parent e3810af commit f44aaf4
Show file tree
Hide file tree
Showing 13 changed files with 521 additions and 31 deletions.
21 changes: 21 additions & 0 deletions ci/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,27 @@ services:
interval: 5s
timeout: 30s
retries: 50
doris-fe:
image: apache/doris:2.0.0_alpha-fe-x86_64
hostname: fe
environment:
- FE_SERVERS=fe1:127.0.0.1:9010
- FE_ID=1
network_mode: host
ports:
- "8030:8030"
- "9030:9030"
doris-be:
image: apache/doris:2.0.0_alpha-be-x86_64
hostname: be
environment:
- FE_SERVERS=fe1:127.0.0.1:9010
- BE_ADDR=127.0.0.1:9050
depends_on:
- fe
ports:
- "9050:9050"
network_mode: host

# Temporary workaround for json schema registry test since redpanda only supports
# protobuf/avro schema registry. Should be removed after the support.
Expand Down
59 changes: 59 additions & 0 deletions ci/scripts/e2e-cassandra-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash

# Exits as soon as any line fails.
set -euo pipefail

source ci/scripts/common.sh

# prepare environment
export CONNECTOR_LIBS_PATH="./connector-node/libs"

while getopts 'p:' opt; do
case ${opt} in
p )
profile=$OPTARG
;;
\? )
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
: )
echo "Invalid option: $OPTARG requires an argument" 1>&2
;;
esac
done
shift $((OPTIND -1))

download_and_prepare_rw "$profile" source

echo "--- Download connector node package"
buildkite-agent artifact download risingwave-connector.tar.gz ./
mkdir ./connector-node
tar xf ./risingwave-connector.tar.gz -C ./connector-node

echo "--- starting risingwave cluster"
cargo make ci-start ci-sink-test
sleep 1

echo "--- create cassandra table"
curl https://downloads.apache.org/cassandra/4.1.3/apache-cassandra-4.1.3-bin.tar.gz --output apache-cassandra-4.1.3-bin.tar.gz
tar xfvz apache-cassandra-4.1.3-bin.tar.gz
./cqlsh -e "CREATE KEYSPACE demo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};use demo;
CREATE table demo_bhv_table(v1 int primary key,v2 smallint,v3 bigint,v4 float,v5 double,v6 text,v7 date,v8 timestamp,v9 boolean);"

echo "--- testing sinks"
sqllogictest -p 4566 -d dev './e2e_test/sink/cassandra_sink.slt'
sleep 1
./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false;"

AND ENCODING = 'UTF-8';
if cat ./query_result.csv | awk -F "," '{
exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01.000+0000" && $9 == "False\r"); }'; then
echo "Cassandra sink check passed"
else
cat ./query_result.csv
echo "The output is not as expected."
fi

echo "--- Kill cluster"
cargo make ci-kill
59 changes: 59 additions & 0 deletions ci/scripts/e2e-doris-sink-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash

# Exits as soon as any line fails.
set -euo pipefail

source ci/scripts/common.sh

while getopts 'p:' opt; do
case ${opt} in
p )
profile=$OPTARG
;;
\? )
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
: )
echo "Invalid option: $OPTARG requires an argument" 1>&2
;;
esac
done
shift $((OPTIND -1))

download_and_prepare_rw "$profile" source

echo "--- starting risingwave cluster"
cargo make ci-start ci-sink-test
sleep 1


echo "--- create doris table"
apt-get install -y mysql-client
sleep 2
mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo;
CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean) UNIQUE KEY(\`v1\`)
DISTRIBUTED BY HASH(\`v1\`) BUCKETS 1
PROPERTIES (
\"replication_allocation\" = \"tag.location.default: 1\"
);
CREATE USER 'users'@'%' IDENTIFIED BY '123456';
GRANT ALL ON *.* TO 'users'@'%';"
sleep 2

echo "--- testing sinks"
sqllogictest -p 4566 -d dev './e2e_test/sink/doris_sink.slt'
sleep 1
mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv


if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ ─╯
exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0); }'; then
echo "Doris sink check passed"
else
cat ./query_result.csv
echo "The output is not as expected."
fi

echo "--- Kill cluster"
cargo make ci-kill
48 changes: 48 additions & 0 deletions ci/scripts/e2e-redis-sink-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/usr/bin/env bash

# Exits as soon as any line fails.
set -euo pipefail

source ci/scripts/common.sh

while getopts 'p:' opt; do
case ${opt} in
p )
profile=$OPTARG
;;
\? )
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
: )
echo "Invalid option: $OPTARG requires an argument" 1>&2
;;
esac
done
shift $((OPTIND -1))

download_and_prepare_rw "$profile" source

echo "--- starting risingwave cluster"
mkdir -p .risingwave/log
cargo make ci-start ci-sink-test
sleep 1

echo "--- testing sinks"
sqllogictest -p 4566 -d dev './e2e_test/sink/redis_sink.slt'
sleep 1

redis-cli -p 6378 get {\"v1\":1} >> ./query_result.txt
redis-cli -p 6378 get V1:1 >> ./query_result.txt

# check sink destination using shell
if cat ./query_result.txt | tr '\n' '\0' | xargs -0 -n1 bash -c '[[ "$0" == "{\"v1\":1,\"v2\":1,\"v3\":1,\"v4\":1.100000023841858,\"v5\":1.2,\"v6\":\"test\",\"v7\":734869,\"v8\":\"2013-01-01T01:01:01.000000Z\",\"v9\":false}" || "$0" == "V2:1,V3:1" ]]'; then
echo "Redis sink check passed"
else
cat ./query_result.txt
echo "The output is not as expected."
exit 1
fi

echo "--- Kill cluster"
cargo make ci-kill
57 changes: 57 additions & 0 deletions ci/scripts/e2e-starrocks-sink-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/usr/bin/env bash

# Exits as soon as any line fails.
set -euo pipefail

source ci/scripts/common.sh

while getopts 'p:' opt; do
case ${opt} in
p )
profile=$OPTARG
;;
\? )
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
: )
echo "Invalid option: $OPTARG requires an argument" 1>&2
;;
esac
done
shift $((OPTIND -1))

download_and_prepare_rw "$profile" source

echo "--- starting risingwave cluster"
cargo make ci-start ci-sink-test
sleep 1


echo "--- create starrocks table"
apt-get install -y mysql-client
sleep 2
mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo;
CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 date,v8 datetime,v9 boolean) ENGINE=OLAP
PRIMARY KEY(\`v1\`)
DISTRIBUTED BY HASH(\`v1\`) properties(\"replication_num\" = \"1\");
CREATE USER 'users'@'%' IDENTIFIED BY '123456';
GRANT ALL ON *.* TO 'users'@'%';"
sleep 2

echo "--- testing sinks"
sqllogictest -p 4566 -d dev './e2e_test/sink/starrocks_sink.slt'
sleep 1
mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv


if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{
exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0); }'; then
echo "Starrocks sink check passed"
else
cat ./query_result.csv
echo "The output is not as expected."
fi

echo "--- Kill cluster"
cargo make ci-kill
57 changes: 57 additions & 0 deletions ci/workflows/main-cron.yml
Original file line number Diff line number Diff line change
Expand Up @@ -718,6 +718,63 @@ steps:
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end doris sink test"
key: "e2e-doris-sink-tests"
command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
if: |
!(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
|| build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
|| build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
depends_on:
- "build"
- "build-other"
plugins:
- docker-compose#v4.9.0:
run: sink-test-env
config: ci/docker-compose.yml
mount-buildkite-agent: true
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end starrocks sink test"
key: "e2e-starrocks-sink-tests"
command: "ci/scripts/e2e-starrocks-sink-test.sh -p ci-release"
if: |
!(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
|| build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests"
|| build.env("CI_STEPS") =~ /(^|,)e2e-starrocks-sink-tests?(,|$$)/
depends_on:
- "build"
- "build-other"
plugins:
- docker-compose#v4.9.0:
run: sink-test-env
config: ci/docker-compose.yml
mount-buildkite-agent: true
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end cassandra sink test"
key: "e2e-cassandra-sink-tests"
command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-release"
if: |
!(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
|| build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests"
|| build.env("CI_STEPS") =~ /(^|,)e2e-cassandra-sink-tests?(,|$$)/
depends_on:
- "build"
- "build-other"
plugins:
- docker-compose#v4.9.0:
run: sink-test-env
config: ci/docker-compose.yml
mount-buildkite-agent: true
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end clickhouse sink test"
key: "e2e-clickhouse-sink-tests"
command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-release"
Expand Down
47 changes: 46 additions & 1 deletion ci/workflows/pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ steps:
retry: *auto-retry

- label: "end-to-end redis sink test"
if: build.pull_request.labels includes "ci/run- e2e-redis-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-redis-sink-tests?(,|$$)/
if: build.pull_request.labels includes "ci/run-e2e-redis-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-redis-sink-tests?(,|$$)/
command: "ci/scripts/e2e-redis-sink-test.sh -p ci-dev"
depends_on:
- "build"
Expand All @@ -307,6 +307,51 @@ steps:
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end doris sink test"
if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
depends_on:
- "build"
- "build-other"
plugins:
- docker-compose#v4.9.0:
run: sink-test-env
config: ci/docker-compose.yml
mount-buildkite-agent: true
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end starrocks sink test"
if: build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-starrocks-sink-tests?(,|$$)/
command: "ci/scripts/e2e-starrocks-sink-test.sh -p ci-dev"
depends_on:
- "build"
- "build-other"
plugins:
- docker-compose#v4.9.0:
run: sink-test-env
config: ci/docker-compose.yml
mount-buildkite-agent: true
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end cassandra sink test"
if: build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-cassandra-sink-tests?(,|$$)/
command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-dev"
depends_on:
- "build"
- "build-other"
plugins:
- docker-compose#v4.9.0:
run: sink-test-env
config: ci/docker-compose.yml
mount-buildkite-agent: true
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 10
retry: *auto-retry

- label: "e2e java-binding test"
if: build.pull_request.labels includes "ci/run-java-binding-tests" || build.env("CI_STEPS") =~ /(^|,)java-binding-tests?(,|$$)/
command: "ci/scripts/java-binding-test.sh -p ci-dev"
Expand Down
Loading

0 comments on commit f44aaf4

Please sign in to comment.