From d7be8da1895afcf71b46ad900f498c8b850413df Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Fri, 22 Dec 2023 11:51:38 +0800 Subject: [PATCH] fix fix fix doris docker fix fix fix fix fix fix fix fix fix fix --- ci/docker-compose.yml | 104 +++++++++++++++--- ...dra-test.sh => e2e-cassandra-sink-test.sh} | 8 +- ci/scripts/e2e-deltalake-sink-rust-test.sh | 1 - ci/scripts/e2e-doris-sink-test.sh | 4 +- ci/scripts/e2e-elasticsearch-sink-test.sh | 0 ci/scripts/e2e-redis-sink-test.sh | 6 +- ci/scripts/e2e-starrocks-sink-test.sh | 6 +- ci/workflows/main-cron.yml | 2 +- ci/workflows/pull-request.yml | 2 +- e2e_test/sink/cassandra_sink.slt | 4 +- e2e_test/sink/deltalake_rust_sink.slt | 24 ++-- e2e_test/sink/doris_sink.slt | 4 +- e2e_test/sink/redis_sink.slt | 13 ++- e2e_test/sink/starrocks_sink.slt | 5 +- .../starrocks-sink/docker-compose.yml | 40 +++---- 15 files changed, 148 insertions(+), 75 deletions(-) rename ci/scripts/{e2e-cassandra-test.sh => e2e-cassandra-sink-test.sh} (91%) mode change 100644 => 100755 mode change 100644 => 100755 ci/scripts/e2e-doris-sink-test.sh mode change 100644 => 100755 ci/scripts/e2e-elasticsearch-sink-test.sh mode change 100644 => 100755 ci/scripts/e2e-redis-sink-test.sh mode change 100644 => 100755 ci/scripts/e2e-starrocks-sink-test.sh diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index a7d81dc61022e..0522c2dd4e354 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -89,10 +89,24 @@ services: - elasticsearch - clickhouse-server - redis-server - - pulsar + - pulsar-server + - cassandra-server + - starrocks-fe-server + - starrocks-be-server volumes: - ..:/risingwave + sink-doris-env: + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231109 + depends_on: + - doris-fe-server + - doris-be-server + volumes: + - ..:/risingwave + networks: + mynetwork: + ipv4_address: 172.21.0.1 + rw-build-env: image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231109 volumes: @@ -153,43 +167,93 @@ services: - "9004:9004" expose: - 9009 + redis-server: + container_name: redis-server image: 'redis:latest' expose: - 6379 ports: - - 6379:6379 + - 6378:6379 healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s timeout: 30s retries: 50 - doris-fe: + + doris-fe-server: + platform: linux/amd64 image: apache/doris:2.0.0_alpha-fe-x86_64 - hostname: fe + hostname: doris-fe-server environment: - - FE_SERVERS=fe1:127.0.0.1:9010 + - FE_SERVERS=fe1:172.21.0.2:9010 - FE_ID=1 - network_mode: host ports: - "8030:8030" - "9030:9030" - doris-be: + networks: + mynetwork: + ipv4_address: 172.21.0.2 + + doris-be-server: + platform: linux/amd64 image: apache/doris:2.0.0_alpha-be-x86_64 - hostname: be + hostname: doris-be-server environment: - - FE_SERVERS=fe1:127.0.0.1:9010 - - BE_ADDR=127.0.0.1:9050 + - FE_SERVERS=fe1:172.21.0.2:9010 + - BE_ADDR=172.21.0.3:9050 depends_on: - fe ports: - "9050:9050" - network_mode: host + networks: + mynetwork: + ipv4_address: 172.21.0.3 + + cassandra-server: + container_name: cassandra-server + image: cassandra:4.0 + ports: + - 9042:9042 + environment: + - CASSANDRA_CLUSTER_NAME=cloudinfra + + starrocks-fe-server: + container_name: starrocks-fe-server + image: starrocks/fe-ubuntu:latest + hostname: starrocks-fe-server + command: + /opt/starrocks/fe/bin/start_fe.sh + ports: + - 28030:8030 + - 29020:9020 + - 29030:9030 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9030"] + interval: 5s + timeout: 5s + retries: 30 + + starrocks-be-server: + image: starrocks/be-ubuntu:latest + command: + - /bin/bash + - -c + - | + sleep 15s; mysql --connect-timeout 2 -h starrocks-fe-server -P9030 -uroot -e "alter system add backend \"starrocks-be-server:9050\";" + /opt/starrocks/be/bin/start_be.sh + ports: + - 28040:8040 + - 29050:9050 + hostname: starrocks-be-server + container_name: starrocks-be-server + depends_on: + - starrocks-fe-server -# Temporary workaround for json schema registry test since redpanda only supports -# protobuf/avro schema registry. Should be removed after the support. -# Related tracking issue: -# https://github.com/redpanda-data/redpanda/issues/1878 +# # Temporary workaround for json schema registry test since redpanda only supports +# # protobuf/avro schema registry. Should be removed after the support. +# # Related tracking issue: +# # https://github.com/redpanda-data/redpanda/issues/1878 zookeeper: container_name: zookeeper image: confluentinc/cp-zookeeper:latest @@ -228,8 +292,8 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - pulsar: - container_name: pulsar + pulsar-server: + container_name: pulsar-server image: apachepulsar/pulsar:latest command: bin/pulsar standalone ports: @@ -243,3 +307,9 @@ services: interval: 5s timeout: 5s retries: 5 +networks: + mynetwork: + ipam: + config: + - subnet: 172.21.80.0/16 + default: diff --git a/ci/scripts/e2e-cassandra-test.sh b/ci/scripts/e2e-cassandra-sink-test.sh old mode 100644 new mode 100755 similarity index 91% rename from ci/scripts/e2e-cassandra-test.sh rename to ci/scripts/e2e-cassandra-sink-test.sh index 30720c61cb0c7..deafbf1f1785d --- a/ci/scripts/e2e-cassandra-test.sh +++ b/ci/scripts/e2e-cassandra-sink-test.sh @@ -38,15 +38,19 @@ sleep 1 echo "--- create cassandra table" curl https://downloads.apache.org/cassandra/4.1.3/apache-cassandra-4.1.3-bin.tar.gz --output apache-cassandra-4.1.3-bin.tar.gz tar xfvz apache-cassandra-4.1.3-bin.tar.gz +cd apache-cassandra-4.1.3/bin +export CQLSH_HOST=cassandra-server +export CQLSH_PORT=9042 ./cqlsh -e "CREATE KEYSPACE demo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};use demo; CREATE table demo_bhv_table(v1 int primary key,v2 smallint,v3 bigint,v4 float,v5 double,v6 text,v7 date,v8 timestamp,v9 boolean);" echo "--- testing sinks" +cd ../../ sqllogictest -p 4566 -d dev './e2e_test/sink/cassandra_sink.slt' sleep 1 -./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false;" +cd apache-cassandra-4.1.3/bin +./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false AND ENCODING = 'UTF-8';" -AND ENCODING = 'UTF-8'; if cat ./query_result.csv | awk -F "," '{ exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01.000+0000" && $9 == "False\r"); }'; then echo "Cassandra sink check passed" diff --git a/ci/scripts/e2e-deltalake-sink-rust-test.sh b/ci/scripts/e2e-deltalake-sink-rust-test.sh index 14476127e9b12..cc0c287e8b572 100755 --- a/ci/scripts/e2e-deltalake-sink-rust-test.sh +++ b/ci/scripts/e2e-deltalake-sink-rust-test.sh @@ -32,7 +32,6 @@ mkdir ./connector-node tar xf ./risingwave-connector.tar.gz -C ./connector-node echo "--- starting risingwave cluster" -mkdir -p .risingwave/log cargo make ci-start ci-sink-test sleep 1 diff --git a/ci/scripts/e2e-doris-sink-test.sh b/ci/scripts/e2e-doris-sink-test.sh old mode 100644 new mode 100755 index f3fb61ad8a805..efe6a3820823d --- a/ci/scripts/e2e-doris-sink-test.sh +++ b/ci/scripts/e2e-doris-sink-test.sh @@ -31,7 +31,7 @@ sleep 1 echo "--- create doris table" apt-get install -y mysql-client sleep 2 -mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo; +mysql -uroot -P 9030 -h doris-fe-server -e "CREATE database demo;use demo; CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean) UNIQUE KEY(\`v1\`) DISTRIBUTED BY HASH(\`v1\`) BUCKETS 1 PROPERTIES ( @@ -44,7 +44,7 @@ sleep 2 echo "--- testing sinks" sqllogictest -p 4566 -d dev './e2e_test/sink/doris_sink.slt' sleep 1 -mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv +mysql -uroot -P 9030 -h doris-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ ─╯ diff --git a/ci/scripts/e2e-elasticsearch-sink-test.sh b/ci/scripts/e2e-elasticsearch-sink-test.sh old mode 100644 new mode 100755 diff --git a/ci/scripts/e2e-redis-sink-test.sh b/ci/scripts/e2e-redis-sink-test.sh old mode 100644 new mode 100755 index 6f3ebb97ecc35..cf64662db4051 --- a/ci/scripts/e2e-redis-sink-test.sh +++ b/ci/scripts/e2e-redis-sink-test.sh @@ -24,16 +24,16 @@ shift $((OPTIND -1)) download_and_prepare_rw "$profile" source echo "--- starting risingwave cluster" -mkdir -p .risingwave/log cargo make ci-start ci-sink-test +apt-get update -y && apt-get install -y redis-server sleep 1 echo "--- testing sinks" sqllogictest -p 4566 -d dev './e2e_test/sink/redis_sink.slt' sleep 1 -redis-cli -p 6378 get {\"v1\":1} >> ./query_result.txt -redis-cli -p 6378 get V1:1 >> ./query_result.txt +redis-cli -h redis-server -p 6379 get {\"v1\":1} >> ./query_result.txt +redis-cli -h redis-server -p 6379 get V1:1 >> ./query_result.txt # check sink destination using shell if cat ./query_result.txt | tr '\n' '\0' | xargs -0 -n1 bash -c '[[ "$0" == "{\"v1\":1,\"v2\":1,\"v3\":1,\"v4\":1.100000023841858,\"v5\":1.2,\"v6\":\"test\",\"v7\":734869,\"v8\":\"2013-01-01T01:01:01.000000Z\",\"v9\":false}" || "$0" == "V2:1,V3:1" ]]'; then diff --git a/ci/scripts/e2e-starrocks-sink-test.sh b/ci/scripts/e2e-starrocks-sink-test.sh old mode 100644 new mode 100755 index eb89ed0bfce14..bdee699a18744 --- a/ci/scripts/e2e-starrocks-sink-test.sh +++ b/ci/scripts/e2e-starrocks-sink-test.sh @@ -29,9 +29,9 @@ sleep 1 echo "--- create starrocks table" -apt-get install -y mysql-client +apt-get update -y && apt-get install -y mysql-client sleep 2 -mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo; +mysql -uroot -P 9030 -h starrocks-fe-server -e "CREATE database demo;use demo; CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 date,v8 datetime,v9 boolean) ENGINE=OLAP PRIMARY KEY(\`v1\`) DISTRIBUTED BY HASH(\`v1\`) properties(\"replication_num\" = \"1\"); @@ -42,7 +42,7 @@ sleep 2 echo "--- testing sinks" sqllogictest -p 4566 -d dev './e2e_test/sink/starrocks_sink.slt' sleep 1 -mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv +mysql -uroot -P 9030 -h starrocks-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index e59db8b176f1b..5f52cbe8c771b 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -730,7 +730,7 @@ steps: - "build-other" plugins: - docker-compose#v4.9.0: - run: sink-test-env + run: sink-doris-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index a12ae48c2829d..66ec85cf8987c 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -315,7 +315,7 @@ steps: - "build-other" plugins: - docker-compose#v4.9.0: - run: sink-test-env + run: sink-doris-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs diff --git a/e2e_test/sink/cassandra_sink.slt b/e2e_test/sink/cassandra_sink.slt index a629df2826d52..97e6b3407ef2f 100644 --- a/e2e_test/sink/cassandra_sink.slt +++ b/e2e_test/sink/cassandra_sink.slt @@ -5,13 +5,13 @@ statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK bhv_cassandra_sink_1 +CREATE SINK s6 FROM mv6 WITH ( connector = 'cassandra', type = 'append-only', force_append_only='true', - cassandra.url = '127.0.0.1:9042', + cassandra.url = 'cassandra-server:9042', cassandra.keyspace = 'demo', cassandra.table = 'demo_bhv_table', cassandra.datacenter = 'datacenter1', diff --git a/e2e_test/sink/deltalake_rust_sink.slt b/e2e_test/sink/deltalake_rust_sink.slt index 310c6a3823a7a..4207c9446dc1b 100644 --- a/e2e_test/sink/deltalake_rust_sink.slt +++ b/e2e_test/sink/deltalake_rust_sink.slt @@ -1,27 +1,23 @@ statement ok -CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamp, v9 boolean); +CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamptz, v9 boolean); statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK bhv_doris_sink_1 -FROM - mv6 WITH ( - connector = 'doris', +create sink s6 as select * from mv6 +with ( + connector = 'deltalake_rust', type = 'append-only', - doris.url = 'http://127.0.0.1:8030', - doris.user = 'users', - doris.password = '123456', - doris.database = 'demo', - doris.table='demo_bhv_table', - force_append_only='true' + force_append_only = 'true', + location = 's3a://deltalake/deltalake-test', + s3.access.key = 'hummockadmin', + s3.secret.key = 'hummockadmin', + s3.endpoint = 'http://127.0.0.1:9301' ); statement ok - -statement ok -INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01' , false); +INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01+00:00' , false); statement ok FLUSH; diff --git a/e2e_test/sink/doris_sink.slt b/e2e_test/sink/doris_sink.slt index d832e89799e7e..2c552bbb26143 100644 --- a/e2e_test/sink/doris_sink.slt +++ b/e2e_test/sink/doris_sink.slt @@ -5,12 +5,12 @@ statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK bhv_doris_sink_1 +CREATE SINK s6 FROM mv6 WITH ( connector = 'doris', type = 'append-only', - doris.url = 'http://127.0.0.1:8030', + doris.url = 'http://doris-fe-server:8030', doris.user = 'users', doris.password = '123456', doris.database = 'demo', diff --git a/e2e_test/sink/redis_sink.slt b/e2e_test/sink/redis_sink.slt index 1d5c068e0e66e..7475a80ae696e 100644 --- a/e2e_test/sink/redis_sink.slt +++ b/e2e_test/sink/redis_sink.slt @@ -5,21 +5,21 @@ statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK bhv_redis_sink_1 +CREATE SINK s61 FROM mv6 WITH ( primary_key = 'v1', connector = 'redis', - redis.url= 'redis://127.0.0.1:6379/', + redis.url= 'redis://redis-server:6379/', )FORMAT PLAIN ENCODE JSON(force_append_only='true'); statement ok -CREATE SINK bhv_redis_sink_2 +CREATE SINK s62 FROM mv6 WITH ( primary_key = 'v1', connector = 'redis', - redis.url= 'redis://127.0.0.1:6379/', + redis.url= 'redis://redis-server:6379/', )FORMAT PLAIN ENCODE TEMPLATE(force_append_only='true', key_format = 'V1:{v1}', value_format = 'V2:{v2},V3:{v3}'); statement ok @@ -29,7 +29,10 @@ statement ok FLUSH; statement ok -DROP SINK s6; +DROP SINK s61; + +statement ok +DROP SINK s62; statement ok DROP MATERIALIZED VIEW mv6; diff --git a/e2e_test/sink/starrocks_sink.slt b/e2e_test/sink/starrocks_sink.slt index 0279d3a29203f..b83f5a8a0142c 100644 --- a/e2e_test/sink/starrocks_sink.slt +++ b/e2e_test/sink/starrocks_sink.slt @@ -1,15 +1,16 @@ +statement ok CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamp, v9 boolean); statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK bhv_starrocks_sink_1 +CREATE SINK s6 FROM mv6 WITH ( connector = 'starrocks', type = 'upsert', - starrocks.host = '127.0.0.1', + starrocks.host = 'starrocks-fe-server', starrocks.mysqlport = '9030', starrocks.httpport = '8030', starrocks.user = 'users', diff --git a/integration_tests/starrocks-sink/docker-compose.yml b/integration_tests/starrocks-sink/docker-compose.yml index bb4a5f6ad25fd..520b678a24def 100644 --- a/integration_tests/starrocks-sink/docker-compose.yml +++ b/integration_tests/starrocks-sink/docker-compose.yml @@ -30,26 +30,26 @@ services: container_name: starrocks-be depends_on: - starrocks-fe - # risingwave-standalone: - # extends: - # file: ../../docker/docker-compose.yml - # service: risingwave-standalone - # etcd-0: - # extends: - # file: ../../docker/docker-compose.yml - # service: etcd-0 - # grafana-0: - # extends: - # file: ../../docker/docker-compose.yml - # service: grafana-0 - # minio-0: - # extends: - # file: ../../docker/docker-compose.yml - # service: minio-0 - # prometheus-0: - # extends: - # file: ../../docker/docker-compose.yml - # service: prometheus-0 + risingwave-standalone: + extends: + file: ../../docker/docker-compose.yml + service: risingwave-standalone + etcd-0: + extends: + file: ../../docker/docker-compose.yml + service: etcd-0 + grafana-0: + extends: + file: ../../docker/docker-compose.yml + service: grafana-0 + minio-0: + extends: + file: ../../docker/docker-compose.yml + service: minio-0 + prometheus-0: + extends: + file: ../../docker/docker-compose.yml + service: prometheus-0 volumes: risingwave-standalone: external: false