fix

fix fix doris docker fix fix fix fix fix fix fix fix fix fix
risingwavelabs · Dec 25, 2023 · d7be8da · d7be8da
1 parent f44aaf4
commit d7be8da
Show file tree

Hide file tree

Showing 15 changed files with 148 additions and 75 deletions.
diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
@@ -89,10 +89,24 @@ services:
       - elasticsearch
       - clickhouse-server
       - redis-server
-      - pulsar
+      - pulsar-server
+      - cassandra-server
+      - starrocks-fe-server
+      - starrocks-be-server
     volumes:
       - ..:/risingwave
 
+  sink-doris-env:
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231109
+    depends_on:
+      - doris-fe-server
+      - doris-be-server
+    volumes:
+      - ..:/risingwave
+    networks:
+      mynetwork:
+        ipv4_address: 172.21.0.1
+
   rw-build-env:
     image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231109
     volumes:
@@ -153,43 +167,93 @@ services:
       - "9004:9004"
     expose:
       - 9009
+
   redis-server:
+    container_name: redis-server
     image: 'redis:latest'
     expose:
       - 6379
     ports:
-      - 6379:6379
+      - 6378:6379
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
       timeout: 30s
       retries: 50
-  doris-fe:
+
+  doris-fe-server:
+    platform: linux/amd64
     image: apache/doris:2.0.0_alpha-fe-x86_64
-    hostname: fe
+    hostname: doris-fe-server
     environment:
-     - FE_SERVERS=fe1:127.0.0.1:9010
+     - FE_SERVERS=fe1:172.21.0.2:9010
      - FE_ID=1
-    network_mode: host
     ports:
       - "8030:8030"
       - "9030:9030"
-  doris-be:
+    networks:
+      mynetwork:
+        ipv4_address: 172.21.0.2
+
+  doris-be-server:
+    platform: linux/amd64
     image: apache/doris:2.0.0_alpha-be-x86_64
-    hostname: be
+    hostname: doris-be-server
     environment:
-     - FE_SERVERS=fe1:127.0.0.1:9010
-     - BE_ADDR=127.0.0.1:9050
+     - FE_SERVERS=fe1:172.21.0.2:9010
+     - BE_ADDR=172.21.0.3:9050
     depends_on:
       - fe
     ports:
       - "9050:9050"
-    network_mode: host
+    networks:
+      mynetwork:
+        ipv4_address: 172.21.0.3
+
+  cassandra-server:
+    container_name: cassandra-server
+    image: cassandra:4.0
+    ports:
+      - 9042:9042
+    environment:
+      - CASSANDRA_CLUSTER_NAME=cloudinfra
+
+  starrocks-fe-server:
+    container_name: starrocks-fe-server
+    image: starrocks/fe-ubuntu:latest
+    hostname: starrocks-fe-server
+    command:
+      /opt/starrocks/fe/bin/start_fe.sh
+    ports:
+      - 28030:8030
+      - 29020:9020
+      - 29030:9030
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9030"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
+  starrocks-be-server:
+    image: starrocks/be-ubuntu:latest
+    command:
+      - /bin/bash
+      - -c
+      - |
+        sleep 15s; mysql --connect-timeout 2 -h starrocks-fe-server -P9030 -uroot -e "alter system add backend \"starrocks-be-server:9050\";"
+        /opt/starrocks/be/bin/start_be.sh
+    ports:
+      - 28040:8040
+      - 29050:9050
+    hostname: starrocks-be-server
+    container_name: starrocks-be-server
+    depends_on:
+      - starrocks-fe-server
 
-# Temporary workaround for json schema registry test since redpanda only supports
-# protobuf/avro schema registry. Should be removed after the support.
-# Related tracking issue:
-# https://github.com/redpanda-data/redpanda/issues/1878
+# # Temporary workaround for json schema registry test since redpanda only supports
+# # protobuf/avro schema registry. Should be removed after the support.
+# # Related tracking issue:
+# # https://github.com/redpanda-data/redpanda/issues/1878
   zookeeper:
     container_name: zookeeper
     image: confluentinc/cp-zookeeper:latest
@@ -228,8 +292,8 @@ services:
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
 
-  pulsar:
-    container_name: pulsar
+  pulsar-server:
+    container_name: pulsar-server
     image: apachepulsar/pulsar:latest
     command: bin/pulsar standalone
     ports:
@@ -243,3 +307,9 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
+networks:
+  mynetwork:
+    ipam:
+      config:
+        - subnet: 172.21.80.0/16
+  default:
diff --git a/ci/scripts/e2e-cassandra-test.sh → ci/scripts/e2e-cassandra-sink-test.sh b/ci/scripts/e2e-cassandra-test.sh → ci/scripts/e2e-cassandra-sink-test.sh
@@ -38,15 +38,19 @@ sleep 1
 echo "--- create cassandra table"
 curl https://downloads.apache.org/cassandra/4.1.3/apache-cassandra-4.1.3-bin.tar.gz  --output apache-cassandra-4.1.3-bin.tar.gz
 tar xfvz apache-cassandra-4.1.3-bin.tar.gz
+cd apache-cassandra-4.1.3/bin
+export CQLSH_HOST=cassandra-server
+export CQLSH_PORT=9042
 ./cqlsh -e "CREATE KEYSPACE demo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};use demo;
 CREATE table demo_bhv_table(v1 int primary key,v2 smallint,v3 bigint,v4 float,v5 double,v6 text,v7 date,v8 timestamp,v9 boolean);"
 
 echo "--- testing sinks"
+cd ../../
 sqllogictest -p 4566 -d dev './e2e_test/sink/cassandra_sink.slt'
 sleep 1
-./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false;"
+cd apache-cassandra-4.1.3/bin
+./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false AND ENCODING = 'UTF-8';"
 
-AND ENCODING = 'UTF-8';
 if cat ./query_result.csv | awk -F "," '{
     exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01.000+0000" && $9 == "False\r"); }'; then
   echo "Cassandra sink check passed"

diff --git a/ci/scripts/e2e-deltalake-sink-rust-test.sh b/ci/scripts/e2e-deltalake-sink-rust-test.sh
@@ -32,7 +32,6 @@ mkdir ./connector-node
 tar xf ./risingwave-connector.tar.gz -C ./connector-node
 
 echo "--- starting risingwave cluster"
-mkdir -p .risingwave/log
 cargo make ci-start ci-sink-test
 sleep 1
 

diff --git a/ci/scripts/e2e-doris-sink-test.sh b/ci/scripts/e2e-doris-sink-test.sh
@@ -31,7 +31,7 @@ sleep 1
 echo "--- create doris table"
 apt-get install -y mysql-client
 sleep 2
-mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo;
+mysql -uroot -P 9030 -h doris-fe-server -e "CREATE database demo;use demo;
 CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean) UNIQUE KEY(\`v1\`)
 DISTRIBUTED BY HASH(\`v1\`) BUCKETS 1
 PROPERTIES (
@@ -44,7 +44,7 @@ sleep 2
 echo "--- testing sinks"
 sqllogictest -p 4566 -d dev './e2e_test/sink/doris_sink.slt'
 sleep 1
-mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv
+mysql -uroot -P 9030 -h doris-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv
 
 
 if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{                                                       ─╯

diff --git a/ci/scripts/e2e-elasticsearch-sink-test.sh b/ci/scripts/e2e-elasticsearch-sink-test.sh
diff --git a/ci/scripts/e2e-redis-sink-test.sh b/ci/scripts/e2e-redis-sink-test.sh
@@ -24,16 +24,16 @@ shift $((OPTIND -1))
 download_and_prepare_rw "$profile" source
 
 echo "--- starting risingwave cluster"
-mkdir -p .risingwave/log
 cargo make ci-start ci-sink-test
+apt-get update -y && apt-get install -y redis-server
 sleep 1
 
 echo "--- testing sinks"
 sqllogictest -p 4566 -d dev './e2e_test/sink/redis_sink.slt'
 sleep 1
 
-redis-cli -p 6378 get {\"v1\":1} >> ./query_result.txt
-redis-cli -p 6378 get V1:1 >> ./query_result.txt
+redis-cli -h redis-server -p 6379 get {\"v1\":1} >> ./query_result.txt
+redis-cli -h redis-server -p 6379 get V1:1 >> ./query_result.txt
 
 # check sink destination using shell
 if cat ./query_result.txt | tr '\n' '\0' | xargs -0 -n1 bash -c '[[ "$0" == "{\"v1\":1,\"v2\":1,\"v3\":1,\"v4\":1.100000023841858,\"v5\":1.2,\"v6\":\"test\",\"v7\":734869,\"v8\":\"2013-01-01T01:01:01.000000Z\",\"v9\":false}" || "$0" == "V2:1,V3:1" ]]'; then

diff --git a/ci/scripts/e2e-starrocks-sink-test.sh b/ci/scripts/e2e-starrocks-sink-test.sh
@@ -29,9 +29,9 @@ sleep 1
 
 
 echo "--- create starrocks table"
-apt-get install -y mysql-client
+apt-get update -y && apt-get install -y mysql-client
 sleep 2
-mysql -uroot -P9030 -h127.0.0.1 -e "CREATE database demo;use demo;
+mysql -uroot -P 9030 -h starrocks-fe-server -e "CREATE database demo;use demo;
 CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 date,v8 datetime,v9 boolean) ENGINE=OLAP
 PRIMARY KEY(\`v1\`)
 DISTRIBUTED BY HASH(\`v1\`) properties(\"replication_num\" = \"1\");
@@ -42,7 +42,7 @@ sleep 2
 echo "--- testing sinks"
 sqllogictest -p 4566 -d dev './e2e_test/sink/starrocks_sink.slt'
 sleep 1
-mysql -uroot -P9030 -h127.0.0.1 -e "select * from demo.demo_bhv_table" > ./query_result.csv
+mysql -uroot -P 9030 -h starrocks-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv
 
 
 if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{

diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
@@ -730,7 +730,7 @@ steps:
       - "build-other"
     plugins:
       - docker-compose#v4.9.0:
-          run: sink-test-env
+          run: sink-doris-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs

diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml
@@ -315,7 +315,7 @@ steps:
       - "build-other"
     plugins:
       - docker-compose#v4.9.0:
-          run: sink-test-env
+          run: sink-doris-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs

diff --git a/e2e_test/sink/cassandra_sink.slt b/e2e_test/sink/cassandra_sink.slt
@@ -5,13 +5,13 @@ statement ok
 CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6;
 
 statement ok
-CREATE SINK bhv_cassandra_sink_1
+CREATE SINK s6
 FROM
     mv6 WITH (
     connector = 'cassandra',
     type = 'append-only',
     force_append_only='true',
-    cassandra.url = '127.0.0.1:9042',
+    cassandra.url = 'cassandra-server:9042',
     cassandra.keyspace  = 'demo',
     cassandra.table = 'demo_bhv_table',
     cassandra.datacenter = 'datacenter1',

diff --git a/e2e_test/sink/deltalake_rust_sink.slt b/e2e_test/sink/deltalake_rust_sink.slt
@@ -1,27 +1,23 @@
 statement ok
-CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamp, v9 boolean);
+CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamptz, v9 boolean);
 
 statement ok
 CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6;
 
 statement ok
-CREATE SINK bhv_doris_sink_1
-FROM
-    mv6 WITH (
-    connector = 'doris',
+create sink s6 as select * from mv6
+with (
+    connector = 'deltalake_rust',
     type = 'append-only',
-    doris.url = 'http://127.0.0.1:8030',
-    doris.user = 'users',
-    doris.password = '123456',
-    doris.database = 'demo',
-    doris.table='demo_bhv_table',
-    force_append_only='true'
+    force_append_only = 'true',
+    location = 's3a://deltalake/deltalake-test',
+    s3.access.key = 'hummockadmin',
+    s3.secret.key = 'hummockadmin',
+    s3.endpoint = 'http://127.0.0.1:9301'
 );
 
 statement ok
-
-statement ok
-INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01' , false);
+INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01+00:00' , false);
 
 statement ok
 FLUSH;

diff --git a/e2e_test/sink/doris_sink.slt b/e2e_test/sink/doris_sink.slt
@@ -5,12 +5,12 @@ statement ok
 CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6;
 
 statement ok
-CREATE SINK bhv_doris_sink_1
+CREATE SINK s6
 FROM
     mv6 WITH (
     connector = 'doris',
     type = 'append-only',
-    doris.url = 'http://127.0.0.1:8030',
+    doris.url = 'http://doris-fe-server:8030',
     doris.user = 'users',
     doris.password = '123456',
     doris.database = 'demo',

diff --git a/e2e_test/sink/redis_sink.slt b/e2e_test/sink/redis_sink.slt
@@ -5,21 +5,21 @@ statement ok
 CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6;
 
 statement ok
-CREATE SINK bhv_redis_sink_1
+CREATE SINK s61
 FROM
     mv6 WITH (
     primary_key = 'v1',
     connector = 'redis',
-    redis.url= 'redis://127.0.0.1:6379/',
+    redis.url= 'redis://redis-server:6379/',
 )FORMAT PLAIN ENCODE JSON(force_append_only='true');
 
 statement ok
-CREATE SINK bhv_redis_sink_2
+CREATE SINK s62
 FROM
     mv6 WITH (
     primary_key = 'v1',
     connector = 'redis',
-    redis.url= 'redis://127.0.0.1:6379/',
+    redis.url= 'redis://redis-server:6379/',
 )FORMAT PLAIN ENCODE TEMPLATE(force_append_only='true', key_format = 'V1:{v1}', value_format = 'V2:{v2},V3:{v3}');
 
 statement ok
@@ -29,7 +29,10 @@ statement ok
 FLUSH;
 
 statement ok
-DROP SINK s6;
+DROP SINK s61;
+
+statement ok
+DROP SINK s62;
 
 statement ok
 DROP MATERIALIZED VIEW mv6;

diff --git a/e2e_test/sink/starrocks_sink.slt b/e2e_test/sink/starrocks_sink.slt
@@ -1,15 +1,16 @@
+statement ok
 CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamp, v9 boolean);
 
 statement ok
 CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6;
 
 statement ok
-CREATE SINK bhv_starrocks_sink_1
+CREATE SINK s6
 FROM
     mv6 WITH (
     connector = 'starrocks',
     type = 'upsert',
-    starrocks.host = '127.0.0.1',
+    starrocks.host = 'starrocks-fe-server',
     starrocks.mysqlport = '9030',
     starrocks.httpport = '8030',
     starrocks.user = 'users',