test: More integrations tests for jni catalog. (#14885)

.licenserc.yaml

@@ -17,6 +17,7 @@ header:
     - "**/*.d.ts"
     - "src/sqlparser/**/*.rs"
     - "java/connector-node/risingwave-source-cdc/src/main/java/com/risingwave/connector/cdc/debezium/internal/*.java"
+    - "java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/**/*.java"
     - "src/meta/model_v2/migration/**/*.rs"
     - "lints/ui/**"
 

ci/scripts/gen-integration-test-yaml.py

@@ -15,6 +15,7 @@
     'mysql-sink': ['json'],
     'postgres-sink': ['json'],
     'iceberg-cdc': ['json'],
+    'iceberg-sink': ['none'],
     'twitter': ['json', 'protobuf'],
     'twitter-pulsar': ['json'],
     'debezium-mysql': ['json'],

ci/workflows/pull-request.yml

@@ -214,7 +214,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 5
+    timeout_in_minutes: 10
     retry: *auto-retry
 
   - label: "end-to-end iceberg sink v2 test"
@@ -229,7 +229,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
+    timeout_in_minutes: 15
     retry: *auto-retry
 
   - label: "end-to-end iceberg cdc test"
@@ -244,7 +244,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
+    timeout_in_minutes: 15
     retry: *auto-retry
 
   - label: "end-to-end pulsar sink test"

e2e_test/iceberg/test_case/cdc/load.slt

@@ -24,6 +24,7 @@ CREATE SINK s1 AS select * from products WITH (
     connector = 'iceberg',
     type = 'upsert',
     force_append_only = 'false',
+    catalog.name = 'demo',
     database.name = 'demo_db',
     table.name = 'demo_table',
     catalog.type = 'storage',

e2e_test/iceberg/test_case/iceberg_sink_append_only.slt

@@ -25,6 +25,7 @@ CREATE SINK s6 AS select * from mv6 WITH (
     force_append_only = 'true',
     database.name = 'demo_db',
     table.name = 'demo_table',
+    catalog.name = 'demo',
     catalog.type = 'storage',
     warehouse.path = 's3://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',

e2e_test/iceberg/test_case/iceberg_sink_upsert.slt

@@ -12,6 +12,7 @@ CREATE SINK s6 AS select mv6.id as id, mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3,
     connector = 'iceberg',
     type = 'upsert',
     force_append_only = 'false',
+    catalog.name = 'demo',
     database.name = 'demo_db',
     table.name = 'demo_table',
     catalog.type = 'storage',

e2e_test/sink/iceberg_sink.slt

@@ -14,6 +14,7 @@ CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH
     s3.access.key = 'hummockadmin',
     s3.secret.key = 'hummockadmin',
     s3.region = 'us-east-1',
+    catalog.name = 'demo',
     catalog.type = 'storage',
     database.name='demo_db',
     table.name='demo_table'

integration_tests/iceberg-sink2/README.md

@@ -1,7 +1,15 @@
-Use following steps to run:
+# How to run the test
 
-1. ./risedev d full-iceberg-bench
-2. cd docker; docker compose up -d
-3. poetry update
-4. poetry run python init.py
-5. poetry run python check.py
+Run following command to run the test:
+
+```bash
+cd python
+poetry update
+poetry run python main.py
+```
+
+# How to override risingwave image version:
+
+```bash
+export RW_IMAGE=<your version>
+```

integration_tests/iceberg-sink2/docker/hive/config.ini

@@ -0,0 +1,20 @@
+[risingwave]
+db=dev
+user=root
+host=127.0.0.1
+port=4566
+
+[sink]
+connector = iceberg
+type=append-only
+force_append_only = true
+catalog.type = hive
+catalog.uri = thrift://metastore:9083
+warehouse.path = s3://icebergdata/demo
+s3.endpoint=http://minio-0:9301
+s3.access.key = hummockadmin
+s3.secret.key = hummockadmin
+s3.region = ap-southeast-1
+catalog.name = demo
+database.name=s1
+table.name=t1

integration_tests/iceberg-sink2/docker/hive/docker-compose.yml

@@ -0,0 +1,116 @@
+version: '3.8'
+
+services:
+  postgres:
+    image: postgres:16.1
+    environment:
+      POSTGRES_USER: admin
+      POSTGRES_PASSWORD: 123456
+      POSTGRES_DB: metastore_db
+    expose:
+      - 5432
+    ports:
+      - "5432:5432"
+    networks:
+      iceberg_net:
+  spark:
+    depends_on:
+      - minio-0
+      - metastore
+    image: ghcr.io/icelake-io/icelake-spark:1.0
+    environment:
+      - AWS_ACCESS_KEY_ID=hummockadmin
+      - AWS_SECRET_ACCESS_KEY=hummockadmin
+      - AWS_REGION=us-east-1
+      - SPARK_HOME=/opt/spark
+      - PYSPARK_PYTHON=/usr/bin/python3.9
+      - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/spark/bin:/opt/spark/sbin
+    user: root
+    networks:
+      iceberg_net:
+    links:
+      - minio-0:icebergdata.minio-0
+    expose:
+      - 15002
+    healthcheck:
+      test: netstat -ltn | grep -c 15002
+      interval: 1s
+      retries: 1200
+    volumes:
+      - ./spark-script:/spark-script
+    entrypoint: [ "/spark-script/spark-connect-server.sh" ]
+
+  risingwave-standalone:
+    extends:
+      file: ../../../../docker/docker-compose.yml
+      service: risingwave-standalone
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;'
+      interval: 1s
+      timeout: 30s
+    environment:
+      - AWS_REGION=us-east-1
+    links:
+      - minio-0:icebergdata.minio-0
+    networks:
+      iceberg_net:
+
+  minio-0:
+    extends:
+      file: ../../../../docker/docker-compose.yml
+      service: minio-0
+    entrypoint: "
+          /bin/sh -c '
+
+          set -e
+
+          mkdir -p \"/data/icebergdata/demo\"
+          mkdir -p \"/data/hummock001\"
+
+          /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\"
+
+          '"
+    networks:
+      iceberg_net:
+
+  etcd-0:
+    extends:
+      file: ../../../../docker/docker-compose.yml
+      service: etcd-0
+    networks:
+      iceberg_net:
+
+  metastore:
+    image: naushadh/hive-metastore
+    depends_on:
+      - postgres
+    environment:
+      - DATABASE_HOST=postgres
+      - DATABASE_DB=metastore_db
+      - DATABASE_USER=admin
+      - DATABASE_PASSWORD=123456
+      - AWS_ACCESS_KEY_ID=hummockadmin
+      - AWS_SECRET_ACCESS_KEY=hummockadmin
+      - S3_ENDPOINT_URL=http://minio-0:9301
+      - S3_BUCKET=icebergdata
+      - S3_PREFIX=demo
+    ports:
+      - "9083:9083"
+    expose:
+      - 9083
+    networks:
+      iceberg_net:
+
+volumes:
+  risingwave-standalone:
+    external: false
+  etcd-0:
+    external: false
+  minio-0:
+    external: false
+
+networks:
+  iceberg_net:
+    name: iceberg

integration_tests/iceberg-sink2/docker/hive/spark-script/spark-connect-server.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -ex
+
+JARS=$(find /opt/spark/deps -type f -name "*.jar" | tr '\n' ':')
+
+/opt/spark/sbin/start-connect-server.sh  \
+  --master local[3] \
+  --driver-class-path $JARS \
+  --conf spark.driver.bindAddress=0.0.0.0 \
+  --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \
+  --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
+  --conf spark.sql.catalog.demo.catalog-impl=org.apache.iceberg.hive.HiveCatalog \
+  --conf spark.sql.catalog.demo.uri=thrift://metastore:9083 \
+  --conf spark.sql.catalog.demo.clients=10 \
+  --conf spark.sql.catalog.demo.warehouse=s3a://icebergdata/demo \
+  --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://minio-0:9301 \
+  --conf spark.sql.catalog.demo.hadoop.fs.s3a.path.style.access=true \
+  --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=hummockadmin \
+  --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=hummockadmin \
+  --conf spark.sql.defaultCatalog=demo
+
+tail -f /opt/spark/logs/spark*.out

integration_tests/iceberg-sink2/docker/jdbc/config.ini

@@ -0,0 +1,22 @@
+[risingwave]
+db=dev
+user=root
+host=127.0.0.1
+port=4566
+
+[sink]
+connector = iceberg
+type=append-only
+force_append_only = true
+warehouse.path = s3://icebergdata/demo
+s3.endpoint=http://minio-0:9301
+s3.access.key = hummockadmin
+s3.secret.key = hummockadmin
+s3.region = ap-southeast-1
+catalog.name = demo
+catalog.type = jdbc
+catalog.uri = jdbc:postgresql://postgres:5432/iceberg
+catalog.jdbc.user = admin
+catalog.jdbc.password = 123456
+database.name=s1
+table.name=t1