test: move client-library tests from risingwave-test repo to integrat…

…ion test (#14272)

ci/scripts/gen-integration-test-yaml.py

@@ -36,6 +36,7 @@
     'doris-sink': ['json'],
     'starrocks-sink': ['json'],
     'deltalake-sink': ['json'],
+    'client-library': ['none'],
 }
 
 def gen_pipeline_steps():

ci/scripts/integration-tests.sh

@@ -34,14 +34,6 @@ docker volume prune -f
 echo "--- ghcr login"
 echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
 
-echo "--- install postgresql"
-sudo yum install -y postgresql15
-
-echo "--- download rwctest-key"
-aws secretsmanager get-secret-value --secret-id "gcp-buildkite-rwctest-key" --region us-east-2 --query "SecretString" --output text >gcp-rwctest.json
-
-cd integration_tests/scripts
-
 echo "--- case: ${case}, format: ${format}"
 
 if [[ -n "${RW_IMAGE_TAG+x}" ]]; then
@@ -55,6 +47,20 @@ if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
   echo Docker image: $RW_IMAGE
 fi
 
+if [ "${case}" == "client-library" ]; then
+  cd integration_tests/client-library
+  python3 client_test.py
+  exit 0
+fi
+
+echo "--- install postgresql"
+sudo yum install -y postgresql15
+
+echo "--- download rwctest-key"
+aws secretsmanager get-secret-value --secret-id "gcp-buildkite-rwctest-key" --region us-east-2 --query "SecretString" --output text >gcp-rwctest.json
+
+cd integration_tests/scripts
+
 echo "--- rewrite docker compose for protobuf"
 if [ "${format}" == "protobuf" ]; then
   python3 gen_pb_compose.py ${case} ${format}

ci/scripts/notify.py

@@ -59,6 +59,7 @@
     "doris-sink": ["xinhao"],
     "starrocks-sink": ["xinhao"],
     "deltalake-sink": ["xinhao"],
+    "client-library": ["tao"],
 }
 
 def get_failed_tests(get_test_status, test_map):

integration_tests/client-library/client_test.py

@@ -0,0 +1,51 @@
+import subprocess
+import sys
+from time import sleep
+
+
+def check_go():
+    print("--- go client test")
+    subprocess.run(["docker", "compose", "exec", "go-lang", "bash", "-c", "cd /go-client && ./run.sh"], check=True)
+
+
+def check_python():
+    print("--- python client test")
+    subprocess.run(["docker", "compose", "exec", "python", "bash", "-c",
+                    "cd /python-client && pip3 install -r requirements.txt && pytest"], check=True)
+
+
+def check_java():
+    print("--- java client test")
+    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "apt-get update && apt-get install -y maven"],
+                   check=True)
+    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "cd /java-client && mvn clean test"], check=True)
+
+
+subprocess.run(["docker", "compose", "up", "-d"], check=True)
+sleep(10)
+
+failed_cases = []
+
+try:
+    check_go()
+except Exception as e:
+    print(e)
+    failed_cases.append("go client failed")
+
+try:
+    check_python()
+except Exception as e:
+    print(e)
+    failed_cases.append("python client failed")
+
+try:
+    check_java()
+except Exception as e:
+    print(e)
+    failed_cases.append("java client failed")
+
+if len(failed_cases) != 0:
+    print(f"--- client check failed for case\n{failed_cases}")
+    sys.exit(1)
+
+subprocess.run(["docker", "compose", "down", "--remove-orphans", "-v"], check=True)

integration_tests/client-library/docker-compose.yml

@@ -0,0 +1,54 @@
+---
+version: "3"
+services:
+  risingwave-standalone:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: risingwave-standalone
+  etcd-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: etcd-0
+  grafana-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: grafana-0
+  minio-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: minio-0
+  prometheus-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: prometheus-0
+
+  go-lang:
+    image: golang:bullseye
+    command: tail -f /dev/null
+    volumes:
+      - ./go:/go-client
+  python:
+    image: python:3.9.18-slim-bullseye
+    command: tail -f /dev/null
+    volumes:
+      - ./python:/python-client
+  java:
+    image: eclipse-temurin:11.0.21_9-jdk-jammy
+    command: tail -f /dev/null
+    volumes:
+      - ./java:/java-client
+
+volumes:
+  risingwave-standalone:
+    external: false
+  etcd-0:
+    external: false
+  grafana-0:
+    external: false
+  minio-0:
+    external: false
+  prometheus-0:
+    external: false
+  message_queue:
+    external: false
+name: risingwave-compose

integration_tests/client-library/go/go.mod

@@ -0,0 +1,20 @@
+module github.com/risingwave/risingwave-test/client-library-test/go
+
+go 1.20
+
+require (
+	github.com/jackc/pgx/v5 v5.4.3
+	github.com/stretchr/testify v1.8.4
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

integration_tests/client-library/go/go.sum

@@ -0,0 +1,31 @@
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
+github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=