Merge remote-tracking branch 'origin/yiming/remove-java-iceberg-delta…

…lake' into yiming/remove-java-iceberg-deltalake

ci/scripts/gen-integration-test-yaml.py

@@ -36,6 +36,7 @@
     'doris-sink': ['json'],
     'starrocks-sink': ['json'],
     'deltalake-sink': ['json'],
+    'client-library': ['none'],
 }
 
 def gen_pipeline_steps():

ci/scripts/integration-tests.sh

@@ -34,14 +34,6 @@ docker volume prune -f
 echo "--- ghcr login"
 echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
 
-echo "--- install postgresql"
-sudo yum install -y postgresql15
-
-echo "--- download rwctest-key"
-aws secretsmanager get-secret-value --secret-id "gcp-buildkite-rwctest-key" --region us-east-2 --query "SecretString" --output text >gcp-rwctest.json
-
-cd integration_tests/scripts
-
 echo "--- case: ${case}, format: ${format}"
 
 if [[ -n "${RW_IMAGE_TAG+x}" ]]; then
@@ -55,6 +47,20 @@ if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
   echo Docker image: $RW_IMAGE
 fi
 
+if [ "${case}" == "client-library" ]; then
+  cd integration_tests/client-library
+  python3 client_test.py
+  exit 0
+fi
+
+echo "--- install postgresql"
+sudo yum install -y postgresql15
+
+echo "--- download rwctest-key"
+aws secretsmanager get-secret-value --secret-id "gcp-buildkite-rwctest-key" --region us-east-2 --query "SecretString" --output text >gcp-rwctest.json
+
+cd integration_tests/scripts
+
 echo "--- rewrite docker compose for protobuf"
 if [ "${format}" == "protobuf" ]; then
   python3 gen_pb_compose.py ${case} ${format}

ci/scripts/notify.py

@@ -59,6 +59,7 @@
     "doris-sink": ["xinhao"],
     "starrocks-sink": ["xinhao"],
     "deltalake-sink": ["xinhao"],
+    "client-library": ["tao"],
 }
 
 def get_failed_tests(get_test_status, test_map):

integration_tests/client-library/client_test.py

@@ -0,0 +1,51 @@
+import subprocess
+import sys
+from time import sleep
+
+
+def check_go():
+    print("--- go client test")
+    subprocess.run(["docker", "compose", "exec", "go-lang", "bash", "-c", "cd /go-client && ./run.sh"], check=True)
+
+
+def check_python():
+    print("--- python client test")
+    subprocess.run(["docker", "compose", "exec", "python", "bash", "-c",
+                    "cd /python-client && pip3 install -r requirements.txt && pytest"], check=True)
+
+
+def check_java():
+    print("--- java client test")
+    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "apt-get update && apt-get install -y maven"],
+                   check=True)
+    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "cd /java-client && mvn clean test"], check=True)
+
+
+subprocess.run(["docker", "compose", "up", "-d"], check=True)
+sleep(10)
+
+failed_cases = []
+
+try:
+    check_go()
+except Exception as e:
+    print(e)
+    failed_cases.append("go client failed")
+
+try:
+    check_python()
+except Exception as e:
+    print(e)
+    failed_cases.append("python client failed")
+
+try:
+    check_java()
+except Exception as e:
+    print(e)
+    failed_cases.append("java client failed")
+
+if len(failed_cases) != 0:
+    print(f"--- client check failed for case\n{failed_cases}")
+    sys.exit(1)
+
+subprocess.run(["docker", "compose", "down", "--remove-orphans", "-v"], check=True)