chore(ci): fix integration test docker clean (#16533)

ci/scripts/integration-tests.sh

@@ -22,14 +22,21 @@ while getopts 'c:f:' opt; do
 done
 shift $((OPTIND -1))
 
+cleanup() {
+  echo "--- clean Demos"
+  python3 integration_tests/scripts/clean_demos.py --case "${case}"
+}
+
+trap cleanup EXIT
+
 echo "export INTEGRATION_TEST_CASE=${case}" > env_vars.sh
 
 echo "~~~ clean up docker"
 if [ $(docker ps -aq |wc -l) -gt 0 ]; then
   docker rm -f $(docker ps -aq)
 fi
 docker network prune -f
-docker volume prune -f
+docker volume prune -f -a
 
 echo "~~~ ghcr login"
 echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
@@ -54,8 +61,7 @@ fi
 echo "--- case: ${case}, format: ${format}"
 
 if [ "${case}" == "client-library" ]; then
-  cd integration_tests/client-library
-  python3 client_test.py
+  python3 integration_tests/client-library/client_test.py
   exit 0
 fi
 
@@ -70,30 +76,25 @@ export PATH=$PATH:$HOME/.local/bin
 echo "--- download rwctest-key"
 aws secretsmanager get-secret-value --secret-id "gcp-buildkite-rwctest-key" --region us-east-2 --query "SecretString" --output text >gcp-rwctest.json
 
-cd integration_tests/scripts
-
 echo "--- rewrite docker compose for protobuf"
 if [ "${format}" == "protobuf" ]; then
-  python3 gen_pb_compose.py "${case}" "${format}"
+  python3 integration_tests/scripts/gen_pb_compose.py "${case}" "${format}"
 fi
 
 echo "--- set vm.max_map_count=2000000 for doris"
 max_map_count_original_value=$(sysctl -n vm.max_map_count)
 sudo sysctl -w vm.max_map_count=2000000
 
 echo "--- run Demos"
-python3 run_demos.py --case "${case}" --format "${format}"
+python3 integration_tests/scripts/run_demos.py --case "${case}" --format "${format}"
 
 echo "--- run docker ps"
 docker ps
 
 echo "--- check if the ingestion is successful"
 # extract the type of upstream source,e.g. mysql,postgres,etc
 upstream=$(echo "${case}" | cut -d'-' -f 1)
-python3 check_data.py "${case}" "${upstream}"
-
-echo "--- clean Demos"
-python3 clean_demos.py --case "${case}"
+python3 integration_tests/scripts/check_data.py "${case}" "${upstream}"
 
 echo "--- reset vm.max_map_count={$max_map_count_original_value}"
 sudo sysctl -w vm.max_map_count="$max_map_count_original_value"

integration_tests/client-library/client_test.py

@@ -4,42 +4,45 @@
 
 
 def check_go():
-    subprocess.run(["docker", "compose", "exec", "go-lang", "bash", "-c", "cd /go-client && ./run.sh"], check=True)
+    subprocess.run(["docker", "compose", "exec", "go-lang", "bash", "-c", "cd /go-client && ./run.sh"],
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_python():
-    subprocess.run(["docker", "compose", "exec", "python", "bash", "-c",
-                    "cd /python-client && pip3 install -r requirements.txt && pytest"], check=True)
+    subprocess.run(["docker", "compose", "exec", "python", "bash", "-c", "cd /python-client && pip3 install -r requirements.txt && pytest"],
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_java():
     subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "apt-get update && apt-get install -y maven"],
-                   check=True)
-    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "cd /java-client && mvn clean test"], check=True)
+                   cwd="integration_tests/client-library", check=True)
+    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "cd /java-client && mvn clean test"],
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_nodejs():
     subprocess.run(
         ["docker", "compose", "exec", "nodejs", "bash", "-c", "cd /nodejs-client && npm install && npm test"],
-        check=True)
+        cwd="integration_tests/client-library", check=True)
 
 
 def check_php():
     subprocess.run(
         ["docker", "compose", "exec", "php", "bash", "-c", "cd /php-client && phpunit tests/RWClientTest.php"],
-        check=True)
+        cwd="integration_tests/client-library", check=True)
 
 
 def check_ruby():
     subprocess.run(["docker", "compose", "exec", "ruby", "bash", "-c", "cd /ruby-client && ruby test/crud_test.rb"],
-                   check=True)
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_spring_boot():
-    subprocess.run(["docker", "compose", "exec", "spring-boot", "bash", "-c",
-                    "cd /spring-boot-client && mvn spring-boot:run"], check=True)
+    subprocess.run(["docker", "compose", "exec", "spring-boot", "bash", "-c", "cd /spring-boot-client && mvn spring-boot:run"],
+                   cwd="integration_tests/client-library", check=True)
 
-subprocess.run(["docker", "compose", "up", "-d"], check=True)
+subprocess.run(["docker", "compose", "up", "-d"],
+               cwd="integration_tests/client-library", check=True)
 sleep(10)
 
 failed_cases = []
@@ -68,6 +71,3 @@ def check_spring_boot():
 if len(failed_cases) != 0:
     print(f"--- client check failed for case\n{failed_cases}")
     sys.exit(1)
-
-print("--- docker compose down")
-subprocess.run(["docker", "compose", "down", "--remove-orphans", "-v"], check=True)