diff --git a/integration_tests/iceberg-cdc/docker-compose.yaml b/integration_tests/iceberg-cdc/docker-compose.yml similarity index 93% rename from integration_tests/iceberg-cdc/docker-compose.yaml rename to integration_tests/iceberg-cdc/docker-compose.yml index 8e9ad1062ef38..a41441eb7f0a5 100644 --- a/integration_tests/iceberg-cdc/docker-compose.yaml +++ b/integration_tests/iceberg-cdc/docker-compose.yml @@ -79,6 +79,15 @@ services: - "./mysql_prepare.sql:/mysql_prepare.sql" container_name: prepare_mysql restart: on-failure + datagen: + build: ../datagen + depends_on: [mysql] + command: + - /bin/sh + - -c + - /datagen --mode clickstream --qps 1 mysql --user mysqluser --password mysqlpw --host mysql --port 3306 --db mydb + container_name: datagen + restart: on-failure rest: image: tabulario/iceberg-rest:0.6.0 diff --git a/integration_tests/iceberg-cdc/python/Dockerfile b/integration_tests/iceberg-cdc/python/Dockerfile new file mode 100644 index 0000000000000..b0e395ada44ed --- /dev/null +++ b/integration_tests/iceberg-cdc/python/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.8 + +RUN curl -sSL https://install.python-poetry.org | python3 - + +WORKDIR /app + +COPY script /app + +RUN $HOME/.local/bin/poetry update + +CMD $HOME/.local/bin/poetry run python init.py;sleep 40;$HOME/.local/bin/poetry run python check.py + diff --git a/integration_tests/iceberg-cdc/python/check.py b/integration_tests/iceberg-cdc/python/script/check.py similarity index 100% rename from integration_tests/iceberg-cdc/python/check.py rename to integration_tests/iceberg-cdc/python/script/check.py diff --git a/integration_tests/iceberg-cdc/python/config.ini b/integration_tests/iceberg-cdc/python/script/config.ini similarity index 53% rename from integration_tests/iceberg-cdc/python/config.ini rename to integration_tests/iceberg-cdc/python/script/config.ini index bd95eddc5b80e..d9c7331c35da8 100644 --- a/integration_tests/iceberg-cdc/python/config.ini +++ b/integration_tests/iceberg-cdc/python/script/config.ini @@ -1,8 +1,8 @@ [spark] -url=sc://localhost:15002 +url=sc://spark:15002 [risingwave] db=dev user=root -host=127.0.0.1 +host=frontend-node-0 port=4566 diff --git a/integration_tests/iceberg-cdc/python/init.py b/integration_tests/iceberg-cdc/python/script/init.py similarity index 100% rename from integration_tests/iceberg-cdc/python/init.py rename to integration_tests/iceberg-cdc/python/script/init.py diff --git a/integration_tests/iceberg-cdc/python/pyproject.toml b/integration_tests/iceberg-cdc/python/script/pyproject.toml similarity index 100% rename from integration_tests/iceberg-cdc/python/pyproject.toml rename to integration_tests/iceberg-cdc/python/script/pyproject.toml diff --git a/integration_tests/iceberg-cdc/run_test.sh b/integration_tests/iceberg-cdc/run_test.sh index 2d8b691bc7284..6e47d6ad59be4 100755 --- a/integration_tests/iceberg-cdc/run_test.sh +++ b/integration_tests/iceberg-cdc/run_test.sh @@ -6,14 +6,8 @@ docker-compose up -d --wait # To avoid exiting by unhealth, set it after start environment. set -ex -# Generate data -docker build -t iceberg-cdc-datagen ../datagen -timeout 20 docker run --network=iceberg-cdc_default iceberg-cdc-datagen /datagen --mode clickstream --qps 1 mysql --user mysqluser --password mysqlpw --host mysql --port 3306 --db mydb & +sleep 20 +docker stop datagen -cd python -poetry update --quiet -# Init source, mv, and sink. -poetry run python init.py -# Wait for sink to be finished. -sleep 40; -poetry run python check.py +docker build -t iceberg-cdc-python python +docker run --network=iceberg-cdc_default iceberg-cdc-python diff --git a/integration_tests/scripts/check_data.py b/integration_tests/scripts/check_data.py index 52cc79dc0ab0a..37679fbdacdae 100644 --- a/integration_tests/scripts/check_data.py +++ b/integration_tests/scripts/check_data.py @@ -54,7 +54,7 @@ def run_psql(sql): demo = sys.argv[1] upstream = sys.argv[2] # mysql, postgres, etc. see scripts/integration_tests.sh -if demo in ['docker', 'iceberg-sink','clickhouse-sink']: +if demo in ['docker', 'iceberg-sink','clickhouse-sink', 'iceberg-cdc']: print('Skip for running test for `%s`' % demo) sys.exit(0)