diff --git a/.github/regression/tpch_sf1_local.csv b/.github/regression/tpch_sf1_local.csv index f98745c..488f2a2 100644 --- a/.github/regression/tpch_sf1_local.csv +++ b/.github/regression/tpch_sf1_local.csv @@ -1,22 +1,22 @@ -benchmark/tpch/sf1/delta/q01.benchmark -benchmark/tpch/sf1/delta/q02.benchmark -benchmark/tpch/sf1/delta/q03.benchmark -benchmark/tpch/sf1/delta/q04.benchmark -benchmark/tpch/sf1/delta/q05.benchmark -benchmark/tpch/sf1/delta/q06.benchmark -benchmark/tpch/sf1/delta/q07.benchmark -benchmark/tpch/sf1/delta/q08.benchmark -benchmark/tpch/sf1/delta/q09.benchmark -benchmark/tpch/sf1/delta/q10.benchmark -benchmark/tpch/sf1/delta/q11.benchmark -benchmark/tpch/sf1/delta/q12.benchmark -benchmark/tpch/sf1/delta/q13.benchmark -benchmark/tpch/sf1/delta/q14.benchmark -benchmark/tpch/sf1/delta/q15.benchmark -benchmark/tpch/sf1/delta/q16.benchmark -benchmark/tpch/sf1/delta/q17.benchmark -benchmark/tpch/sf1/delta/q18.benchmark -benchmark/tpch/sf1/delta/q19.benchmark -benchmark/tpch/sf1/delta/q20.benchmark -benchmark/tpch/sf1/delta/q21.benchmark -benchmark/tpch/sf1/delta/q22.benchmark +benchmark/tpch/sf1/local/delta/q01.benchmark +benchmark/tpch/sf1/local/delta/q02.benchmark +benchmark/tpch/sf1/local/delta/q03.benchmark +benchmark/tpch/sf1/local/delta/q04.benchmark +benchmark/tpch/sf1/local/delta/q05.benchmark +benchmark/tpch/sf1/local/delta/q06.benchmark +benchmark/tpch/sf1/local/delta/q07.benchmark +benchmark/tpch/sf1/local/delta/q08.benchmark +benchmark/tpch/sf1/local/delta/q09.benchmark +benchmark/tpch/sf1/local/delta/q10.benchmark +benchmark/tpch/sf1/local/delta/q11.benchmark +benchmark/tpch/sf1/local/delta/q12.benchmark +benchmark/tpch/sf1/local/delta/q13.benchmark +benchmark/tpch/sf1/local/delta/q14.benchmark +benchmark/tpch/sf1/local/delta/q15.benchmark +benchmark/tpch/sf1/local/delta/q16.benchmark +benchmark/tpch/sf1/local/delta/q17.benchmark +benchmark/tpch/sf1/local/delta/q18.benchmark +benchmark/tpch/sf1/local/delta/q19.benchmark +benchmark/tpch/sf1/local/delta/q20.benchmark +benchmark/tpch/sf1/local/delta/q21.benchmark +benchmark/tpch/sf1/local/delta/q22.benchmark \ No newline at end of file diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index d0f7c23..19d419f 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -281,13 +281,13 @@ jobs: if: always() shell: bash run: | - python3 duckdb/scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch_sf1_local.csv --verbose --threads=2 --root-dir=. + python3 ./scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch_sf1_local.csv --verbose --threads=2 --root-dir=. - name: Regression Test TPC-DS if: always() shell: bash run: | - python duckdb/scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds_sf1_local.csv --verbose --threads=2 --root-dir=. + python ./scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds_sf1_local.csv --verbose --threads=2 --root-dir=. - name: Test benchmark makefile shell: bash diff --git a/Makefile b/Makefile index 9416f84..8cc8bc9 100644 --- a/Makefile +++ b/Makefile @@ -27,5 +27,5 @@ include benchmark/benchmark.Makefile # Generate some test data to test with generate-data: - python3 -m pip install delta-spark duckdb pandas deltalake pyspark delta + python3 -m pip install delta-spark duckdb pandas deltalake pyspark python3 scripts/generate_test_data.py diff --git a/scripts/generate_test_data.py b/scripts/generate_test_data.py index bf26430..b817faa 100644 --- a/scripts/generate_test_data.py +++ b/scripts/generate_test_data.py @@ -107,7 +107,8 @@ def generate_test_data_pyspark(name, current_path, input_path, delete_predicate builder = SparkSession.builder.appName("MyApp") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ - .config("spark.driver.memory", "8g") + .config("spark.driver.memory", "8g")\ + .config('spark.driver.host','127.0.0.1') spark = configure_spark_with_delta_pip(builder).getOrCreate()