Merge feature into main

.github/regression/tpcds_sf1_local.csv

@@ -0,0 +1,100 @@
+benchmark/tpcds/sf1/local/delta/q01.benchmark
+benchmark/tpcds/sf1/local/delta/q02.benchmark
+benchmark/tpcds/sf1/local/delta/q03.benchmark
+benchmark/tpcds/sf1/local/delta/q04.benchmark
+benchmark/tpcds/sf1/local/delta/q05.benchmark
+benchmark/tpcds/sf1/local/delta/q06.benchmark
+benchmark/tpcds/sf1/local/delta/q07.benchmark
+benchmark/tpcds/sf1/local/delta/q08.benchmark
+benchmark/tpcds/sf1/local/delta/q09.benchmark
+benchmark/tpcds/sf1/local/delta/q10.benchmark
+benchmark/tpcds/sf1/local/delta/q11.benchmark
+benchmark/tpcds/sf1/local/delta/q12.benchmark
+benchmark/tpcds/sf1/local/delta/q13.benchmark
+benchmark/tpcds/sf1/local/delta/q14.benchmark
+benchmark/tpcds/sf1/local/delta/q15.benchmark
+benchmark/tpcds/sf1/local/delta/q16.benchmark
+benchmark/tpcds/sf1/local/delta/q17.benchmark
+benchmark/tpcds/sf1/local/delta/q18.benchmark
+benchmark/tpcds/sf1/local/delta/q19.benchmark
+benchmark/tpcds/sf1/local/delta/q20.benchmark
+benchmark/tpcds/sf1/local/delta/q21.benchmark
+benchmark/tpcds/sf1/local/delta/q22.benchmark
+benchmark/tpcds/sf1/local/delta/q23.benchmark
+benchmark/tpcds/sf1/local/delta/q24.benchmark
+benchmark/tpcds/sf1/local/delta/q25.benchmark
+benchmark/tpcds/sf1/local/delta/q26.benchmark
+benchmark/tpcds/sf1/local/delta/q27.benchmark
+benchmark/tpcds/sf1/local/delta/q28.benchmark
+
+benchmark/tpcds/sf1/local/delta/q29.benchmark
+benchmark/tpcds/sf1/local/delta/q30.benchmark
+benchmark/tpcds/sf1/local/delta/q31.benchmark
+benchmark/tpcds/sf1/local/delta/q32.benchmark
+benchmark/tpcds/sf1/local/delta/q33.benchmark
+benchmark/tpcds/sf1/local/delta/q34.benchmark
+benchmark/tpcds/sf1/local/delta/q35.benchmark
+benchmark/tpcds/sf1/local/delta/q36.benchmark
+benchmark/tpcds/sf1/local/delta/q37.benchmark
+benchmark/tpcds/sf1/local/delta/q38.benchmark
+benchmark/tpcds/sf1/local/delta/q39.benchmark
+benchmark/tpcds/sf1/local/delta/q40.benchmark
+benchmark/tpcds/sf1/local/delta/q41.benchmark
+benchmark/tpcds/sf1/local/delta/q42.benchmark
+benchmark/tpcds/sf1/local/delta/q43.benchmark
+benchmark/tpcds/sf1/local/delta/q44.benchmark
+benchmark/tpcds/sf1/local/delta/q45.benchmark
+benchmark/tpcds/sf1/local/delta/q46.benchmark
+benchmark/tpcds/sf1/local/delta/q47.benchmark
+benchmark/tpcds/sf1/local/delta/q48.benchmark
+benchmark/tpcds/sf1/local/delta/q49.benchmark
+benchmark/tpcds/sf1/local/delta/q50.benchmark
+benchmark/tpcds/sf1/local/delta/q51.benchmark
+benchmark/tpcds/sf1/local/delta/q52.benchmark
+benchmark/tpcds/sf1/local/delta/q53.benchmark
+benchmark/tpcds/sf1/local/delta/q54.benchmark
+benchmark/tpcds/sf1/local/delta/q55.benchmark
+benchmark/tpcds/sf1/local/delta/q56.benchmark
+benchmark/tpcds/sf1/local/delta/q57.benchmark
+benchmark/tpcds/sf1/local/delta/q58.benchmark
+benchmark/tpcds/sf1/local/delta/q59.benchmark
+benchmark/tpcds/sf1/local/delta/q60.benchmark
+benchmark/tpcds/sf1/local/delta/q61.benchmark
+benchmark/tpcds/sf1/local/delta/q62.benchmark
+benchmark/tpcds/sf1/local/delta/q63.benchmark
+benchmark/tpcds/sf1/local/delta/q64.benchmark
+benchmark/tpcds/sf1/local/delta/q65.benchmark
+benchmark/tpcds/sf1/local/delta/q66.benchmark
+benchmark/tpcds/sf1/local/delta/q67.benchmark
+benchmark/tpcds/sf1/local/delta/q68.benchmark
+benchmark/tpcds/sf1/local/delta/q69.benchmark
+benchmark/tpcds/sf1/local/delta/q70.benchmark
+benchmark/tpcds/sf1/local/delta/q71.benchmark
+benchmark/tpcds/sf1/local/delta/q72.benchmark
+benchmark/tpcds/sf1/local/delta/q73.benchmark
+benchmark/tpcds/sf1/local/delta/q74.benchmark
+benchmark/tpcds/sf1/local/delta/q75.benchmark
+benchmark/tpcds/sf1/local/delta/q76.benchmark
+benchmark/tpcds/sf1/local/delta/q77.benchmark
+benchmark/tpcds/sf1/local/delta/q78.benchmark
+benchmark/tpcds/sf1/local/delta/q79.benchmark
+benchmark/tpcds/sf1/local/delta/q80.benchmark
+benchmark/tpcds/sf1/local/delta/q81.benchmark
+benchmark/tpcds/sf1/local/delta/q82.benchmark
+benchmark/tpcds/sf1/local/delta/q83.benchmark
+benchmark/tpcds/sf1/local/delta/q84.benchmark
+benchmark/tpcds/sf1/local/delta/q85.benchmark
+benchmark/tpcds/sf1/local/delta/q86.benchmark
+benchmark/tpcds/sf1/local/delta/q87.benchmark
+benchmark/tpcds/sf1/local/delta/q88.benchmark
+benchmark/tpcds/sf1/local/delta/q89.benchmark
+benchmark/tpcds/sf1/local/delta/q90.benchmark
+benchmark/tpcds/sf1/local/delta/q91.benchmark
+benchmark/tpcds/sf1/local/delta/q92.benchmark
+benchmark/tpcds/sf1/local/delta/q93.benchmark
+benchmark/tpcds/sf1/local/delta/q94.benchmark
+benchmark/tpcds/sf1/local/delta/q95.benchmark
+benchmark/tpcds/sf1/local/delta/q96.benchmark
+benchmark/tpcds/sf1/local/delta/q97.benchmark
+benchmark/tpcds/sf1/local/delta/q98.benchmark
+benchmark/tpcds/sf1/local/delta/q99.benchmark

.github/regression/tpch_sf1_local.csv

@@ -0,0 +1,22 @@
+benchmark/tpch/sf1/local/delta/q01.benchmark
+benchmark/tpch/sf1/local/delta/q02.benchmark
+benchmark/tpch/sf1/local/delta/q03.benchmark
+benchmark/tpch/sf1/local/delta/q04.benchmark
+benchmark/tpch/sf1/local/delta/q05.benchmark
+benchmark/tpch/sf1/local/delta/q06.benchmark
+benchmark/tpch/sf1/local/delta/q07.benchmark
+benchmark/tpch/sf1/local/delta/q08.benchmark
+benchmark/tpch/sf1/local/delta/q09.benchmark
+benchmark/tpch/sf1/local/delta/q10.benchmark
+benchmark/tpch/sf1/local/delta/q11.benchmark
+benchmark/tpch/sf1/local/delta/q12.benchmark
+benchmark/tpch/sf1/local/delta/q13.benchmark
+benchmark/tpch/sf1/local/delta/q14.benchmark
+benchmark/tpch/sf1/local/delta/q15.benchmark
+benchmark/tpch/sf1/local/delta/q16.benchmark
+benchmark/tpch/sf1/local/delta/q17.benchmark
+benchmark/tpch/sf1/local/delta/q18.benchmark
+benchmark/tpch/sf1/local/delta/q19.benchmark
+benchmark/tpch/sf1/local/delta/q20.benchmark
+benchmark/tpch/sf1/local/delta/q21.benchmark
+benchmark/tpch/sf1/local/delta/q22.benchmark

.github/workflows/CloudTesting.yml

@@ -41,6 +41,12 @@ jobs:
         with:
           vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
 
+      - name: Configure OpenSSL for Rust
+        run: |
+          echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@stable
 
@@ -57,15 +63,15 @@ jobs:
           AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}
           AZURE_STORAGE_ACCOUNT: ${{secrets.AZURE_STORAGE_ACCOUNT}}
         run: |
-          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest "*test/sql/cloud/*"
+          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest `pwd`/test/sql/cloud/*
 
       - name: Test with SPN logged in in azure-cli
         env:
           AZURE_STORAGE_ACCOUNT: ${{secrets.AZURE_STORAGE_ACCOUNT}}
           DUCKDB_AZ_CLI_LOGGED_IN: 1
         run: |
           az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -p ${{secrets.AZURE_CLIENT_SECRET}} --tenant ${{secrets.AZURE_TENANT_ID}}
-          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest "*test/sql/cloud/*"
+          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest `pwd`/test/sql/cloud/*
 
       - name: Log out azure-cli
         if: always()
@@ -77,4 +83,4 @@ jobs:
           AZURE_STORAGE_ACCOUNT: ${{secrets.AZURE_STORAGE_ACCOUNT}}
           DUCKDB_AZURE_PUBLIC_CONTAINER_AVAILABLE: 1
         run: |
-          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest "*test/sql/cloud/*"
+          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest `pwd`/test/sql/cloud/*

.github/workflows/LocalTesting.yml

@@ -7,6 +7,9 @@ defaults:
   run:
     shell: bash
 
+env:
+  BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}
+
 jobs:
   azurite-tests-linux:
     name: Azurite (local azure test server) tests (Linux)
@@ -117,6 +120,12 @@ jobs:
         with:
           vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
 
+      - name: Configure OpenSSL for Rust
+        run: |
+          echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
       - name: Build
         shell: bash
         run: make
@@ -194,11 +203,94 @@ jobs:
         with:
           vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
 
+      - name: Configure OpenSSL for Rust
+        run: |
+          echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
       - name: Build
         shell: bash
         run: make generate-data
 
       - name: Test
         shell: bash
         run: |
-          GENERATED_DATA_AVAILABLE=1 make test
+          GENERATED_DATA_AVAILABLE=1 make test
+
+  regression-test-benchmark-runner:
+    name: Performance Regression Tests
+    runs-on: ubuntu-latest
+    env:
+      GEN: ninja
+      BUILD_BENCHMARK: 1
+      VCPKG_TARGET_TRIPLET: x64-linux
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+
+      - name: Setup vcpkg
+        uses: lukka/[email protected]
+        with:
+          vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+      - name: Configure OpenSSL for Rust
+        run: |
+          echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
+      - name: Build current release
+        shell: bash
+        run: |
+          make
+          rm -rf build/release/rust
+
+      - name: Build ${{ env.BASE_BRANCH }} branch
+        shell: bash
+        run: | 
+          git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb_delta.git --depth=1 
+          cd duckdb_delta
+          git submodule init
+          git submodule update
+          make
+          rm -rf build/release/rust
+
+      - name: Generate test data
+        shell: bash
+        run: make generate-data
+
+      - name: Regression Test TPC-H
+        if: always()
+        shell: bash
+        run: |
+          python3 ./scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch_sf1_local.csv --verbose --threads=2 --root-dir=.
+
+      - name: Regression Test TPC-DS
+        if: always()
+        shell: bash
+        run: |
+          python ./scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds_sf1_local.csv --verbose --threads=2 --root-dir=.
+
+      - name: Test benchmark makefile
+        shell: bash
+        run: |
+          make bench-run-tpch-sf1
+          make bench-run-tpcds-sf1

.github/workflows/MainDistributionPipeline.yml

@@ -14,20 +14,20 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.0.0
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.0
     with:
-      duckdb_version: v1.0.0
+      duckdb_version: v1.1.0
       extension_name: delta
       enable_rust: true
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
 
   duckdb-stable-deploy:
     name: Deploy extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.0.0
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.0
     secrets: inherit
     with:
       extension_name: delta
-      duckdb_version: v1.0.0
+      duckdb_version: v1.1.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
-      deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
+      deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

CMakeLists.txt

@@ -24,7 +24,7 @@
 elseif(UNIX)
     set(PLATFORM_LIBS m c resolv)
 elseif(WIN32)
-    set(PLATFORM_LIBS ntdll ncrypt secur32 ws2_32 userenv bcrypt msvcrt advapi32)
+    set(PLATFORM_LIBS ntdll ncrypt secur32 ws2_32 userenv bcrypt msvcrt advapi32 RuntimeObject)
 else()
     message(STATUS "UNKNOWN OS")
 endif()

Makefile

@@ -11,6 +11,9 @@ test_release: export DAT_PATH=./build/release/rust/src/delta_kernel/acceptance/t
 test_debug: export DELTA_KERNEL_TESTS_PATH=./build/debug/rust/src/delta_kernel/kernel/tests/data
 test_debug: export DAT_PATH=./build/debug/rust/src/delta_kernel/acceptance/tests/dat
 
+# Core extensions that we need for testing
+CORE_EXTENSIONS='tpcds;tpch;aws;azure;httpfs'
+
 # Set this flag during building to enable the benchmark runner
 ifeq (${BUILD_BENCHMARK}, 1)
 	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DBUILD_BENCHMARKS=1
@@ -24,5 +27,5 @@ include benchmark/benchmark.Makefile
 
 # Generate some test data to test with
 generate-data:
-	python3 -m pip install delta-spark duckdb pandas deltalake pyspark delta
+	python3 -m pip install delta-spark duckdb pandas deltalake pyspark
 	python3 scripts/generate_test_data.py

benchmark/README.md

@@ -29,13 +29,4 @@ Specific benchmarks can be run from a suite using the `BENCHMARK_PATTERN` variab
 only Q01 from TPCH SF1, run:
 ```shell
 BENCHMARK_PATTERN=q01.benchmark make bench-run-tpch-sf1
-```
-
-Also, we can run all local benchmarks using:
-```shell
-make  bench-run-all-local
-```
-Or all remote benchmarks using
-```shell
-make bench-run-all-remote
 ```