Merge

.github/workflows/build.yml

@@ -0,0 +1,312 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Python Release Build
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    tags: ["*-rc*"]
+    branches: ["branch-*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff
+      # Update output format to enable automatic inline annotations.
+#      - name: Run Ruff
+#        run: |
+#          ruff check --output-format=github python/
+#          ruff format --check python/
+
+  generate-license:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Generate license file
+        run: python ./dev/create_license.py
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-wheel-license
+          path: LICENSE.txt
+
+  build-python-mac-win:
+    needs: [generate-license]
+    name: Mac/Win
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        os: [macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Install maturin
+        run: pip install maturin==1.5.1
+
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cargo build
+        run: cd python && cargo build 
+
+      - name: Build Python package
+        run: cd python && maturin build --release --strip
+
+      - name: List Windows wheels
+        if: matrix.os == 'windows-latest'
+        run: dir python\target\wheels\
+        # since the runner is dynamic shellcheck (from actionlint) can't infer this is powershell
+        # so we specify it explicitly
+        shell: powershell
+
+      - name: List Mac wheels
+        if: matrix.os != 'windows-latest'
+        run: cd python/target/wheels/
+
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-${{ matrix.os  }}
+          path: python/target/wheels/*
+
+  build-macos-x86_64:
+    needs: [generate-license]
+    name: Mac x86_64
+    runs-on: macos-13
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Install maturin
+        run: pip install maturin==1.5.1
+
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Cargo build
+        run: cd python && cargo build
+      - name: Build Python package
+        run: cd python && maturin build --release --strip
+      - name: List Mac wheels
+        run: cd python/target/wheels/
+
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-macos-aarch64
+          path: python/target/wheels/*
+
+  build-manylinux-x86_64:
+    needs: [generate-license]
+    name: Manylinux x86_64
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rm LICENSE.txt
+
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - run: cat LICENSE.txt
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Install maturin
+        run: pip install maturin==1.5.1
+      - name: Cargo Build
+        run: cd python && cargo build
+
+      - name: Build Python package
+        run: cd python && maturin build --release --strip
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-manylinux-x86_64
+          path: python/target/wheels/*
+
+  build-manylinux-aarch64:
+    needs: [generate-license]
+    name: Manylinux arm64
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - run: cat LICENSE.txt
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Install maturin
+        run: pip install maturin==1.5.1
+      - name: Cargo Build
+        run: cd python && cargo build
+
+      - name: Build Python package
+        run: cd python && maturin build --release --strip
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-manylinux-aarch64
+          path: python/target/wheels/*
+
+  build-sdist:
+    needs: [generate-license]
+    name: Source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - run: cat LICENSE.txt
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Install maturin
+        run: pip install maturin==1.5.1
+      - name: Cargo Build
+        run: cd python && cargo build
+      - name: Build Python package
+        run: cd python && maturin build --release --sdist --out dist --strip
+
+      - name: Assert sdist build does not generate wheels
+        run: |
+            if [ "$(ls -A target/wheels)" ]; then
+              echo "Error: Sdist build generated wheels"
+              exit 1
+            else
+              echo "Directory is clean"
+            fi
+        shell: bash
+
+  merge-build-artifacts:
+    runs-on: ubuntu-latest
+    needs:
+      - build-python-mac-win
+      - build-macos-x86_64
+      - build-manylinux-x86_64
+      - build-manylinux-aarch64
+      - build-sdist
+    steps:
+      - name: Merge Build Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: dist
+          pattern: dist-*
+
+  # NOTE: PyPI publish needs to be done manually for now after release passed the vote
+  # release:
+  #   name: Publish in PyPI
+  #   needs: [build-manylinux, build-python-mac-win]
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/download-artifact@v4
+  #     - name: Publish to PyPI
+  #       uses: pypa/gh-action-pypi-publish@master
+  #       with:
+  #         user: __token__
+  #         password: ${{ secrets.pypi_password }}

.github/workflows/rust.yml

@@ -117,10 +117,8 @@ jobs:
           export PATH=$PATH:$HOME/d/protoc/bin
           export ARROW_TEST_DATA=$(pwd)/testing/data
           export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
-          cargo test
-          cd examples
-          cargo run --example standalone_sql --features=ballista/standalone
-          cd ../python
+          cargo test --features=testcontainers
+          cd python
           cargo test
         env:
           CARGO_HOME: "/github/home/.cargo"

Cargo.toml

@@ -23,7 +23,7 @@ resolver = "2"
 [workspace.dependencies]
 arrow = { version = "53", features = ["ipc_compression"] }
 arrow-flight = { version = "53", features = ["flight-sql-experimental"] }
-clap = { version = "3", features = ["derive", "cargo"] }
+clap = { version = "4.5", features = ["derive", "cargo"] }
 configure_me = { version = "0.4.0" }
 configure_me_codegen = { version = "0.4.4" }
 datafusion = "43.0.0"
@@ -38,9 +38,11 @@ tonic-build = { version = "0.12", default-features = false, features = [
     "transport",
     "prost"
 ] }
-tracing = "0.1.36"
+tracing = "0.1"
 tracing-appender = "0.2.2"
-tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+ctor = { version = "0.2" }
+mimalloc = { version = "0.1" }
 
 tokio = { version = "1" }
 uuid = { version = "1.10", features = ["v4", "v7"] }
@@ -54,7 +56,6 @@ dashmap = { version = "6.1" }
 async-trait = { version = "0.1.4" }
 serde = { version = "1.0" }
 tokio-stream = { version = "0.1" }
-parse_arg = { version = "0.1" }
 url = { version = "2.5" }
 
 # cargo build --profile release-lto

ballista-cli/Cargo.toml

@@ -25,18 +25,16 @@ keywords = ["ballista", "cli"]
 license = "Apache-2.0"
 homepage = "https://github.com/apache/arrow-ballista"
 repository = "https://github.com/apache/arrow-ballista"
-rust-version = "1.72"
 readme = "README.md"
 
 [dependencies]
 ballista = { path = "../ballista/client", version = "0.12.0", features = ["standalone"] }
-# datafusion-cli uses 4.5 clap, thus it does not depend on workspace
-clap = { version = "4.5", features = ["derive", "cargo"] }
+clap = { workspace = true, features = ["derive", "cargo"] }
 datafusion = { workspace = true }
 datafusion-cli = { workspace = true }
 dirs = "5.0.1"
 env_logger = { workspace = true }
-mimalloc = { version = "0.1", default-features = false }
+mimalloc = { workspace = true }
 rustyline = "14.0.0"
 tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }