Merge pull request #636 from rapidsai/branch-24.10

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ concurrency:
 jobs:
   python-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -37,7 +37,7 @@ jobs:
   upload-conda:
     needs: [python-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -47,7 +47,7 @@ jobs:
     if: github.ref_type == 'branch'
     needs: python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
     with:
       arch: "amd64"
       branch: ${{ inputs.branch }}
@@ -59,7 +59,7 @@ jobs:
       sha: ${{ inputs.sha }}
   wheel-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -71,7 +71,7 @@ jobs:
   wheel-publish:
     needs: wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}

.github/workflows/pr.yaml

@@ -20,27 +20,27 @@ jobs:
       - wheel-build
       - wheel-tests
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
   checks:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10
   conda-python-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
     with:
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
     with:
       build_type: pull-request
       run_codecov: false
   conda-notebook-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -50,7 +50,7 @@ jobs:
   docs-build:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -60,7 +60,7 @@ jobs:
   wheel-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
     with:
       build_type: pull-request
       script: ci/build_wheel.sh
@@ -69,7 +69,7 @@ jobs:
   wheel-tests:
     needs: wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10
     with:
       build_type: pull-request
       script: ci/test_wheel.sh

.github/workflows/test-external.yaml

@@ -23,7 +23,7 @@ on:
 jobs:
   test-external:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
     with:
       build_type: branch
       node_type: "gpu-v100-latest-1"

.github/workflows/test.yaml

@@ -16,15 +16,15 @@ on:
 jobs:
   conda-python-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}

.pre-commit-config.yaml

@@ -9,13 +9,13 @@ repos:
                 # Explicitly specify the pyproject.toml at the repo root, not per-project.
                 args: ["--config", "pyproject.toml"]
       - repo: https://github.com/PyCQA/flake8
-        rev: 6.0.0
+        rev: 7.1.1
         hooks:
               - id: flake8
                 args: ["--config=.flake8"]
                 files: python/.*$
       - repo: https://github.com/rapidsai/pre-commit-hooks
-        rev: v0.2.0
+        rev: v0.4.0
         hooks:
             - id: verify-alpha-spec
               args: ["--fix", "--mode=release"]

CHANGELOG.md

@@ -1,3 +1,29 @@
+# cuxfilter 24.10.00 (9 Oct 2024)
+
+## 🐛 Bug Fixes
+
+- Fix/external tests ([#621](https://github.com/rapidsai/cuxfilter/pull/621)) [@AjayThorve](https://github.com/AjayThorve)
+- Fix tests ([#620](https://github.com/rapidsai/cuxfilter/pull/620)) [@AjayThorve](https://github.com/AjayThorve)
+
+## 📖 Documentation
+
+- Recommend `miniforge` for conda installation ([#626](https://github.com/rapidsai/cuxfilter/pull/626)) [@bdice](https://github.com/bdice)
+- Branch 24.10 merge 24.08 ([#611](https://github.com/rapidsai/cuxfilter/pull/611)) [@jameslamb](https://github.com/jameslamb)
+
+## 🛠️ Improvements
+
+- Switch `traceback` to `native` in `pytest` ([#633](https://github.com/rapidsai/cuxfilter/pull/633)) [@galipremsagar](https://github.com/galipremsagar)
+- Update update-version.sh to use packaging lib ([#631](https://github.com/rapidsai/cuxfilter/pull/631)) [@AyodeAwe](https://github.com/AyodeAwe)
+- Use CI workflow branch 'branch-24.10' again ([#629](https://github.com/rapidsai/cuxfilter/pull/629)) [@jameslamb](https://github.com/jameslamb)
+- Update to flake8 7.1.1. ([#627](https://github.com/rapidsai/cuxfilter/pull/627)) [@bdice](https://github.com/bdice)
+- Add support for Python 3.12, drop pyppeteer dependency ([#625](https://github.com/rapidsai/cuxfilter/pull/625)) [@jameslamb](https://github.com/jameslamb)
+- Update rapidsai/pre-commit-hooks ([#624](https://github.com/rapidsai/cuxfilter/pull/624)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+- Drop Python 3.9 support ([#623](https://github.com/rapidsai/cuxfilter/pull/623)) [@jameslamb](https://github.com/jameslamb)
+- Remove NumPy <2 pin, remove panel ceiling ([#622](https://github.com/rapidsai/cuxfilter/pull/622)) [@seberg](https://github.com/seberg)
+- [BUG] Fix `cudf.pandas` integration issues with `cuxfilter` ([#619](https://github.com/rapidsai/cuxfilter/pull/619)) [@Matt711](https://github.com/Matt711)
+- Update pre-commit hooks ([#617](https://github.com/rapidsai/cuxfilter/pull/617)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+- Improve update-version.sh ([#616](https://github.com/rapidsai/cuxfilter/pull/616)) [@bdice](https://github.com/bdice)
+
 # cuxfilter 24.08.00 (7 Aug 2024)
 
 ## 🛠️ Improvements

README.md

@@ -134,7 +134,6 @@ Troubleshooting help can be found [on our troubleshooting page](https://docs.rap
 - bokeh
 - pyproj
 - geopandas
-- pyppeteer
 - jupyter-server-proxy
 
 ## Quick Start
@@ -146,38 +145,39 @@ Please see the [Demo Docker Repository](https://hub.docker.com/r/rapidsai/rapids
 ### CUDA/GPU requirements
 
 - CUDA 11.2+
-- NVIDIA driver 450.80.02+
-- Pascal architecture or better (Compute Capability >=6.0)
+- Volta architecture or newer (Compute Capability >=7.0)
 
 ### Conda
 
-cuxfilter can be installed with conda ([miniconda](https://conda.io/miniconda.html), or the full [Anaconda distribution](https://www.anaconda.com/download)) from the `rapidsai` channel:
+cuxfilter can be installed with conda. You can get a minimal conda installation with [miniforge](https://github.com/conda-forge/miniforge).
 
-For nightly version `cuxfilter version == 24.08` :
+For the nightly version of `cuxfilter`:
 
 ```bash
-# for CUDA 12.0
+# for CUDA 12.5
 conda install -c rapidsai-nightly -c conda-forge -c nvidia \
-    cuxfilter=24.08 python=3.11 cuda-version=12.0
+    cuxfilter=24.10 python=3.12 cuda-version=12.5
 
 # for CUDA 11.8
 conda install -c rapidsai-nightly -c conda-forge -c nvidia \
-    cuxfilter=24.08 python=3.11 cuda-version=11.8
+    cuxfilter=24.10 python=3.12 cuda-version=11.8
 ```
 
-For the stable version of `cuxfilter` :
+For the stable version of `cuxfilter`:
 
 ```bash
-# for CUDA 12.0
+# for CUDA 12.5
 conda install -c rapidsai -c conda-forge -c nvidia \
-    cuxfilter python=3.11 cuda-version=12.0
+    cuxfilter python=3.12 cuda-version=12.5
 
 # for CUDA 11.8
 conda install -c rapidsai -c conda-forge -c nvidia \
-    cuxfilter python=3.11 cuda-version=11.8
+    cuxfilter python=3.12 cuda-version=11.8
 ```
 
-Note: cuxfilter is supported only on Linux, and with Python versions 3.8 and later.
+Note: cuxfilter is supported only on Linux, and with Python versions 3.10, 3.11, and 3.12.
+
+> Above are sample install snippets for cuxfilter, see the [Get RAPIDS version picker](https://rapids.ai/start.html) for installing the latest `cuxfilter` version.
 
 ### PyPI
 

VERSION

@@ -1 +1 @@
-24.08.00
+24.10.00

ci/release/update-version.sh

@@ -18,13 +18,13 @@ CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}')
 CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}')
 CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}
 
-#Get <major>.<minor> for next version
+# Get <major>.<minor> for next version
 NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}')
 NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
 NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
 
 # Need to distutils-normalize the original version
-NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
+NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))")
 
 echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG"
 
@@ -39,7 +39,6 @@ sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_S
 # Centralized version file update
 echo "${NEXT_FULL_TAG}" > VERSION
 
-
 DEPENDENCIES=(
   cudf
   cuxfilter
@@ -61,8 +60,7 @@ done
 sed_runner "/version == / s/== .*\`/== ${NEXT_SHORT_TAG}\`/g" README.md
 sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_SHORT_TAG}/g" README.md
 
-
 # CI files
 for FILE in .github/workflows/*.yaml; do
-  sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" ${FILE};
+  sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
 done

ci/test_external.sh

@@ -7,18 +7,37 @@ rapids-logger "Create test_external conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
 # Install external dependencies into test_external conda environment
-rapids-conda-retry env update -f ./ci/utils/external_dependencies.yaml
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key test_external \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
+rapids-mamba-retry env create --yes -f env.yaml -n test_external
+
+# Temporarily allow unbound variables for conda activation.
+set +u
 conda activate test_external
+set -u
+
+rapids-print-env
 
 # Define input parameter
-PROJECT=$1
-PR_NUMBER=$2
+PROJECT="${1:-all}"
+PR_NUMBER="${2:-0}"
 LIBRARIES=("datashader" "holoviews")
 
+# set conda env variable NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS to 0
+# to suppress low occupancy warnings
+export NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS=0
+
 # Change directory to /tmp
 pushd /tmp
 
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
 # Clone the specified Python libraries
 if [ "$PROJECT" = "all" ]; then
     # Loop through each library and install dependencies
@@ -33,7 +52,11 @@ if [ "$PROJECT" = "all" ]; then
         # Change directory to the library
         pushd $LIBRARY
         # Run setup.py with test dependencies
-        python -m pip install -e .[tests]
+        python -m pip install .[tests]
+
+        rapids-logger "Run GPU tests for $LIBRARY"
+
+        python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu
 
         popd
     done
@@ -42,7 +65,7 @@ else
     git clone https://github.com/pyviz/$PROJECT.git
 
     # Check if PR_NUMBER is a non-empty, valid number
-    if [ -n "$PR_NUMBER" ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then
+    if [ "$PR_NUMBER" -ne 0 ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then
         rapids-logger "checkout PR $PR_NUMBER"
         # Fetch the pull request and check it out
         git fetch origin pull/$PR_NUMBER/head:pr/$PR_NUMBER
@@ -53,42 +76,14 @@ else
     # Change directory to the specified project
     pushd $PROJECT
     # Run setup.py with test dependencies
-    python -m pip install -e .[tests]
-    popd
-fi
+    python -m pip install .[tests]
 
-FILES=""
-# Install and run tests
-if [ "$PROJECT" = "all" ]; then
-    # Loop through each library and install dependencies
-    for LIBRARY in "${LIBRARIES[@]}"
-    do
-        rapids-logger "gathering GPU tests for $LIBRARY"
-        TEST_DIR="$LIBRARY/$LIBRARY/tests"
-        # Find all Python scripts containing the keywords cudf or dask_cudf except test_quadmesh.py
-        FILES+=" $(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR" | grep -v test_quadmesh.py)"
-    done
-else
-    rapids-logger "gathering GPU tests for $PROJECT"
-    TEST_DIR="$PROJECT/$PROJECT/tests"
-    # Find all Python scripts containing the keywords cudf or dask_cudf
-    FILES+=$(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR")
-fi
 
-EXITCODE=0
-trap "EXITCODE=1" ERR
-set +e
+    rapids-logger "Run GPU tests for $PROJECT"
 
-rapids-logger "running all gathered tests"
-DATASHADER_TEST_GPU=1 pytest \
-  --numprocesses=8 \
-  --dist=worksteal \
-  $FILES
+    python -m pytest $PROJECT/tests/ --numprocesses=8 --dist=worksteal --gpu
 
-if [[ "$PROJECT" = "all" ]] || [[ "$PROJECT" = "datashader" ]]; then
-    # run test_quadmesh.py separately as dask.array tests fail with numprocesses
-    rapids-logger "running test_quadmesh.py"
-    DATASHADER_TEST_GPU=1 pytest datashader/datashader/tests/test_quadmesh.py
+    popd
 fi
 
 rapids-logger "Test script exiting with value: $EXITCODE"

ci/test_python.sh

@@ -43,8 +43,8 @@ pushd python/
 pytest \
   --cache-clear \
   --junitxml="${RAPIDS_TESTS_DIR}/junit-cuxfilter.xml" \
-   --numprocesses=8 \
-   --dist=worksteal \
+  --numprocesses=8 \
+  --dist=worksteal \
   --cov-config=.coveragerc \
   --cov=cuxfilter \
   --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuxfilter-coverage.xml" \