Merge branch 'branch-24.10' into bug/cudf-pandas-integration

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
                 args: ["--config=.flake8"]
                 files: python/.*$
       - repo: https://github.com/rapidsai/pre-commit-hooks
-        rev: v0.2.0
+        rev: v0.3.1
         hooks:
             - id: verify-alpha-spec
       - repo: https://github.com/rapidsai/dependency-file-generator

ci/release/update-version.sh

@@ -18,7 +18,7 @@ CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}')
 CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}')
 CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}
 
-#Get <major>.<minor> for next version
+# Get <major>.<minor> for next version
 NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}')
 NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
 NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
@@ -39,7 +39,6 @@ sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_S
 # Centralized version file update
 echo "${NEXT_FULL_TAG}" > VERSION
 
-
 DEPENDENCIES=(
   cudf
   cuxfilter
@@ -61,8 +60,7 @@ done
 sed_runner "/version == / s/== .*\`/== ${NEXT_SHORT_TAG}\`/g" README.md
 sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_SHORT_TAG}/g" README.md
 
-
 # CI files
 for FILE in .github/workflows/*.yaml; do
-  sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" ${FILE};
+  sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
 done

ci/test_external.sh

@@ -7,18 +7,37 @@ rapids-logger "Create test_external conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
 # Install external dependencies into test_external conda environment
-rapids-conda-retry env update -f ./ci/utils/external_dependencies.yaml
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key test_external \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
+rapids-mamba-retry env create --yes -f env.yaml -n test_external
+
+# Temporarily allow unbound variables for conda activation.
+set +u
 conda activate test_external
+set -u
+
+rapids-print-env
 
 # Define input parameter
-PROJECT=$1
-PR_NUMBER=$2
+PROJECT="${1:-all}"
+PR_NUMBER="${2:-0}"
 LIBRARIES=("datashader" "holoviews")
 
+# set conda env variable NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS to 0
+# to suppress low occupancy warnings
+export NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS=0
+
 # Change directory to /tmp
 pushd /tmp
 
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
 # Clone the specified Python libraries
 if [ "$PROJECT" = "all" ]; then
     # Loop through each library and install dependencies
@@ -33,7 +52,11 @@ if [ "$PROJECT" = "all" ]; then
         # Change directory to the library
         pushd $LIBRARY
         # Run setup.py with test dependencies
-        python -m pip install -e .[tests]
+        python -m pip install .[tests]
+
+        rapids-logger "Run GPU tests for $LIBRARY"
+
+        python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu
 
         popd
     done
@@ -42,7 +65,7 @@ else
     git clone https://github.com/pyviz/$PROJECT.git
 
     # Check if PR_NUMBER is a non-empty, valid number
-    if [ -n "$PR_NUMBER" ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then
+    if [ "$PR_NUMBER" -ne 0 ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then
         rapids-logger "checkout PR $PR_NUMBER"
         # Fetch the pull request and check it out
         git fetch origin pull/$PR_NUMBER/head:pr/$PR_NUMBER
@@ -53,42 +76,14 @@ else
     # Change directory to the specified project
     pushd $PROJECT
     # Run setup.py with test dependencies
-    python -m pip install -e .[tests]
-    popd
-fi
+    python -m pip install .[tests]
 
-FILES=""
-# Install and run tests
-if [ "$PROJECT" = "all" ]; then
-    # Loop through each library and install dependencies
-    for LIBRARY in "${LIBRARIES[@]}"
-    do
-        rapids-logger "gathering GPU tests for $LIBRARY"
-        TEST_DIR="$LIBRARY/$LIBRARY/tests"
-        # Find all Python scripts containing the keywords cudf or dask_cudf except test_quadmesh.py
-        FILES+=" $(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR" | grep -v test_quadmesh.py)"
-    done
-else
-    rapids-logger "gathering GPU tests for $PROJECT"
-    TEST_DIR="$PROJECT/$PROJECT/tests"
-    # Find all Python scripts containing the keywords cudf or dask_cudf
-    FILES+=$(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR")
-fi
 
-EXITCODE=0
-trap "EXITCODE=1" ERR
-set +e
+    rapids-logger "Run GPU tests for $PROJECT"
 
-rapids-logger "running all gathered tests"
-DATASHADER_TEST_GPU=1 pytest \
-  --numprocesses=8 \
-  --dist=worksteal \
-  $FILES
+    python -m pytest $PROJECT/tests/ --numprocesses=8 --dist=worksteal --gpu
 
-if [[ "$PROJECT" = "all" ]] || [[ "$PROJECT" = "datashader" ]]; then
-    # run test_quadmesh.py separately as dask.array tests fail with numprocesses
-    rapids-logger "running test_quadmesh.py"
-    DATASHADER_TEST_GPU=1 pytest datashader/datashader/tests/test_quadmesh.py
+    popd
 fi
 
 rapids-logger "Test script exiting with value: $EXITCODE"

dependencies.yaml

@@ -70,6 +70,10 @@ files:
       key: test
     includes:
       - test_python
+  test_external:
+    output: none
+    includes:
+      - test_external
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -257,3 +261,16 @@ dependencies:
       - output_types: conda
         packages:
           - panel>=1.0,<=1.3.4
+  test_external:
+    common:
+      - output_types: [conda]
+        packages:
+          - coverage
+          - *cudf_unsuffixed
+          - cupy>=12.0.0
+          - *cuspatial_unsuffixed
+          - *dask_cudf_unsuffixed
+          - cuxfilter==24.10.*,>=0.0.0a0
+          - python>=3.9,<3.12
+          - pytest-benchmark
+          - pytest-xdist

python/cuxfilter/charts/core/non_aggregate/core_graph.py

@@ -51,14 +51,14 @@ def edge_columns(self):
 
     @property
     def node_columns(self):
-        if self.node_aggregate_col:
-            return [
-                self.node_id,
-                self.node_x,
-                self.node_y,
-                self.node_aggregate_col,
-            ]
-        return [self.node_id, self.node_x, self.node_y]
+        cols = [
+            self.node_id,
+            self.node_x,
+            self.node_y,
+        ]
+        if self.node_aggregate_col != self.node_id:
+            cols.append(self.node_aggregate_col)
+        return cols
 
     def __init__(
         self,