From c325c7fc085f10662bf02f66cd32349f94391c56 Mon Sep 17 00:00:00 2001
From: Ajay Thorve <AjayThorve@users.noreply.github.com>
Date: Wed, 14 Aug 2024 15:28:16 -0700
Subject: [PATCH 1/4] Fix tests (#620)

This PR fixes

1. [failing tests](https://github.com/rapidsai/cuxfilter/actions/runs/10303789190/job/28671747182?pr=617), which were caused by duplicate columns in the core_graph source_df.

2. Failing [external tests](https://github.com/rapidsai/cuxfilter/actions/workflows/test-external.yaml)

Authors:
  - Ajay Thorve (https://github.com/AjayThorve)

Approvers:
  - Matthew Roeschke (https://github.com/mroeschke)
  - Bradley Dice (https://github.com/bdice)
  - Allan (https://github.com/exactlyallan)

URL: https://github.com/rapidsai/cuxfilter/pull/620
---
 ci/test_external.sh                           | 59 ++++++++-----------
 ci/utils/external_dependencies.yaml           | 17 ------
 dependencies.yaml                             | 22 +++++++
 .../charts/core/non_aggregate/core_graph.py   | 16 ++---
 4 files changed, 55 insertions(+), 59 deletions(-)
 delete mode 100644 ci/utils/external_dependencies.yaml

diff --git a/ci/test_external.sh b/ci/test_external.sh
index 613b7d1f..ce8b42b6 100755
--- a/ci/test_external.sh
+++ b/ci/test_external.sh
@@ -7,9 +7,19 @@ rapids-logger "Create test_external conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
 # Install external dependencies into test_external conda environment
-rapids-conda-retry env update -f ./ci/utils/external_dependencies.yaml
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key test_external \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
+rapids-mamba-retry env create --yes -f env.yaml -n test_external
+
+# Temporarily allow unbound variables for conda activation.
+set +u
 conda activate test_external
+set -u
+
+rapids-print-env
 
 # Define input parameter
 PROJECT=$1
@@ -19,6 +29,11 @@ LIBRARIES=("datashader" "holoviews")
 # Change directory to /tmp
 pushd /tmp
 
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
 # Clone the specified Python libraries
 if [ "$PROJECT" = "all" ]; then
     # Loop through each library and install dependencies
@@ -33,7 +48,11 @@ if [ "$PROJECT" = "all" ]; then
         # Change directory to the library
         pushd $LIBRARY
         # Run setup.py with test dependencies
-        python -m pip install -e .[tests]
+        python -m pip install .[tests]
+
+        rapids-logger "Run GPU tests for $LIBRARY"
+
+        python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu
 
         popd
     done
@@ -53,42 +72,14 @@ else
     # Change directory to the specified project
     pushd $PROJECT
     # Run setup.py with test dependencies
-    python -m pip install -e .[tests]
-    popd
-fi
+    python -m pip install .[tests]
 
-FILES=""
-# Install and run tests
-if [ "$PROJECT" = "all" ]; then
-    # Loop through each library and install dependencies
-    for LIBRARY in "${LIBRARIES[@]}"
-    do
-        rapids-logger "gathering GPU tests for $LIBRARY"
-        TEST_DIR="$LIBRARY/$LIBRARY/tests"
-        # Find all Python scripts containing the keywords cudf or dask_cudf except test_quadmesh.py
-        FILES+=" $(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR" | grep -v test_quadmesh.py)"
-    done
-else
-    rapids-logger "gathering GPU tests for $PROJECT"
-    TEST_DIR="$PROJECT/$PROJECT/tests"
-    # Find all Python scripts containing the keywords cudf or dask_cudf
-    FILES+=$(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR")
-fi
 
-EXITCODE=0
-trap "EXITCODE=1" ERR
-set +e
+    rapids-logger "Run GPU tests for $LIBRARY"
 
-rapids-logger "running all gathered tests"
-DATASHADER_TEST_GPU=1 pytest \
-  --numprocesses=8 \
-  --dist=worksteal \
-  $FILES
+    python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu
 
-if [[ "$PROJECT" = "all" ]] || [[ "$PROJECT" = "datashader" ]]; then
-    # run test_quadmesh.py separately as dask.array tests fail with numprocesses
-    rapids-logger "running test_quadmesh.py"
-    DATASHADER_TEST_GPU=1 pytest datashader/datashader/tests/test_quadmesh.py
+    popd
 fi
 
 rapids-logger "Test script exiting with value: $EXITCODE"
diff --git a/ci/utils/external_dependencies.yaml b/ci/utils/external_dependencies.yaml
deleted file mode 100644
index 518c0a96..00000000
--- a/ci/utils/external_dependencies.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-name: test_external
-channels:
-  - rapidsai-nightly
-  - conda-forge
-  - nvidia
-dependencies:
-  - cudf==24.10.*,>=0.0.0a0
-  - dask-cudf==24.10.*,>=0.0.0a0
-  - cuxfilter==24.10.*,>=0.0.0a0
-  - cuda-version=12.0
-  - python>=3.9,<3.12
-  - xarray-spatial
-  - pycaret
-  - graphistry
-  - dash
-  - dask-sql
-  - pytest-benchmark
diff --git a/dependencies.yaml b/dependencies.yaml
index faaacb67..ae83599d 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -70,6 +70,10 @@ files:
       key: test
     includes:
       - test_python
+  test_external:
+    output: none
+    includes:
+      - test_external
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -257,3 +261,21 @@ dependencies:
       - output_types: conda
         packages:
           - panel>=1.0,<=1.3.4
+  test_external:
+    common:
+      - output_types: [conda]
+        packages:
+          - coverage
+          - *cudf_unsuffixed
+          - cupy>=12.0.0
+          - *cuspatial_unsuffixed
+          - *dask_cudf_unsuffixed
+          - cuxfilter==24.10.*,>=0.0.0a0
+          - python>=3.9,<3.12
+          - xarray-spatial
+          - pycaret
+          - graphistry
+          - dash
+          - dask-sql
+          - pytest-benchmark
+          - pytest-xdist
diff --git a/python/cuxfilter/charts/core/non_aggregate/core_graph.py b/python/cuxfilter/charts/core/non_aggregate/core_graph.py
index 13d30a14..ab9b4101 100644
--- a/python/cuxfilter/charts/core/non_aggregate/core_graph.py
+++ b/python/cuxfilter/charts/core/non_aggregate/core_graph.py
@@ -50,14 +50,14 @@ def edge_columns(self):
 
     @property
     def node_columns(self):
-        if self.node_aggregate_col:
-            return [
-                self.node_id,
-                self.node_x,
-                self.node_y,
-                self.node_aggregate_col,
-            ]
-        return [self.node_id, self.node_x, self.node_y]
+        cols = [
+            self.node_id,
+            self.node_x,
+            self.node_y,
+        ]
+        if self.node_aggregate_col != self.node_id:
+            cols.append(self.node_aggregate_col)
+        return cols
 
     def __init__(
         self,

From 95bd8db72f6f6cfed9dc0f44db55105f6e4f848b Mon Sep 17 00:00:00 2001
From: Bradley Dice <bdice@bradleydice.com>
Date: Thu, 15 Aug 2024 12:11:17 -0500
Subject: [PATCH 2/4] Improve update-version.sh (#616)

A few small tweaks to `update-version.sh` for alignment across RAPIDS.

Authors:
  - Bradley Dice (https://github.com/bdice)

Approvers:
  - James Lamb (https://github.com/jameslamb)
  - Ray Douglass (https://github.com/raydouglass)

URL: https://github.com/rapidsai/cuxfilter/pull/616
---
 ci/release/update-version.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
index d5ccca1b..a9ccbea8 100755
--- a/ci/release/update-version.sh
+++ b/ci/release/update-version.sh
@@ -18,7 +18,7 @@ CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}')
 CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}')
 CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}
 
-#Get <major>.<minor> for next version
+# Get <major>.<minor> for next version
 NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}')
 NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
 NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
@@ -39,7 +39,6 @@ sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_S
 # Centralized version file update
 echo "${NEXT_FULL_TAG}" > VERSION
 
-
 DEPENDENCIES=(
   cudf
   cuxfilter
@@ -61,8 +60,7 @@ done
 sed_runner "/version == / s/== .*\`/== ${NEXT_SHORT_TAG}\`/g" README.md
 sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_SHORT_TAG}/g" README.md
 
-
 # CI files
 for FILE in .github/workflows/*.yaml; do
-  sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" ${FILE};
+  sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
 done

From 76fc52e1e6daf6e97f8668a3d5ad855ce304230e Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Thu, 15 Aug 2024 16:23:37 -0400
Subject: [PATCH 3/4] Update pre-commit hooks (#617)

This PR updates pre-commit hooks to the latest versions that are supported without causing style check errors.

Authors:
  - Kyle Edwards (https://github.com/KyleFromNVIDIA)

Approvers:
  - James Lamb (https://github.com/jameslamb)

URL: https://github.com/rapidsai/cuxfilter/pull/617
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 50fee348..21362a37 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@ repos:
                 args: ["--config=.flake8"]
                 files: python/.*$
       - repo: https://github.com/rapidsai/pre-commit-hooks
-        rev: v0.2.0
+        rev: v0.3.1
         hooks:
             - id: verify-alpha-spec
       - repo: https://github.com/rapidsai/dependency-file-generator

From a00bf9d5d4fe3a17c872566deaae5ef4e759bf25 Mon Sep 17 00:00:00 2001
From: Ajay Thorve <AjayThorve@users.noreply.github.com>
Date: Thu, 15 Aug 2024 17:02:01 -0700
Subject: [PATCH 4/4] Fix/external tests (#621)

This PR aims to fix the [unbound input variable](https://github.com/rapidsai/cuxfilter/actions/runs/10402536485/job/28807203732#step:9:734) issue when running the external testing workflow

Authors:
  - Ajay Thorve (https://github.com/AjayThorve)

Approvers:
  - Bradley Dice (https://github.com/bdice)

URL: https://github.com/rapidsai/cuxfilter/pull/621
---
 ci/test_external.sh | 14 +++++++++-----
 dependencies.yaml   |  5 -----
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/ci/test_external.sh b/ci/test_external.sh
index ce8b42b6..d12a2cdb 100755
--- a/ci/test_external.sh
+++ b/ci/test_external.sh
@@ -22,10 +22,14 @@ set -u
 rapids-print-env
 
 # Define input parameter
-PROJECT=$1
-PR_NUMBER=$2
+PROJECT="${1:-all}"
+PR_NUMBER="${2:-0}"
 LIBRARIES=("datashader" "holoviews")
 
+# set conda env variable NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS to 0
+# to suppress low occupancy warnings
+export NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS=0
+
 # Change directory to /tmp
 pushd /tmp
 
@@ -61,7 +65,7 @@ else
     git clone https://github.com/pyviz/$PROJECT.git
 
     # Check if PR_NUMBER is a non-empty, valid number
-    if [ -n "$PR_NUMBER" ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then
+    if [ "$PR_NUMBER" -ne 0 ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then
         rapids-logger "checkout PR $PR_NUMBER"
         # Fetch the pull request and check it out
         git fetch origin pull/$PR_NUMBER/head:pr/$PR_NUMBER
@@ -75,9 +79,9 @@ else
     python -m pip install .[tests]
 
 
-    rapids-logger "Run GPU tests for $LIBRARY"
+    rapids-logger "Run GPU tests for $PROJECT"
 
-    python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu
+    python -m pytest $PROJECT/tests/ --numprocesses=8 --dist=worksteal --gpu
 
     popd
 fi
diff --git a/dependencies.yaml b/dependencies.yaml
index ae83599d..fe0d4e3c 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -272,10 +272,5 @@ dependencies:
           - *dask_cudf_unsuffixed
           - cuxfilter==24.10.*,>=0.0.0a0
           - python>=3.9,<3.12
-          - xarray-spatial
-          - pycaret
-          - graphistry
-          - dash
-          - dask-sql
           - pytest-benchmark
           - pytest-xdist