diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50fee348..21362a37 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: args: ["--config=.flake8"] files: python/.*$ - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.2.0 + rev: v0.3.1 hooks: - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d5ccca1b..a9ccbea8 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -18,7 +18,7 @@ CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}') CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} -#Get . for next version +# Get . for next version NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} @@ -39,7 +39,6 @@ sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_S # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION - DEPENDENCIES=( cudf cuxfilter @@ -61,8 +60,7 @@ done sed_runner "/version == / s/== .*\`/== ${NEXT_SHORT_TAG}\`/g" README.md sed_runner "/cuxfilter=[0-9]\{2\}.[0-9]\{2\}/ s/=[0-9]\{2\}.[0-9]\{2\}/=${NEXT_SHORT_TAG}/g" README.md - # CI files for FILE in .github/workflows/*.yaml; do - sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" ${FILE}; + sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" done diff --git a/ci/test_external.sh b/ci/test_external.sh index 613b7d1f..d12a2cdb 100755 --- a/ci/test_external.sh +++ b/ci/test_external.sh @@ -7,18 +7,37 @@ rapids-logger "Create test_external conda environment" . /opt/conda/etc/profile.d/conda.sh # Install external dependencies into test_external conda environment -rapids-conda-retry env update -f ./ci/utils/external_dependencies.yaml +rapids-dependency-file-generator \ + --output conda \ + --file-key test_external \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml +rapids-mamba-retry env create --yes -f env.yaml -n test_external + +# Temporarily allow unbound variables for conda activation. +set +u conda activate test_external +set -u + +rapids-print-env # Define input parameter -PROJECT=$1 -PR_NUMBER=$2 +PROJECT="${1:-all}" +PR_NUMBER="${2:-0}" LIBRARIES=("datashader" "holoviews") +# set conda env variable NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS to 0 +# to suppress low occupancy warnings +export NUMBA_CUDA_LOW_OCCUPANCY_WARNINGS=0 + # Change directory to /tmp pushd /tmp + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + # Clone the specified Python libraries if [ "$PROJECT" = "all" ]; then # Loop through each library and install dependencies @@ -33,7 +52,11 @@ if [ "$PROJECT" = "all" ]; then # Change directory to the library pushd $LIBRARY # Run setup.py with test dependencies - python -m pip install -e .[tests] + python -m pip install .[tests] + + rapids-logger "Run GPU tests for $LIBRARY" + + python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu popd done @@ -42,7 +65,7 @@ else git clone https://github.com/pyviz/$PROJECT.git # Check if PR_NUMBER is a non-empty, valid number - if [ -n "$PR_NUMBER" ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then + if [ "$PR_NUMBER" -ne 0 ] && [ "$PR_NUMBER" -eq "$PR_NUMBER" ] 2>/dev/null; then rapids-logger "checkout PR $PR_NUMBER" # Fetch the pull request and check it out git fetch origin pull/$PR_NUMBER/head:pr/$PR_NUMBER @@ -53,42 +76,14 @@ else # Change directory to the specified project pushd $PROJECT # Run setup.py with test dependencies - python -m pip install -e .[tests] - popd -fi + python -m pip install .[tests] -FILES="" -# Install and run tests -if [ "$PROJECT" = "all" ]; then - # Loop through each library and install dependencies - for LIBRARY in "${LIBRARIES[@]}" - do - rapids-logger "gathering GPU tests for $LIBRARY" - TEST_DIR="$LIBRARY/$LIBRARY/tests" - # Find all Python scripts containing the keywords cudf or dask_cudf except test_quadmesh.py - FILES+=" $(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR" | grep -v test_quadmesh.py)" - done -else - rapids-logger "gathering GPU tests for $PROJECT" - TEST_DIR="$PROJECT/$PROJECT/tests" - # Find all Python scripts containing the keywords cudf or dask_cudf - FILES+=$(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR") -fi -EXITCODE=0 -trap "EXITCODE=1" ERR -set +e + rapids-logger "Run GPU tests for $PROJECT" -rapids-logger "running all gathered tests" -DATASHADER_TEST_GPU=1 pytest \ - --numprocesses=8 \ - --dist=worksteal \ - $FILES + python -m pytest $PROJECT/tests/ --numprocesses=8 --dist=worksteal --gpu -if [[ "$PROJECT" = "all" ]] || [[ "$PROJECT" = "datashader" ]]; then - # run test_quadmesh.py separately as dask.array tests fail with numprocesses - rapids-logger "running test_quadmesh.py" - DATASHADER_TEST_GPU=1 pytest datashader/datashader/tests/test_quadmesh.py + popd fi rapids-logger "Test script exiting with value: $EXITCODE" diff --git a/ci/utils/external_dependencies.yaml b/ci/utils/external_dependencies.yaml deleted file mode 100644 index 518c0a96..00000000 --- a/ci/utils/external_dependencies.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: test_external -channels: - - rapidsai-nightly - - conda-forge - - nvidia -dependencies: - - cudf==24.10.*,>=0.0.0a0 - - dask-cudf==24.10.*,>=0.0.0a0 - - cuxfilter==24.10.*,>=0.0.0a0 - - cuda-version=12.0 - - python>=3.9,<3.12 - - xarray-spatial - - pycaret - - graphistry - - dash - - dask-sql - - pytest-benchmark diff --git a/dependencies.yaml b/dependencies.yaml index faaacb67..fe0d4e3c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -70,6 +70,10 @@ files: key: test includes: - test_python + test_external: + output: none + includes: + - test_external channels: - rapidsai - rapidsai-nightly @@ -257,3 +261,16 @@ dependencies: - output_types: conda packages: - panel>=1.0,<=1.3.4 + test_external: + common: + - output_types: [conda] + packages: + - coverage + - *cudf_unsuffixed + - cupy>=12.0.0 + - *cuspatial_unsuffixed + - *dask_cudf_unsuffixed + - cuxfilter==24.10.*,>=0.0.0a0 + - python>=3.9,<3.12 + - pytest-benchmark + - pytest-xdist diff --git a/python/cuxfilter/charts/core/non_aggregate/core_graph.py b/python/cuxfilter/charts/core/non_aggregate/core_graph.py index f0216ec9..d9fc9a51 100644 --- a/python/cuxfilter/charts/core/non_aggregate/core_graph.py +++ b/python/cuxfilter/charts/core/non_aggregate/core_graph.py @@ -51,14 +51,14 @@ def edge_columns(self): @property def node_columns(self): - if self.node_aggregate_col: - return [ - self.node_id, - self.node_x, - self.node_y, - self.node_aggregate_col, - ] - return [self.node_id, self.node_x, self.node_y] + cols = [ + self.node_id, + self.node_x, + self.node_y, + ] + if self.node_aggregate_col != self.node_id: + cols.append(self.node_aggregate_col) + return cols def __init__( self,