Build CUDA 11.8 and Python 3.10 Packages (rapidsai#12457)

This PR updates `cudf` to build against branch [cuda-118](https://github.com/rapidsai/shared-action-workflows/compare/cuda-118) of the `shared-action-workflow` repository.

That branch contains updates for CUDA 11.8 and Python 3.10 packages.

It also includes some minor file renames.

Authors:
  - AJ Schmidt (https://github.com/ajschmidt8)
  - Bradley Dice (https://github.com/bdice)

Approvers:
  - Robert Maynard (https://github.com/robertmaynard)
  - https://github.com/brandon-b-miller
  - GALI PREM SAGAR (https://github.com/galipremsagar)
  - Ray Douglass (https://github.com/raydouglass)
  - Jake Awe (https://github.com/AyodeAwe)

URL: rapidsai#12457

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ concurrency:
 jobs:
   cpp-build:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -37,7 +37,7 @@ jobs:
   python-build:
     needs: [cpp-build]
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -46,7 +46,7 @@ jobs:
   upload-conda:
     needs: [cpp-build, python-build]
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-118
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}

.github/workflows/pr.yaml

@@ -21,47 +21,47 @@ jobs:
       - conda-java-tests
       - conda-notebook-tests
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-118
   checks:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-118
   conda-cpp-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118
     with:
       build_type: pull-request
   conda-cpp-tests:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118
     with:
       build_type: pull-request
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118
     with:
       build_type: pull-request
   conda-python-cudf-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118
     with:
       build_type: pull-request
       test_script: "ci/test_python_cudf.sh"
   conda-python-other-tests:
     # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118
     with:
       build_type: pull-request
       test_script: "ci/test_python_other.sh"
   conda-java-tests:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118
     with:
       build_type: pull-request
       node_type: "gpu-latest-1"
@@ -71,7 +71,7 @@ jobs:
   conda-notebook-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118
     with:
       build_type: pull-request
       node_type: "gpu-latest-1"

.github/workflows/test.yaml

@@ -16,15 +16,15 @@ on:
 jobs:
   conda-cpp-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   conda-python-cudf-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
@@ -34,7 +34,7 @@ jobs:
   conda-python-other-tests:
     # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
@@ -43,7 +43,7 @@ jobs:
       test_script: "ci/test_python_other.sh"
   conda-java-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
@@ -55,7 +55,7 @@ jobs:
       run_script: "ci/test_java.sh"
   conda-notebook-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}

CONTRIBUTING.md

@@ -105,7 +105,7 @@ Instructions for a minimal build environment without conda are included below.
 # create the conda environment (assuming in base `cudf` directory)
 # note: RAPIDS currently doesn't support `channel_priority: strict`;
 # use `channel_priority: flexible` instead
-conda env create --name cudf_dev --file conda/environments/all_cuda-115_arch-x86_64.yaml
+conda env create --name cudf_dev --file conda/environments/all_cuda-118_arch-x86_64.yaml
 # activate the environment
 conda activate cudf_dev
 ```

README.md

@@ -57,7 +57,7 @@ Please see the [Demo Docker Repository](https://hub.docker.com/r/rapidsai/rapids
 
 ### CUDA/GPU requirements
 
-* CUDA 11.0+
+* CUDA 11.2+
 * NVIDIA driver 450.80.02+
 * Pascal architecture or better (Compute Capability >=6.0)
 
@@ -66,12 +66,8 @@ Please see the [Demo Docker Repository](https://hub.docker.com/r/rapidsai/rapids
 cuDF can be installed with conda ([miniconda](https://conda.io/miniconda.html), or the full [Anaconda distribution](https://www.anaconda.com/download)) from the `rapidsai` channel:
 
 ```bash
-# for CUDA 11.5
 conda install -c rapidsai -c conda-forge -c nvidia \
-    cudf=22.10 python=3.9 cudatoolkit=11.5
-# for CUDA 11.2
-conda install -c rapidsai -c conda-forge -c nvidia \
-    cudf=22.10 python=3.9 cudatoolkit=11.2
+    cudf=23.02 python=3.10 cudatoolkit=11.8
 ```
 
 We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD

ci/test_notebooks.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -9,7 +9,7 @@ rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file_key test_notebooks \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --force -f env.yaml -n test
 

conda/environments/all_cuda-115_arch-x86_64.yaml → conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -16,7 +16,7 @@ dependencies:
 - cmake>=3.23.1,!=3.25.0
 - cubinlinker
 - cuda-python>=11.7.1,<12.0
-- cudatoolkit=11.5
+- cudatoolkit=11.8
 - cupy>=9.5.0,<12.0.0a0
 - cxx-compiler
 - cython>=0.29,<0.30
@@ -41,7 +41,7 @@ dependencies:
 - numba>=0.56.2
 - numpy
 - numpydoc
-- nvcc_linux-64=11.5
+- nvcc_linux-64=11.8
 - nvtx>=0.2.1
 - packaging
 - pandas>=1.0,<1.6.0dev0
@@ -60,7 +60,7 @@ dependencies:
 - pytest-xdist
 - python-confluent-kafka=1.7.0
 - python-snappy>=0.6.0
-- python>=3.8,<3.10
+- python>=3.8,<3.11
 - pytorch<1.12.0
 - rmm=23.02.*
 - s3fs>=2022.3.0
@@ -78,4 +78,4 @@ dependencies:
 - typing_extensions
 - pip:
   - git+https://github.com/python-streamz/streamz.git@master
-name: all_cuda-115_arch-x86_64
+name: all_cuda-118_arch-x86_64

conda/recipes/libcudf/meta.yaml

@@ -337,6 +337,7 @@ outputs:
         - cmake {{ cmake_version }}
         - {{ compiler('c') }}
         - {{ compiler('cxx') }}
+        - {{ compiler('cuda') }} {{ cuda_version }}
         - ninja
         - sysroot_{{ target_platform }} {{ sysroot_version }}
       host:

dependencies.yaml

@@ -3,7 +3,7 @@ files:
   all:
     output: conda
     matrix:
-      cuda: ["11.5"]
+      cuda: ["11.8"]
       arch: [x86_64]
     includes:
       - build
@@ -84,14 +84,14 @@ dependencies:
         matrices:
           - matrix:
               arch: x86_64
-              cuda: "11.5"
+              cuda: "11.8"
             packages:
-              - &nvcc_amd64_115 nvcc_linux-64=11.5
+              - nvcc_linux-64=11.8
           - matrix:
               arch: aarch64
-              cuda: "11.5"
+              cuda: "11.8"
             packages:
-              - &nvcc_aarch64_115 nvcc_linux-aarch64=11.5
+              - nvcc_linux-aarch64=11.8
   cudatoolkit:
     specific:
       - output_types: conda
@@ -108,6 +108,10 @@ dependencies:
               cuda: "11.5"
             packages:
               - cudatoolkit=11.5
+          - matrix:
+              cuda: "11.8"
+            packages:
+              - cudatoolkit=11.8
   develop:
     common:
       - output_types: [conda, requirements]
@@ -174,12 +178,22 @@ dependencies:
               arch: x86_64
               cuda: "11.5"
             packages:
-              - *nvcc_amd64_115
+              - nvcc_linux-64=11.5
           - matrix:
               arch: aarch64
               cuda: "11.5"
             packages:
-              - *nvcc_aarch64_115
+              - nvcc_linux-aarch64=11.5
+          - matrix:
+              arch: x86_64
+              cuda: "11.8"
+            packages:
+              - nvcc_linux-64=11.8
+          - matrix:
+              arch: aarch64
+              cuda: "11.8"
+            packages:
+              - nvcc_linux-aarch64=11.8
   notebooks:
     common:
       - output_types: [conda, requirements]
@@ -200,8 +214,12 @@ dependencies:
             packages:
               - python=3.9
           - matrix:
+              py: "3.10"
             packages:
-              - python>=3.8,<3.10
+              - python=3.10
+          - matrix:
+            packages:
+              - python>=3.8,<3.11
   run:
     common:
       - output_types: [conda, requirements]
@@ -261,6 +279,10 @@ dependencies:
               cuda: "11.5"
             packages:
               - cuda-nvtx=11.5
+          - matrix:
+              cuda: "11.8"
+            packages:
+              - cuda-nvtx=11.8
   test_python:
     common:
       - output_types: [conda, requirements]

python/cudf/setup.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+# Copyright (c) 2018-2023, NVIDIA CORPORATION.
 
 import os
 
@@ -69,6 +69,7 @@ def get_versions():
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     cmdclass=versioneer.get_cmdclass(),
     include_package_data=True,

python/cudf_kafka/setup.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 import os
 import shutil
 import sysconfig
@@ -95,6 +95,7 @@
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     # Include the separately-compiled shared library
     ext_modules=cythonize(

python/custreamz/setup.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
 import versioneer
 from setuptools import find_packages, setup
@@ -23,6 +23,7 @@
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     packages=find_packages(include=["custreamz", "custreamz.*"]),
     cmdclass=versioneer.get_cmdclass(),

python/dask_cudf/setup.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2022, NVIDIA CORPORATION.
+# Copyright (c) 2019-2023, NVIDIA CORPORATION.
 
 import os
 
@@ -54,6 +54,7 @@ def get_versions():
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     packages=find_packages(exclude=["tests", "tests.*"]),
     cmdclass=versioneer.get_cmdclass(),

python/strings_udf/setup.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 import os
 import re
@@ -69,6 +69,7 @@ def get_cuda_version_from_header(cuda_include_dir, delimiter=""):
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     packages=find_packages(include=["strings_udf", "strings_udf.*"]),
     package_data={

python/strings_udf/strings_udf/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 import glob
 import os
 
@@ -15,7 +15,7 @@
 logger = get_logger()
 
 # tracks the version of CUDA used to build the c++ and PTX components
-STRINGS_UDF_PTX_VERSION = (11, 5)
+STRINGS_UDF_PTX_VERSION = (11, 8)
 
 
 def _get_appropriate_file(sms, cc):