From f011c85d239e2578abcec9012a35a1e2bf5a4f9c Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 9 Jan 2023 19:33:07 -0500 Subject: [PATCH] Build CUDA `11.8` and Python `3.10` Packages (#12457) This PR updates `cudf` to build against branch [cuda-118](https://github.com/rapidsai/shared-action-workflows/compare/cuda-118) of the `shared-action-workflow` repository. That branch contains updates for CUDA 11.8 and Python 3.10 packages. It also includes some minor file renames. Authors: - AJ Schmidt (https://github.com/ajschmidt8) - Bradley Dice (https://github.com/bdice) Approvers: - Robert Maynard (https://github.com/robertmaynard) - https://github.com/brandon-b-miller - GALI PREM SAGAR (https://github.com/galipremsagar) - Ray Douglass (https://github.com/raydouglass) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/12457 --- .github/workflows/build.yaml | 6 +-- .github/workflows/pr.yaml | 18 ++++----- .github/workflows/test.yaml | 10 ++--- CONTRIBUTING.md | 2 +- README.md | 8 +--- ci/test_notebooks.sh | 4 +- ..._64.yaml => all_cuda-118_arch-x86_64.yaml} | 8 ++-- conda/recipes/libcudf/meta.yaml | 1 + dependencies.yaml | 38 +++++++++++++++---- python/cudf/setup.py | 3 +- python/cudf_kafka/setup.py | 3 +- python/custreamz/setup.py | 3 +- python/dask_cudf/setup.py | 3 +- python/strings_udf/setup.py | 3 +- python/strings_udf/strings_udf/__init__.py | 4 +- 15 files changed, 69 insertions(+), 45 deletions(-) rename conda/environments/{all_cuda-115_arch-x86_64.yaml => all_cuda-118_arch-x86_64.yaml} (93%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 620a13fe173..d7e81bcd3a6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3100af1d371..fd910e68854 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,32 +21,32 @@ jobs: - conda-java-tests - conda-notebook-tests secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-118 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-118 conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 with: build_type: pull-request conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 with: build_type: pull-request conda-python-cudf-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 with: build_type: pull-request test_script: "ci/test_python_cudf.sh" @@ -54,14 +54,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 with: build_type: pull-request test_script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 with: build_type: pull-request node_type: "gpu-latest-1" @@ -71,7 +71,7 @@ jobs: conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 with: build_type: pull-request node_type: "gpu-latest-1" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 38e9f2f6daa..d29c30cdb58 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} @@ -43,7 +43,7 @@ jobs: test_script: "ci/test_python_other.sh" conda-java-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} @@ -55,7 +55,7 @@ jobs: run_script: "ci/test_java.sh" conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d117f8d7e64..f5959de10ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,7 +105,7 @@ Instructions for a minimal build environment without conda are included below. # create the conda environment (assuming in base `cudf` directory) # note: RAPIDS currently doesn't support `channel_priority: strict`; # use `channel_priority: flexible` instead -conda env create --name cudf_dev --file conda/environments/all_cuda-115_arch-x86_64.yaml +conda env create --name cudf_dev --file conda/environments/all_cuda-118_arch-x86_64.yaml # activate the environment conda activate cudf_dev ``` diff --git a/README.md b/README.md index a013d3a9ea4..d99bee5fc0c 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Please see the [Demo Docker Repository](https://hub.docker.com/r/rapidsai/rapids ### CUDA/GPU requirements -* CUDA 11.0+ +* CUDA 11.2+ * NVIDIA driver 450.80.02+ * Pascal architecture or better (Compute Capability >=6.0) @@ -66,12 +66,8 @@ Please see the [Demo Docker Repository](https://hub.docker.com/r/rapidsai/rapids cuDF can be installed with conda ([miniconda](https://conda.io/miniconda.html), or the full [Anaconda distribution](https://www.anaconda.com/download)) from the `rapidsai` channel: ```bash -# for CUDA 11.5 conda install -c rapidsai -c conda-forge -c nvidia \ - cudf=22.10 python=3.9 cudatoolkit=11.5 -# for CUDA 11.2 -conda install -c rapidsai -c conda-forge -c nvidia \ - cudf=22.10 python=3.9 cudatoolkit=11.2 + cudf=23.02 python=3.10 cudatoolkit=11.8 ``` We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 0665b74144e..f1e17162195 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. set -euo pipefail @@ -9,7 +9,7 @@ rapids-logger "Generate notebook testing dependencies" rapids-dependency-file-generator \ --output conda \ --file_key test_notebooks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --force -f env.yaml -n test diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml similarity index 93% rename from conda/environments/all_cuda-115_arch-x86_64.yaml rename to conda/environments/all_cuda-118_arch-x86_64.yaml index 120f7cb013d..1bcb1978373 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -16,7 +16,7 @@ dependencies: - cmake>=3.23.1,!=3.25.0 - cubinlinker - cuda-python>=11.7.1,<12.0 -- cudatoolkit=11.5 +- cudatoolkit=11.8 - cupy>=9.5.0,<12.0.0a0 - cxx-compiler - cython>=0.29,<0.30 @@ -41,7 +41,7 @@ dependencies: - numba>=0.56.2 - numpy - numpydoc -- nvcc_linux-64=11.5 +- nvcc_linux-64=11.8 - nvtx>=0.2.1 - packaging - pandas>=1.0,<1.6.0dev0 @@ -60,7 +60,7 @@ dependencies: - pytest-xdist - python-confluent-kafka=1.7.0 - python-snappy>=0.6.0 -- python>=3.8,<3.10 +- python>=3.8,<3.11 - pytorch<1.12.0 - rmm=23.02.* - s3fs>=2022.3.0 @@ -78,4 +78,4 @@ dependencies: - typing_extensions - pip: - git+https://github.com/python-streamz/streamz.git@master -name: all_cuda-115_arch-x86_64 +name: all_cuda-118_arch-x86_64 diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 54ded116106..fce3b848372 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -337,6 +337,7 @@ outputs: - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} + - {{ compiler('cuda') }} {{ cuda_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: diff --git a/dependencies.yaml b/dependencies.yaml index 5fbf6de2cb5..8790853fbb3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.5"] + cuda: ["11.8"] arch: [x86_64] includes: - build @@ -84,14 +84,14 @@ dependencies: matrices: - matrix: arch: x86_64 - cuda: "11.5" + cuda: "11.8" packages: - - &nvcc_amd64_115 nvcc_linux-64=11.5 + - nvcc_linux-64=11.8 - matrix: arch: aarch64 - cuda: "11.5" + cuda: "11.8" packages: - - &nvcc_aarch64_115 nvcc_linux-aarch64=11.5 + - nvcc_linux-aarch64=11.8 cudatoolkit: specific: - output_types: conda @@ -108,6 +108,10 @@ dependencies: cuda: "11.5" packages: - cudatoolkit=11.5 + - matrix: + cuda: "11.8" + packages: + - cudatoolkit=11.8 develop: common: - output_types: [conda, requirements] @@ -174,12 +178,22 @@ dependencies: arch: x86_64 cuda: "11.5" packages: - - *nvcc_amd64_115 + - nvcc_linux-64=11.5 - matrix: arch: aarch64 cuda: "11.5" packages: - - *nvcc_aarch64_115 + - nvcc_linux-aarch64=11.5 + - matrix: + arch: x86_64 + cuda: "11.8" + packages: + - nvcc_linux-64=11.8 + - matrix: + arch: aarch64 + cuda: "11.8" + packages: + - nvcc_linux-aarch64=11.8 notebooks: common: - output_types: [conda, requirements] @@ -200,8 +214,12 @@ dependencies: packages: - python=3.9 - matrix: + py: "3.10" packages: - - python>=3.8,<3.10 + - python=3.10 + - matrix: + packages: + - python>=3.8,<3.11 run: common: - output_types: [conda, requirements] @@ -261,6 +279,10 @@ dependencies: cuda: "11.5" packages: - cuda-nvtx=11.5 + - matrix: + cuda: "11.8" + packages: + - cuda-nvtx=11.8 test_python: common: - output_types: [conda, requirements] diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 1ea9e045a22..df7c627c3ce 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. import os @@ -69,6 +69,7 @@ def get_versions(): "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], cmdclass=versioneer.get_cmdclass(), include_package_data=True, diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 7e0cdd76e2d..6d8f954f337 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import os import shutil import sysconfig @@ -95,6 +95,7 @@ "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], # Include the separately-compiled shared library ext_modules=cythonize( diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py index 911f83ff6bd..2fe12a54855 100644 --- a/python/custreamz/setup.py +++ b/python/custreamz/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import versioneer from setuptools import find_packages, setup @@ -23,6 +23,7 @@ "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], packages=find_packages(include=["custreamz", "custreamz.*"]), cmdclass=versioneer.get_cmdclass(), diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 19583a15654..d807034d3d8 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. import os @@ -54,6 +54,7 @@ def get_versions(): "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], packages=find_packages(exclude=["tests", "tests.*"]), cmdclass=versioneer.get_cmdclass(), diff --git a/python/strings_udf/setup.py b/python/strings_udf/setup.py index 9eee3d670f7..5b4c503e032 100644 --- a/python/strings_udf/setup.py +++ b/python/strings_udf/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import os import re @@ -69,6 +69,7 @@ def get_cuda_version_from_header(cuda_include_dir, delimiter=""): "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], packages=find_packages(include=["strings_udf", "strings_udf.*"]), package_data={ diff --git a/python/strings_udf/strings_udf/__init__.py b/python/strings_udf/strings_udf/__init__.py index bf13b79ab90..b84743fd2d9 100644 --- a/python/strings_udf/strings_udf/__init__.py +++ b/python/strings_udf/strings_udf/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import glob import os @@ -15,7 +15,7 @@ logger = get_logger() # tracks the version of CUDA used to build the c++ and PTX components -STRINGS_UDF_PTX_VERSION = (11, 5) +STRINGS_UDF_PTX_VERSION = (11, 8) def _get_appropriate_file(sms, cc):