From 8b87915c8c5e068caef04be4eeeb9cf7ae27b488 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Wed, 27 Sep 2023 12:08:12 -0700 Subject: [PATCH] Add cuGraph devcontainers (#3838) This PR adds some [devcontainers](https://containers.dev/) to help simplify building the cuGraph C++ and Python libraries. It also adds an optional job to the `pr.yaml` to [build the cuGraph libs in each devcontainer](https://github.com/trxcllnt/cugraph/blob/fea/devcontainers/.github/workflows/pr.yaml#L113-L119), so the build caches are populated for devs by CI. A devcontainer can be launched by clicking the "Reopen in Container" button that VSCode shows when opening the repo (or by using the "Rebuild and Reopen in Container" command from the command palette): ![image](https://user-images.githubusercontent.com/178183/221771999-97ab29d5-e718-4e5f-b32f-2cdd51bba25c.png) Clicking this button will cause VSCode to prompt the user to select one of these devcontainer variants: ![image](https://github.com/rapidsai/rmm/assets/178183/68d4b264-4fc2-4008-92b6-cb4bdd19b29f) On startup, the devcontainer creates or updates the conda/pip environment using `cugraph/dependencies.yaml`. The envs/package caches are cached on the host via volume mounts, which are described in more detail in [`.devcontainer/README.md`](https://github.com/trxcllnt/cugraph/blob/fea/devcontainers/.devcontainer/README.md). The container includes convenience functions to clean, configure, and build the various cuGraph components: ```shell $ clean-cugraph-cpp # only cleans the C++ build dir $ clean-cugraph-python # only cleans the Python build dir $ clean-cugraph # cleans both C++ and Python build dirs $ configure-cugraph-cpp # only configures cugraph C++ lib $ build-cugraph-cpp # only builds cugraph C++ lib $ build-cugraph-python # only builds cugraph Python lib $ build-cugraph # builds both C++ and Python libs ``` * The C++ build script is a small wrapper around `cmake -S ~/cugraph/cpp -B ~/cugraph/cpp/build` and `cmake --build ~/cugraph/cpp/build` * The Python build script is a small wrapper around `pip install --editable ~/cugraph/cpp` Unlike `build.sh`, these convenience scripts *don't* install the libraries after building them. Instead, they automatically inject the correct arguments to build the C++ libraries from source and use their build dirs as package roots: ```shell $ cmake -S ~/cugraph/cpp -B ~/cugraph/cpp/build $ CMAKE_ARGS="-Dcugraph_ROOT=~/cugraph/cpp/build" \ # <-- this argument is automatic pip install -e ~/cugraph/cpp ``` Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Chuck Hastings (https://github.com/ChuckHastings) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cugraph/pull/3838 --- .devcontainer/Dockerfile | 33 +++ .devcontainer/README.md | 34 +++ .../cuda11.8-conda/devcontainer.json | 37 +++ .devcontainer/cuda11.8-pip/devcontainer.json | 37 +++ .../cuda12.0-conda/devcontainer.json | 37 +++ .devcontainer/cuda12.0-pip/devcontainer.json | 37 +++ .github/workflows/pr.yaml | 10 + .gitignore | 4 + ci/release/update-version.sh | 7 + .../all_cuda-118_arch-x86_64.yaml | 2 + .../all_cuda-120_arch-x86_64.yaml | 2 + cpp/.clangd | 65 +++++ dependencies.yaml | 263 +++++++++++++++--- 13 files changed, 527 insertions(+), 41 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/README.md create mode 100644 .devcontainer/cuda11.8-conda/devcontainer.json create mode 100644 .devcontainer/cuda11.8-pip/devcontainer.json create mode 100644 .devcontainer/cuda12.0-conda/devcontainer.json create mode 100644 .devcontainer/cuda12.0-pip/devcontainer.json create mode 100644 cpp/.clangd diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000000..3d0ac075be3 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1.5 + +ARG BASE +ARG PYTHON_PACKAGE_MANAGER=conda + +FROM ${BASE} as pip-base + +ENV DEFAULT_VIRTUAL_ENV=rapids + +FROM ${BASE} as conda-base + +ENV DEFAULT_CONDA_ENV=rapids + +FROM ${PYTHON_PACKAGE_MANAGER}-base + +ARG CUDA +ENV CUDAARCHS="RAPIDS" +ENV CUDA_VERSION="${CUDA_VERSION:-${CUDA}}" + +ARG PYTHON_PACKAGE_MANAGER +ENV PYTHON_PACKAGE_MANAGER="${PYTHON_PACKAGE_MANAGER}" + +ENV PYTHONSAFEPATH="1" +ENV PYTHONUNBUFFERED="1" +ENV PYTHONDONTWRITEBYTECODE="1" + +ENV SCCACHE_REGION="us-east-2" +ENV SCCACHE_BUCKET="rapids-sccache-devs" +ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai" +ENV HISTFILE="/home/coder/.cache/._bash_history" + +# cugraph_pyg's setup.py needs this defined when building in a conda env +ENV CUDA_HOME="${CUDA_HOME:-/home/coder/.conda/envs/$DEFAULT_CONDA_ENV}" diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000000..e645c51de8b --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,34 @@ +# cuGraph Development Containers + +This directory contains [devcontainer configurations](https://containers.dev/implementors/json_reference/) for using VSCode to [develop in a container](https://code.visualstudio.com/docs/devcontainers/containers) via the `Remote Containers` [extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) or [GitHub Codespaces](https://github.com/codespaces). + +This container is a turnkey development environment for building and testing the cuGraph C++ and Python libraries. + +## Table of Contents + +* [Prerequisites](#prerequisites) +* [Host bind mounts](#host-bind-mounts) +* [Launch a Dev Container](#launch-a-dev-container) +## Prerequisites + +* [VSCode](https://code.visualstudio.com/download) +* [VSCode Remote Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +## Host bind mounts + +By default, the following directories are bind-mounted into the devcontainer: + +* `${repo}:/home/coder/cugraph` +* `${repo}/../.aws:/home/coder/.aws` +* `${repo}/../.local:/home/coder/.local` +* `${repo}/../.cache:/home/coder/.cache` +* `${repo}/../.conda:/home/coder/.conda` +* `${repo}/../.config:/home/coder/.config` + +This ensures caches, configurations, dependencies, and your commits are persisted on the host across container runs. + +## Launch a Dev Container + +To launch a devcontainer from VSCode, open the cuGraph repo and select the "Reopen in Container" button in the bottom right:
+ +Alternatively, open the VSCode command palette (typically `cmd/ctrl + shift + P`) and run the "Rebuild and Reopen in Container" command. diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json new file mode 100644 index 00000000000..cf4ba5aa114 --- /dev/null +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "11.8", + "PYTHON_PACKAGE_MANAGER": "conda", + "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cugraph,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda11.8-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json new file mode 100644 index 00000000000..e86a38abbde --- /dev/null +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "11.8", + "PYTHON_PACKAGE_MANAGER": "pip", + "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cugraph,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json new file mode 100644 index 00000000000..863eeea48ff --- /dev/null +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "12.0", + "PYTHON_PACKAGE_MANAGER": "conda", + "BASE": "rapidsai/devcontainers:23.10-cpp-mambaforge-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cugraph,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.0-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json new file mode 100644 index 00000000000..c7612771fd3 --- /dev/null +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "12.0", + "PYTHON_PACKAGE_MANAGER": "pip", + "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda12.0-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cugraph,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d49ae7f8d3d..7b267d7edf3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,7 @@ jobs: - wheel-tests-cugraph - wheel-build-nx-cugraph - wheel-tests-nx-cugraph + - devcontainer secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.10 checks: @@ -125,3 +126,12 @@ jobs: with: build_type: pull-request script: ci/test_wheel_nx-cugraph.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-23.10 + with: + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + build_command: | + sccache -z; + build-all --verbose; + sccache -s; diff --git a/.gitignore b/.gitignore index 3fda9f8a037..c6bcf6965d7 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,7 @@ python/cugraph/cugraph/tests/dask-worker-space docs/cugraph/source/api_docs/api/* _html _text + +# clang tooling +compile_commands.json +.clangd/ diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index bd3aa6bc370..f3892fbd3c4 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -128,3 +128,10 @@ done sed_runner "s/RAPIDS_VERSION_NUMBER=\".*/RAPIDS_VERSION_NUMBER=\"${NEXT_SHORT_TAG}\"/g" ci/build_docs.sh sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" python/nx-cugraph/README.md + +# .devcontainer files +find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r -d '' filename; do + sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}" + sed_runner "s@rapidsai/devcontainers/features/ucx:[0-9.]*@rapidsai/devcontainers/features/ucx:${NEXT_SHORT_TAG_PEP440}@" "${filename}" + sed_runner "s@rapidsai/devcontainers/features/rapids-build-utils:[0-9.]*@rapidsai/devcontainers/features/rapids-build-utils:${NEXT_SHORT_TAG_PEP440}@" "${filename}" +done diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index c66890f8ae5..86de24c991d 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -66,10 +66,12 @@ dependencies: - scikit-build>=0.13.1 - scikit-learn>=0.23.1 - scipy +- setuptools>=61.0.0 - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport - ucx-proc=*=gpu - ucx-py==0.34.* +- wheel name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 3afb1415572..1054f75ba54 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -65,10 +65,12 @@ dependencies: - scikit-build>=0.13.1 - scikit-learn>=0.23.1 - scipy +- setuptools>=61.0.0 - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport - ucx-proc=*=gpu - ucx-py==0.34.* +- wheel name: all_cuda-120_arch-x86_64 diff --git a/cpp/.clangd b/cpp/.clangd new file mode 100644 index 00000000000..7c4fe036ddf --- /dev/null +++ b/cpp/.clangd @@ -0,0 +1,65 @@ +# https://clangd.llvm.org/config + +# Apply a config conditionally to all C files +If: + PathMatch: .*\.(c|h)$ + +--- + +# Apply a config conditionally to all C++ files +If: + PathMatch: .*\.(c|h)pp + +--- + +# Apply a config conditionally to all CUDA files +If: + PathMatch: .*\.cuh? +CompileFlags: + Add: + - "-x" + - "cuda" + # No error on unknown CUDA versions + - "-Wno-unknown-cuda-version" + # Allow variadic CUDA functions + - "-Xclang=-fcuda-allow-variadic-functions" +Diagnostics: + Suppress: + - "variadic_device_fn" + - "attributes_not_allowed" + +--- + +# Tweak the clangd parse settings for all files +CompileFlags: + Add: + # report all errors + - "-ferror-limit=0" + - "-fmacro-backtrace-limit=0" + - "-ftemplate-backtrace-limit=0" + # Skip the CUDA version check + - "--no-cuda-version-check" + Remove: + # remove gcc's -fcoroutines + - -fcoroutines + # remove nvc++ flags unknown to clang + - "-gpu=*" + - "-stdpar*" + # remove nvcc flags unknown to clang + - "-arch*" + - "-gencode*" + - "--generate-code*" + - "-ccbin*" + - "-t=*" + - "--threads*" + - "-Xptxas*" + - "-Xcudafe*" + - "-Xfatbin*" + - "-Xcompiler*" + - "--diag-suppress*" + - "--diag_suppress*" + - "--compiler-options*" + - "--expt-extended-lambda" + - "--expt-relaxed-constexpr" + - "-forward-unknown-to-host-compiler" + - "-Werror=cross-execution-space-call" diff --git a/dependencies.yaml b/dependencies.yaml index 04ec1b6e957..a162ac01354 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -11,9 +11,15 @@ files: - cpp_build - cudatoolkit - docs + - python_build_wheel - python_build_cythonize + - depends_on_rmm + - depends_on_cudf + - depends_on_dask_cudf + - depends_on_pylibraft + - depends_on_raft_dask + - depends_on_cupy - python_run_cugraph - - python_run_pylibcugraph - python_run_nx_cugraph - python_run_cugraph_dgl - python_run_cugraph_pyg @@ -50,6 +56,7 @@ files: output: none includes: - cudatoolkit + - depends_on_cudf - py_version - test_python_common - test_python_cugraph @@ -62,14 +69,22 @@ files: includes: - common_build - python_build_wheel + - depends_on_rmm + - depends_on_pylibraft + - depends_on_pylibcugraph - python_build_cythonize - - python_build_cugraph py_run_cugraph: output: pyproject pyproject_dir: python/cugraph extras: table: project includes: + - depends_on_rmm + - depends_on_cudf + - depends_on_dask_cudf + - depends_on_raft_dask + - depends_on_pylibcugraph + - depends_on_cupy - python_run_cugraph py_test_cugraph: output: pyproject @@ -88,6 +103,8 @@ files: includes: - common_build - python_build_wheel + - depends_on_rmm + - depends_on_pylibraft - python_build_cythonize py_run_pylibcugraph: output: pyproject @@ -95,7 +112,8 @@ files: extras: table: project includes: - - python_run_pylibcugraph + - depends_on_rmm + - depends_on_pylibraft py_test_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -103,6 +121,7 @@ files: table: project.optional-dependencies key: test includes: + - depends_on_cudf - test_python_common - test_python_pylibcugraph py_build_nx_cugraph: @@ -118,6 +137,8 @@ files: extras: table: project includes: + - depends_on_pylibcugraph + - depends_on_cupy - python_run_nx_cugraph py_test_nx_cugraph: output: pyproject @@ -183,6 +204,10 @@ files: extras: table: project includes: + - depends_on_rmm + - depends_on_cudf + - depends_on_dask_cudf + - depends_on_cupy - python_run_cugraph_service_server py_test_cugraph_service_server: output: pyproject @@ -334,41 +359,29 @@ dependencies: - python>=3.9,<3.11 python_build_wheel: common: - - output_types: [conda, pyproject] + - output_types: [conda, pyproject, requirements] packages: - - wheel - setuptools>=61.0.0 + - wheel python_build_cythonize: common: - - output_types: [conda, pyproject] + - output_types: [conda, pyproject, requirements] packages: - cython>=3.0.0 - - &pylibraft pylibraft==23.10.* - - &rmm rmm==23.10.* - scikit-build>=0.13.1 - python_build_cugraph: - common: - - output_types: [conda, pyproject] - packages: - - &pylibcugraph pylibcugraph==23.10.* python_run_cugraph: common: - output_types: [conda, pyproject] packages: - - &cudf cudf==23.10.* - &dask dask>=2023.7.1 - &distributed distributed>=2023.7.1 - &dask_cuda dask-cuda==23.10.* - - &dask_cudf dask-cudf==23.10.* - &numba numba>=0.57 - - raft-dask==23.10.* - - *rmm - &ucx_py ucx-py==0.34.* - output_types: conda packages: - aiohttp - - &cupy cupy>=12.0.0 - - &dask-core dask-core>=2023.7.1 + - &dask-core_conda dask-core>=2023.7.1 - fsspec>=0.6.0 - libcudf==23.10.* - requests @@ -376,29 +389,14 @@ dependencies: - ucx-proc=*=gpu - output_types: pyproject packages: - - &cupy_pip cupy-cuda11x>=12.0.0 # cudf uses fsspec but is protocol independent. cugraph # dataset APIs require [http] extras for use with cudf. - fsspec[http]>=0.6.0 - - *pylibcugraph - python_run_pylibcugraph: - common: - - output_types: [conda, pyproject] - packages: - - *pylibraft - - *rmm python_run_nx_cugraph: common: - output_types: [conda, pyproject] packages: - networkx>=3.0 - - output_types: conda - packages: - - *cupy - - output_types: pyproject - packages: - - *cupy_pip - - *pylibcugraph python_run_cugraph_dgl: common: - output_types: [conda, pyproject] @@ -426,23 +424,18 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - *cudf - *dask - *dask_cuda - - *dask_cudf - *distributed - *numba - *numpy - - *rmm - *thrift - *ucx_py - output_types: conda packages: - - *cupy - - *dask-core + - *dask-core_conda - output_types: pyproject packages: - - *cupy_pip - *cugraph - cugraph-service-client==23.10.* doc: @@ -492,7 +485,6 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - *cudf - *numpy test_python_nx_cugraph: common: @@ -519,3 +511,192 @@ dependencies: - pytorch==2.0 - pytorch-cuda==11.8 - pyg=2.3.1=*torch_2.0.0*cu118* + + depends_on_rmm: + common: + - output_types: conda + packages: + - &rmm_conda rmm==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &rmm_packages_pip_cu12 + - rmm-cu12==23.10.* + - {matrix: {cuda: "12.1"}, packages: *rmm_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *rmm_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &rmm_packages_pip_cu11 + - rmm-cu11==23.10.* + - {matrix: {cuda: "11.5"}, packages: *rmm_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *rmm_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *rmm_packages_pip_cu11} + - {matrix: null, packages: [*rmm_conda]} + + depends_on_cudf: + common: + - output_types: conda + packages: + - &cudf_conda cudf==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &cudf_packages_pip_cu12 + - cudf-cu12==23.10.* + - {matrix: {cuda: "12.1"}, packages: *cudf_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *cudf_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &cudf_packages_pip_cu11 + - cudf-cu11==23.10.* + - {matrix: {cuda: "11.5"}, packages: *cudf_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *cudf_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *cudf_packages_pip_cu11} + - {matrix: null, packages: [*cudf_conda]} + + depends_on_dask_cudf: + common: + - output_types: conda + packages: + - &dask_cudf_conda dask-cudf==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &dask_cudf_packages_pip_cu12 + - dask-cudf-cu12==23.10.* + - {matrix: {cuda: "12.1"}, packages: *dask_cudf_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *dask_cudf_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &dask_cudf_packages_pip_cu11 + - dask-cudf-cu11==23.10.* + - {matrix: {cuda: "11.5"}, packages: *dask_cudf_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *dask_cudf_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *dask_cudf_packages_pip_cu11} + - {matrix: null, packages: [*dask_cudf_conda]} + + depends_on_pylibraft: + common: + - output_types: conda + packages: + - &pylibraft_conda pylibraft==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &pylibraft_packages_pip_cu12 + - pylibraft-cu12==23.10.* + - {matrix: {cuda: "12.1"}, packages: *pylibraft_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *pylibraft_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &pylibraft_packages_pip_cu11 + - pylibraft-cu11==23.10.* + - {matrix: {cuda: "11.5"}, packages: *pylibraft_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *pylibraft_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *pylibraft_packages_pip_cu11} + - {matrix: null, packages: [*pylibraft_conda]} + + depends_on_raft_dask: + common: + - output_types: conda + packages: + - &raft_dask_conda raft-dask==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &raft_dask_packages_pip_cu12 + - raft-dask-cu12==23.10.* + - {matrix: {cuda: "12.1"}, packages: *raft_dask_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *raft_dask_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &raft_dask_packages_pip_cu11 + - raft-dask-cu11==23.10.* + - {matrix: {cuda: "11.5"}, packages: *raft_dask_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *raft_dask_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *raft_dask_packages_pip_cu11} + - {matrix: null, packages: [*raft_dask_conda]} + + depends_on_pylibcugraph: + common: + - output_types: conda + packages: + - &pylibcugraph_conda pylibcugraph==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &pylibcugraph_packages_pip_cu12 + - pylibcugraph-cu12==23.10.* + - {matrix: {cuda: "12.1"}, packages: *pylibcugraph_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *pylibcugraph_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &pylibcugraph_packages_pip_cu11 + - pylibcugraph-cu11==23.10.* + - {matrix: {cuda: "11.5"}, packages: *pylibcugraph_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *pylibcugraph_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *pylibcugraph_packages_pip_cu11} + - {matrix: null, packages: [*pylibcugraph_conda]} + + depends_on_cupy: + common: + - output_types: conda + packages: + - cupy>=12.0.0 + specific: + - output_types: [requirements, pyproject] + matrices: + # All CUDA 12 + x86_64 versions + - matrix: {cuda: "12.2", arch: x86_64} + packages: &cupy_packages_cu12_x86_64 + - cupy-cuda12x>=12.0.0 + - {matrix: {cuda: "12.1", arch: x86_64}, packages: *cupy_packages_cu12_x86_64} + - {matrix: {cuda: "12.0", arch: x86_64}, packages: *cupy_packages_cu12_x86_64} + + # All CUDA 12 + aarch64 versions + - matrix: {cuda: "12.2", arch: aarch64} + packages: &cupy_packages_cu12_aarch64 + - cupy-cuda12x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. + - {matrix: {cuda: "12.1", arch: aarch64}, packages: *cupy_packages_cu12_aarch64} + - {matrix: {cuda: "12.0", arch: aarch64}, packages: *cupy_packages_cu12_aarch64} + + # All CUDA 11 + x86_64 versions + - matrix: {cuda: "11.8", arch: x86_64} + packages: &cupy_packages_cu11_x86_64 + - cupy-cuda11x>=12.0.0 + - {matrix: {cuda: "11.5", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} + - {matrix: {cuda: "11.4", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} + - {matrix: {cuda: "11.2", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} + + # All CUDA 11 + aarch64 versions + - matrix: {cuda: "11.8", arch: aarch64} + packages: &cupy_packages_cu11_aarch64 + - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. + - {matrix: {cuda: "11.5", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} + - {matrix: {cuda: "11.4", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} + - {matrix: {cuda: "11.2", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} + - {matrix: null, packages: [cupy-cuda11x>=12.0.0]}