diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..03ded27 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,114 @@ +name: build + +on: + push: + branches: + - "branch-*" + tags: + - v[0-9][0-9].[0-9][0-9].[0-9][0-9] + workflow_dispatch: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + build_type: + type: string + default: nightly + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + cpp-build: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: cpu32 + python-build: + needs: [cpp-build] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + upload-conda: + needs: [cpp-build, python-build] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + wheel-build-cugraph-dgl: + needs: wheel-publish-pylibwholegraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cugraph-dgl.sh + wheel-publish-cugraph-dgl: + needs: wheel-build-cugraph-dgl + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph-dgl + wheel-build-cugraph-pyg: + needs: wheel-publish-pylibwholegraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cugraph-pyg.sh + wheel-publish-cugraph-pyg: + needs: wheel-build-cugraph-pyg + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph-pyg + wheel-build-pylibwholegraph: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_pylibwholegraph.sh + wheel-publish-pylibwholegraph: + needs: wheel-build-pylibwholegraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: pylibwholegraph diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8628d46..06fee99 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,10 +16,17 @@ jobs: - checks - conda-cpp-build - conda-cpp-tests + # - conda-cpp-checks + - conda-notebook-tests - conda-python-build - conda-python-tests - wheel-build-pylibwholegraph - wheel-tests-pylibwholegraph + - wheel-build-cugraph-dgl + - wheel-tests-cugraph-dgl + - wheel-build-cugraph-pyg + - wheel-tests-cugraph-pyg + - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() @@ -74,6 +81,14 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + # with: + # build_type: pull-request + # enable_check_symbols: true + # symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) conda-python-build: needs: conda-cpp-build secrets: inherit @@ -87,6 +102,17 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request + conda-notebook-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + run_script: "ci/test_notebooks.sh" wheel-build-pylibwholegraph: needs: checks secrets: inherit @@ -104,3 +130,46 @@ jobs: build_type: pull-request script: ci/test_wheel_pylibwholegraph.sh matrix_filter: map(select(.ARCH == "amd64")) + wheel-build-cugraph-dgl: + needs: wheel-tests-pylibwholegraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: pull-request + script: ci/build_wheel_cugraph-dgl.sh + wheel-tests-cugraph-dgl: + needs: [wheel-build-cugraph-dgl, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph-dgl.sh + matrix_filter: map(select(.ARCH == "amd64")) + wheel-build-cugraph-pyg: + needs: wheel-tests-pylibwholegraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 + with: + build_type: pull-request + script: ci/build_wheel_cugraph-pyg.sh + wheel-tests-cugraph-pyg: + needs: [wheel-build-cugraph-pyg, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph-pyg.sh + matrix_filter: map(select(.ARCH == "amd64")) + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + node_type: cpu32 + build_command: | + sccache -z; + build-all --verbose -j$(nproc --ignore=1); + sccache -s; diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..187f4b4 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,72 @@ +name: test + +on: + workflow_dispatch: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + +jobs: + conda-cpp-checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + enable_check_symbols: true + symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) + conda-cpp-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + conda-python-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + wheel-tests-pylibwholegraph: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_pylibwholegraph.sh + matrix_filter: map(select(.ARCH == "amd64")) + wheel-tests-cugraph-dgl: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_cugraph-dgl.sh + matrix_filter: map(select(.ARCH == "amd64")) + wheel-tests-cugraph-pyg: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel_cugraph-pyg.sh + matrix_filter: map(select(.ARCH == "amd64")) diff --git a/build.sh b/build.sh index edece26..435567b 100755 --- a/build.sh +++ b/build.sh @@ -29,12 +29,14 @@ VALIDARGS=" pylibwholegraph libwholegraph tests + benchmarks all -v -g -n --pydevelop --allgpuarch + --compile-cmd --clean -h --help @@ -45,19 +47,22 @@ HELP="$0 [ ...] [ ...] clean - remove all existing build artifacts and configuration (start over) uninstall - uninstall libcugraph and cugraph from a prior build/install (see also -n) cugraph-pyg - build the cugraph-pyg Python package - cugraph-dgl - build the cugraph-dgl extensions for DGL + cugraph-dgl - build the cugraph-dgl Python package pylibwholegraph - build the pylibwholegraph Python package libwholegraph - build the libwholegraph library tests - build the C++ tests + benchmarks - build benchmarks. all - build everything and is: - -v - verbose build mode - -g - build for debug - -n - do not install after a successful build (does not affect Python packages) - --pydevelop - install the Python packages in editable mode - --allgpuarch - build for all supported GPU architectures - --clean - clean an individual target (note: to do a complete rebuild, use the clean target described above) - -h - print this text + -v - verbose build mode + -g - build for debug + -n - do not install after a successful build (does not affect Python packages) + --pydevelop - install the Python packages in editable mode + --allgpuarch - build for all supported GPU architectures + --enable-nvshmem - build with nvshmem support (beta). + --compile-cmd - only output compile commands (invoke CMake without build) + --clean - clean an individual target (note: to do a complete rebuild, use the clean target described above) + -h - print this text default action (no args) is to build and install 'libwholegraph' then 'pylibwholegraph' then 'cugraph-pyg' then 'cugraph-dgl' @@ -139,6 +144,21 @@ fi if hasArg --pydevelop; then PYTHON_ARGS_FOR_INSTALL="${PYTHON_ARGS_FOR_INSTALL} -e" fi +if hasArg --enable-nvshmem; then + BUILD_WITH_NVSHMEM=ON +else + BUILD_WITH_NVSHMEM=OFF +fi +if hasArg tests; then + BUILD_TESTS=ON +else + BUILD_TESTS=OFF +fi +if hasArg benchmarks; then + BUILD_BENCHMARKS=ON +else + BUILD_BENCHMARKS=OFF +fi if hasArg tests; then BUILD_TESTS=ON @@ -189,6 +209,7 @@ if hasArg clean; then fi ################################################################################ + # Build and install the libwholegraph library if hasArg libwholegraph || buildDefault || hasArg all ; then @@ -250,7 +271,7 @@ if hasArg cugraph-pyg || buildDefault || hasArg all; then fi fi -# Install the cugraph-dgl extensions for DGL +# Build and install the cugraph-dgl Python package if hasArg cugraph-dgl || buildDefault ||hasArg all; then if hasArg --clean; then cleanPythonDir ${REPODIR}/python/cugraph-dgl diff --git a/ci/build_python.sh b/ci/build_python.sh index 49d3c12..9f42948 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -25,4 +25,16 @@ RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/pylibwholegraph +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + conda/recipes/cugraph-pyg + +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + conda/recipes/cugraph-dgl + rapids-upload-conda-to-s3 python diff --git a/ci/run_cugraph_dgl_pytests.sh b/ci/run_cugraph_dgl_pytests.sh index 83c26a5..e85eb04 100755 --- a/ci/run_cugraph_dgl_pytests.sh +++ b/ci/run_cugraph_dgl_pytests.sh @@ -4,6 +4,6 @@ set -euo pipefail # Support invoking run_cugraph_dgl_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/tests +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cugraph-dgl/cugraph_dgl pytest --cache-clear --ignore=mg "$@" . diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 31ec560..59b60e4 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -5,6 +5,8 @@ set -Eeuo pipefail . /opt/conda/etc/profile.d/conda.sh +RAPIDS_VERSION="$(rapids-version)" + rapids-logger "Generate notebook testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -27,7 +29,9 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libcugraph pylibcugraph cugraph + "libcugraph=${RAPIDS_VERSION}" \ + "pylibcugraph=${RAPIDS_VERSION}" \ + "cugraph=${RAPIDS_VERSION}" NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" NOTEBOOK_LIST="$(realpath "$(dirname "$0")/notebook_list.py")" diff --git a/ci/test_python.sh b/ci/test_python.sh index dd5f539..68cdf2c 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -27,10 +27,113 @@ RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" +# RAPIDS_DATASET_ROOT_DIR is used by test scripts +export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" +mkdir -p "${RAPIDS_DATASET_ROOT_DIR}" +pushd "${RAPIDS_DATASET_ROOT_DIR}" +./get_test_data.sh --benchmark +popd + EXITCODE=0 trap "EXITCODE=1" ERR set +e +# Test runs that include tests that use dask require +# --import-mode=append. Those tests start a LocalCUDACluster that inherits +# changes from pytest's modifications to PYTHONPATH (which defaults to +# prepending source tree paths to PYTHONPATH). This causes the +# LocalCUDACluster subprocess to import cugraph from the source tree instead of +# the install location, and in most cases, the source tree does not have +# extensions built in-place and will result in ImportErrors. +# +# FIXME: TEMPORARILY disable MG PropertyGraph tests (experimental) tests and +# bulk sampler IO tests (hangs in CI) + +if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_dgl + + # activate test_cugraph_dgl environment for dgl + set +u + conda activate test_cugraph_dgl + set -u + + rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + --channel pytorch \ + --channel conda-forge \ + --channel dglteam/label/th23_cu118 \ + --channel nvidia \ + "pylibwholegraph=${RAPIDS_VERSION}" \ + "pylibcugraphops=${RAPIDS_VERSION}" \ + "cugraph=${RAPIDS_VERSION}" \ + "cugraph-dgl=${RAPIDS_VERSION}" \ + 'pytorch::pytorch>=2.3,<2.4' \ + 'cuda-version=11.8' \ + "ogb" + + rapids-print-env + + rapids-logger "Check GPU usage" + nvidia-smi + + rapids-logger "pytest cugraph_dgl (single GPU)" + ./ci/run_cugraph_dgl_pytests.sh \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-dgl.xml" \ + --cov-config=../../.coveragerc \ + --cov=cugraph_dgl \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-dgl-coverage.xml" \ + --cov-report=term + + # Reactivate the test environment back + set +u + conda deactivate + set -u +else + rapids-logger "skipping cugraph_dgl pytest on ARM64" +fi + +if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then + rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_pyg + + # Temporarily allow unbound variables for conda activation. + set +u + conda activate test_cugraph_pyg + set -u + + # Will automatically install built dependencies of cuGraph-PyG + rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + --channel pytorch \ + "pylibwholegraph=${RAPIDS_VERSION}" \ + "pylibcugraphops=${RAPIDS_VERSION}" \ + "cugraph=${RAPIDS_VERSION}" \ + "cugraph-pyg=${RAPIDS_VERSION}" \ + 'pytorch::pytorch>=2.3,<2.4' \ + 'ogb' + + rapids-print-env + + rapids-logger "Check GPU usage" + nvidia-smi + + rapids-logger "pytest cugraph_pyg (single GPU)" + ./ci/run_cugraph_pyg_pytests.sh \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-pyg.xml" \ + --cov-config=../../.coveragerc \ + --cov=cugraph_pyg \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-pyg-coverage.xml" \ + --cov-report=term + + # Reactivate the test environment back + set +u + conda deactivate + set -u +else + rapids-logger "skipping cugraph_pyg pytest on ARM64" +fi + if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then rapids-mamba-retry env create --yes -f env.yaml -n test_pylibwholegraph diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 564b46c..d709145 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -4,24 +4,15 @@ set -eoxu pipefail package_name="cugraph-dgl" -package_dir="python/cugraph-dgl" - -python_package_name=$(echo ${package_name}|sed 's/-/_/g') mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download wheels built during this job. -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -python -m pip install ./local-deps/*.whl - -# use 'ls' to expand wildcard before adding `[extra]` requires for pip +# Download the pylibwholegraph and cugraph-dgl built in the previous step +RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] - +# determine pytorch and DGL sources PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then @@ -30,20 +21,16 @@ else PYTORCH_CUDA_VER=$PKG_CUDA_VER fi PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}" -DGL_URL="https://data.dgl.ai/wheels/cu${PYTORCH_CUDA_VER}/repo.html" - -# Starting from 2.2, PyTorch wheels depend on nvidia-nccl-cuxx>=2.19 wheel and -# dynamically link to NCCL. RAPIDS CUDA 11 CI images have an older NCCL version that -# might shadow the newer NCCL required by PyTorch during import (when importing -# `cupy` before `torch`). -if [[ "${NCCL_VERSION}" < "2.19" ]]; then - PYTORCH_VER="2.1.0" -else - PYTORCH_VER="2.3.0" -fi - -rapids-logger "Installing PyTorch and DGL" -rapids-retry python -m pip install "torch==${PYTORCH_VER}" --index-url ${PYTORCH_URL} -rapids-retry python -m pip install dgl==2.0.0 --find-links ${DGL_URL} - -python -m pytest python/cugraph-dgl/tests +DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html" + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + -v \ + --extra-index-url "${PYTORCH_URL}" \ + --find-links "${DGL_URL}" \ + "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + 'dgl==2.4.0' \ + 'torch>=2.3.0,<2.4' + +python -m pytest python/cugraph-dgl/cugraph_dgl/tests diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 1004063..3a4bf5f 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -4,46 +4,42 @@ set -eoxu pipefail package_name="cugraph-pyg" -package_dir="python/cugraph-pyg" - -python_package_name=$(echo ${package_name}|sed 's/-/_/g') mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download wheels built during this job. -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -python -m pip install ./local-deps/*.whl - -# use 'ls' to expand wildcard before adding `[extra]` requires for pip +# Download the pylibwholegraph and cugraph-pyg built in the previous step +RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# pip creates wheels using python package names -python -m pip install $(ls ./dist/${python_package_name}*.whl)[test] - -# RAPIDS_DATASET_ROOT_DIR is used by test scripts -export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" - -# Used to skip certain examples in CI due to memory limitations -export CI_RUN=1 +# determine pytorch and pyg sources if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then PYTORCH_URL="https://download.pytorch.org/whl/cu118" - PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html" + PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html" else PYTORCH_URL="https://download.pytorch.org/whl/cu121" - PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html" + PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html" fi -rapids-logger "Installing PyTorch and PyG dependencies" -rapids-retry python -m pip install torch==2.1.0 --index-url ${PYTORCH_URL} -rapids-retry python -m pip install "torch-geometric>=2.5,<2.6" -rapids-retry python -m pip install \ - ogb \ - pyg_lib \ - torch_scatter \ - torch_sparse \ - tensordict \ - -f ${PYG_URL} + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + -v \ + --extra-index-url "${PYTORCH_URL}" \ + --find-links "${PYG_URL}" \ + "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + 'ogb' \ + 'pyg_lib' \ + 'torch>=2.3.0,<2.4' \ + 'torch-geometric>=2.5,<2.6' \ + 'torch_scatter' \ + 'torch_sparse' + +# RAPIDS_DATASET_ROOT_DIR is used by test scripts +export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" + +# Used to skip certain examples in CI due to memory limitations +export CI_RUN=1 rapids-logger "pytest cugraph-pyg (single GPU)" pushd python/cugraph-pyg/cugraph_pyg diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 91af633..faf7d28 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -60,7 +60,7 @@ for nb in $*; do echo -------------------------------------------------------------------------------- echo STARTING: ${NBNAME} echo -------------------------------------------------------------------------------- - jupyter nbconvert --to script ${NBFILENAME} --output ${NBTMPDIR}/${NBNAME}-test + jupyter nbconvert --to python ${NBFILENAME} --output ${NBTMPDIR}/${NBNAME}-test echo "${MAGIC_OVERRIDE_CODE}" > ${NBTMPDIR}/tmpfile cat ${NBTESTSCRIPT} >> ${NBTMPDIR}/tmpfile mv ${NBTMPDIR}/tmpfile ${NBTESTSCRIPT} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 6383b57..56438bf 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -9,19 +9,25 @@ channels: - nvidia dependencies: - breathe +- c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-nvtx - cuda-version=11.8 - cudatoolkit - cudf==24.12.*,>=0.0.0a0 +- cudnn=8.8 - cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 +- cxx-compiler - cython>=3.0.0 - dask-cudf==24.12.*,>=0.0.0a0 -- dglteam/label/th21_cu118::dgl - doxygen - graphviz - ipython +- libraft-headers==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 +- nanobind>=0.2.0 +- nbconvert>=7.16 - nbsphinx - nccl>=2.19 - ninja @@ -33,7 +39,6 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pyg::pyg - pylibcugraphops==24.12.*,>=0.0.0a0 - pylibraft==24.12.*,>=0.0.0a0 - pytest @@ -41,13 +46,13 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=11.8 -- pytorch::pytorch>=2.0,<2.4.0a0 +- pytorch::pytorch>=2.3,<2.4.0a0 - raft-dask==24.12.*,>=0.0.0a0 - recommonmark - rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy +- setuptools - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 7429b0d..95aba20 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -9,17 +9,19 @@ channels: - nvidia dependencies: - breathe +- c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.1 - cudf==24.12.*,>=0.0.0a0 +- cudnn=8.8 - cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 +- cxx-compiler - cython>=3.0.0 - dask-cudf==24.12.*,>=0.0.0a0 -- dglteam/label/th21_cu121::dgl - doxygen - graphviz - ipython @@ -27,6 +29,10 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libraft-headers==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 +- nanobind>=0.2.0 +- nbconvert>=7.16 - nbsphinx - nccl>=2.19 - ninja @@ -38,7 +44,6 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pyg::pyg - pylibcugraphops==24.12.*,>=0.0.0a0 - pylibraft==24.12.*,>=0.0.0a0 - pytest @@ -46,13 +51,13 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=12.1 -- pytorch::pytorch>=2.0,<2.4.0a0 +- pytorch::pytorch>=2.3,<2.4.0a0 - raft-dask==24.12.*,>=0.0.0a0 - recommonmark - rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy +- setuptools - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/conda/environments/all_cuda-124_arch-x86_64.yaml b/conda/environments/all_cuda-124_arch-x86_64.yaml index 08c0ee6..214688f 100644 --- a/conda/environments/all_cuda-124_arch-x86_64.yaml +++ b/conda/environments/all_cuda-124_arch-x86_64.yaml @@ -9,17 +9,19 @@ channels: - nvidia dependencies: - breathe +- c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.4 - cudf==24.12.*,>=0.0.0a0 +- cudnn=8.8 - cugraph==24.12.*,>=0.0.0a0 - cupy>=12.0.0 +- cxx-compiler - cython>=3.0.0 - dask-cudf==24.12.*,>=0.0.0a0 -- dglteam/label/th21_cu121::dgl - doxygen - graphviz - ipython @@ -27,6 +29,10 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libraft-headers==24.12.*,>=0.0.0a0 +- librmm==24.12.*,>=0.0.0a0 +- nanobind>=0.2.0 +- nbconvert>=7.16 - nbsphinx - nccl>=2.19 - ninja @@ -38,7 +44,6 @@ dependencies: - pre-commit - pydantic - pydata-sphinx-theme -- pyg::pyg - pylibcugraphops==24.12.*,>=0.0.0a0 - pylibraft==24.12.*,>=0.0.0a0 - pytest @@ -46,13 +51,13 @@ dependencies: - pytest-cov - pytest-forked - pytest-xdist -- pytorch-cuda=12.1 -- pytorch::pytorch>=2.0,<2.4.0a0 +- pytorch::pytorch>=2.3,<2.4.0a0 - raft-dask==24.12.*,>=0.0.0a0 - recommonmark - rmm==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - scipy +- setuptools - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml index 3c90075..f08c831 100644 --- a/conda/recipes/cugraph-dgl/meta.yaml +++ b/conda/recipes/cugraph-dgl/meta.yaml @@ -21,15 +21,17 @@ build: requirements: host: - python + - rapids-build-backend>=0.3.1,<0.4.0.dev0 + - setuptools>=61.0.0 run: - cugraph ={{ version }} - - dgl >=1.1.0.cu* + - dgl >=2.4.0.th23.cu* - numba >=0.57 - numpy >=1.23,<3.0a0 - pylibcugraphops ={{ minor_version }} - tensordict >=0.1.2 - python - - pytorch >=2.0 + - pytorch >=2.3,<2.4.0a0 - cupy >=12.0.0 tests: @@ -41,4 +43,4 @@ about: dev_url: https://github.com/rapidsai/cugraph license: Apache-2.0 license_file: ../../../LICENSE - summary: cuGraph library + summary: cuGraph-dgl library diff --git a/conda/recipes/cugraph-pyg/conda_build_config.yaml b/conda/recipes/cugraph-pyg/conda_build_config.yaml index 2d7aff1..47d98b4 100644 --- a/conda/recipes/cugraph-pyg/conda_build_config.yaml +++ b/conda/recipes/cugraph-pyg/conda_build_config.yaml @@ -10,7 +10,7 @@ cuda_compiler: - nvcc cmake_version: - - ">=3.26.4" + - ">=3.26.4,!=3.30.0" c_stdlib: - sysroot diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml index 5bce780..7d3e503 100644 --- a/conda/recipes/cugraph-pyg/meta.yaml +++ b/conda/recipes/cugraph-pyg/meta.yaml @@ -24,18 +24,19 @@ requirements: host: - cython >=3.0.0 - python - - scikit-build-core >=0.7.0 + - rapids-build-backend>=0.3.1,<0.4.0.dev0 + - setuptools>=61.0.0 run: - rapids-dask-dependency ={{ minor_version }} - numba >=0.57 - numpy >=1.23,<3.0a0 - python - - pytorch >=2.0 + - pytorch >=2.3,<2.4.0a0 - cupy >=12.0.0 - cugraph ={{ version }} - pylibcugraphops ={{ minor_version }} - tensordict >=0.1.2 - - pyg >=2.5,<2.6 + - pytorch_geometric >=2.5,<2.6 tests: imports: diff --git a/datasets/get_test_data.sh b/datasets/get_test_data.sh new file mode 100755 index 0000000..6778166 --- /dev/null +++ b/datasets/get_test_data.sh @@ -0,0 +1,128 @@ +# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -e +set -o pipefail + +# Ensure we're in the cugraph/datasets dir +cd "$( cd "$( dirname "$(realpath -m "${BASH_SOURCE[0]}")" )" && pwd )"; + +# Update this to add/remove/change a dataset, using the following format: +# +# comment about the dataset +# dataset download URL +# destination dir to untar to +# blank line separator +# +# FIXME: some test data needs to be extracted to "benchmarks", which is +# confusing now that there's dedicated datasets for benchmarks. +CPP_CI_DATASET_DATA=" +# ~10s download +https://data.rapids.ai/cugraph/test/cpp_ci_datasets.tgz +test +" + +BASE_DATASET_DATA=" +# ~22s download +https://data.rapids.ai/cugraph/test/datasets.tgz +test + +# ~14s download +https://data.rapids.ai/cugraph/test/ref/pagerank.tgz +test/ref + +# ~1s download +https://data.rapids.ai/cugraph/test/ref/sssp.tgz +test/ref + +# ~15s download +https://data.rapids.ai/cugraph/benchmark/hibench/hibench_1_large.tgz +benchmark + +# ~1s download +https://data.rapids.ai/cugraph/benchmark/hibench/hibench_1_small.tgz +benchmark + +# ~0.6s download +https://data.rapids.ai/cugraph/test/tsplib/datasets.tar.gz +tsplib +" + +EXTENDED_DATASET_DATA=" +# ~42s download - tests using this dataset are currently not run in test.sh with --quick +https://data.rapids.ai/cugraph/benchmark/hibench/hibench_1_huge.tgz +benchmark +" + +BENCHMARK_DATASET_DATA=" +# ~90s download - these are used for benchmarks runs (code in /benchmarks) +https://data.rapids.ai/cugraph/benchmark/benchmark_csv_data.tgz +csv +" + +SELF_LOOPS_DATASET_DATA=" +# ~1s download +https://data.rapids.ai/cugraph/benchmark/benchmark_csv_data_self_loops.tgz +self_loops +" +################################################################################ +# Do not change the script below this line if only adding/updating a dataset + +NUMARGS=$# +ARGS=$* +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +if hasArg -h || hasArg --help; then + echo "$0 [--subset | --benchmark | --self_loops]" + exit 0 +fi + +# Select the datasets to install +if hasArg "--benchmark"; then + DATASET_DATA="${BENCHMARK_DATASET_DATA}" +elif hasArg "--subset"; then + DATASET_DATA="${BASE_DATASET_DATA}" +elif hasArg "--cpp_ci_subset"; then + DATASET_DATA="${CPP_CI_DATASET_DATA}" +elif hasArg "--self_loops"; then + DATASET_DATA="${SELF_LOOPS_DATASET_DATA}" +# Do not include benchmark datasets by default - too big +else + DATASET_DATA="${BASE_DATASET_DATA} ${EXTENDED_DATASET_DATA}" +fi + +URLS=($(echo "$DATASET_DATA"|awk '{if (NR%4 == 3) print $0}')) # extract 3rd fields to a bash array +DESTDIRS=($(echo "$DATASET_DATA"|awk '{if (NR%4 == 0) print $0}')) # extract 4th fields to a bash array + +echo Downloading ... + +# Download all tarfiles to a tmp dir +mkdir -p tmp +cd tmp +for url in ${URLS[*]}; do + time wget -N --progress=dot:giga ${url} +done +cd .. + +# create the destination dirs +mkdir -p "${DESTDIRS[@]}" + +# Iterate over the arrays and untar the nth tarfile to the nth dest directory. +# The tarfile name is derived from the download url. +echo Decompressing ... +for index in ${!DESTDIRS[*]}; do + echo "tmp/$(basename "${URLS[$index]}") -C ${DESTDIRS[$index]}" | tr '\n' '\0' +done | xargs -0 -t -r -n1 -P$(nproc --all) sh -c 'tar -xzvf $0 --overwrite' diff --git a/datasets/karate.csv b/datasets/karate.csv new file mode 100644 index 0000000..4ed9f43 --- /dev/null +++ b/datasets/karate.csv @@ -0,0 +1,156 @@ +1 0 1.0 +2 0 1.0 +3 0 1.0 +4 0 1.0 +5 0 1.0 +6 0 1.0 +7 0 1.0 +8 0 1.0 +10 0 1.0 +11 0 1.0 +12 0 1.0 +13 0 1.0 +17 0 1.0 +19 0 1.0 +21 0 1.0 +31 0 1.0 +2 1 1.0 +3 1 1.0 +7 1 1.0 +13 1 1.0 +17 1 1.0 +19 1 1.0 +21 1 1.0 +30 1 1.0 +3 2 1.0 +7 2 1.0 +8 2 1.0 +9 2 1.0 +13 2 1.0 +27 2 1.0 +28 2 1.0 +32 2 1.0 +7 3 1.0 +12 3 1.0 +13 3 1.0 +6 4 1.0 +10 4 1.0 +6 5 1.0 +10 5 1.0 +16 5 1.0 +16 6 1.0 +30 8 1.0 +32 8 1.0 +33 8 1.0 +33 9 1.0 +33 13 1.0 +32 14 1.0 +33 14 1.0 +32 15 1.0 +33 15 1.0 +32 18 1.0 +33 18 1.0 +33 19 1.0 +32 20 1.0 +33 20 1.0 +32 22 1.0 +33 22 1.0 +25 23 1.0 +27 23 1.0 +29 23 1.0 +32 23 1.0 +33 23 1.0 +25 24 1.0 +27 24 1.0 +31 24 1.0 +31 25 1.0 +29 26 1.0 +33 26 1.0 +33 27 1.0 +31 28 1.0 +33 28 1.0 +32 29 1.0 +33 29 1.0 +32 30 1.0 +33 30 1.0 +32 31 1.0 +33 31 1.0 +33 32 1.0 +0 1 1.0 +0 2 1.0 +0 3 1.0 +0 4 1.0 +0 5 1.0 +0 6 1.0 +0 7 1.0 +0 8 1.0 +0 10 1.0 +0 11 1.0 +0 12 1.0 +0 13 1.0 +0 17 1.0 +0 19 1.0 +0 21 1.0 +0 31 1.0 +1 2 1.0 +1 3 1.0 +1 7 1.0 +1 13 1.0 +1 17 1.0 +1 19 1.0 +1 21 1.0 +1 30 1.0 +2 3 1.0 +2 7 1.0 +2 8 1.0 +2 9 1.0 +2 13 1.0 +2 27 1.0 +2 28 1.0 +2 32 1.0 +3 7 1.0 +3 12 1.0 +3 13 1.0 +4 6 1.0 +4 10 1.0 +5 6 1.0 +5 10 1.0 +5 16 1.0 +6 16 1.0 +8 30 1.0 +8 32 1.0 +8 33 1.0 +9 33 1.0 +13 33 1.0 +14 32 1.0 +14 33 1.0 +15 32 1.0 +15 33 1.0 +18 32 1.0 +18 33 1.0 +19 33 1.0 +20 32 1.0 +20 33 1.0 +22 32 1.0 +22 33 1.0 +23 25 1.0 +23 27 1.0 +23 29 1.0 +23 32 1.0 +23 33 1.0 +24 25 1.0 +24 27 1.0 +24 31 1.0 +25 31 1.0 +26 29 1.0 +26 33 1.0 +27 33 1.0 +28 31 1.0 +28 33 1.0 +29 32 1.0 +29 33 1.0 +30 32 1.0 +30 33 1.0 +31 32 1.0 +31 33 1.0 +32 33 1.0 diff --git a/dependencies.yaml b/dependencies.yaml index e0e74d2..021e749 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,8 +24,6 @@ files: - depends_on_pylibcugraphops - depends_on_cupy - depends_on_pytorch - - depends_on_dgl - - depends_on_pyg - python_run_cugraph_dgl - python_run_cugraph_pyg - test_cpp @@ -56,6 +54,7 @@ files: output: none includes: - cuda_version + - depends_on_pytorch - py_version - test_notebook - test_python_common @@ -107,6 +106,7 @@ files: extras: table: build-system includes: + - python_build_rapids - python_build_wheel py_run_cugraph_dgl: output: pyproject @@ -115,6 +115,7 @@ files: table: project includes: - python_run_cugraph_dgl + - depends_on_cugraph - depends_on_pylibcugraphops py_test_cugraph_dgl: output: pyproject @@ -126,12 +127,14 @@ files: - test_python_common - test_python_cugraph_dgl - depends_on_pylibwholegraph + - depends_on_pytorch py_build_cugraph_pyg: output: pyproject pyproject_dir: python/cugraph-pyg extras: table: build-system includes: + - python_build_rapids - python_build_wheel py_run_cugraph_pyg: output: pyproject @@ -140,6 +143,7 @@ files: table: project includes: - python_run_cugraph_pyg + - depends_on_cugraph - depends_on_pylibcugraphops py_test_cugraph_pyg: output: pyproject @@ -151,6 +155,7 @@ files: - test_python_common - test_python_cugraph_pyg - depends_on_pylibwholegraph + - depends_on_pytorch cugraph_dgl_dev: @@ -254,10 +259,20 @@ dependencies: - cuda-nvtx common_build: common: - - output_types: [conda, pyproject] + - output_types: [requirements, pyproject] packages: - - &cmake_ver cmake>=3.26.4,!=3.30.0 - ninja + - output_types: [conda] + packages: + - &cmake_ver cmake>=3.26.4,!=3.30.0 + - cudnn=8.8 + - c-compiler + - cxx-compiler + - libraft-headers==24.12.*,>=0.0.0a0 + - librmm==24.12.*,>=0.0.0a0 + - nanobind>=0.2.0 + - &cython cython>=3.0.0 + - &nccl nccl>=2.19 docs: common: @@ -294,6 +309,11 @@ dependencies: - matrix: packages: - python>=3.10,<3.13 + python_build_rapids: + common: + - output_types: [conda, pyproject, requirements] + packages: + - rapids-build-backend>=0.3.1,<0.4.0.dev0 python_build_wheel: common: - output_types: [conda, pyproject, requirements] @@ -301,11 +321,12 @@ dependencies: - cmake>=3.26.4,!=3.30.0 - cython>=3.0.0 - ninja + - setuptools python_build_cythonize: common: - output_types: [conda, pyproject, requirements] packages: - - cython>=3.0.0 + - *cython - output_types: conda packages: - scikit-build-core>=0.10.0 @@ -323,18 +344,12 @@ dependencies: packages: - &numba numba>=0.57 - *numpy - - output_types: [pyproject] - packages: - - &cugraph cugraph==24.12.*,>=0.0.0a0 python_run_cugraph_pyg: common: - output_types: [conda, pyproject] packages: - *numba - *numpy - - output_types: [pyproject] - packages: - - *cugraph rapids_build_skbuild: common: - output_types: [conda, requirements, pyproject] @@ -357,10 +372,17 @@ dependencies: - output_types: [conda, requirements] packages: - ipython + - nbconvert>=7.16 - notebook>=0.5.0 - output_types: [conda] packages: - wget + test_cpp: + common: + - output_types: conda + packages: + - *cmake_ver + - *nccl test_python_common: common: - output_types: [conda, pyproject] @@ -368,6 +390,7 @@ dependencies: - pytest - pytest-benchmark - pytest-cov + - pytest-forked - pytest-xdist test_python_cugraph_dgl: common: @@ -389,24 +412,22 @@ dependencies: common: - output_types: [conda] packages: - - cugraph==24.12.*,>=0.0.0a0 - - pytorch::pytorch>=2.0 - - dgl>=1.1.0.cu* + - &pytorch_conda pytorch::pytorch>=2.3,<2.4.0a0 + - dgl>=2.4.0.cu* - &tensordict tensordict>=0.1.2 cugraph_pyg_dev: common: - output_types: [conda] packages: - - cugraph==24.12.*,>=0.0.0a0 - - pytorch::pytorch>=2.0 + - *pytorch_conda - *tensordict - - pyg>=2.5,<2.6 + - pytorch_geometric>=2.5,<2.6 depends_on_pytorch: common: - output_types: [conda] packages: - - pytorch::pytorch>=2.0,<2.4.0a0 + - *pytorch_conda - torchdata - pydantic specific: @@ -423,70 +444,13 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - &pytorch_pip torch>=2.0,<2.4.0a0 + - &pytorch_pip torch>=2.3,<2.4.0a0 - *tensordict - matrix: {cuda: "11.*"} packages: - *pytorch_pip - *tensordict - {matrix: null, packages: [*pytorch_pip, *tensordict]} - - output_types: [conda] - matrices: - - matrix: {cuda: "12.*"} - packages: - - pytorch-cuda=12.1 - - matrix: {cuda: "11.*"} - packages: - - pytorch-cuda=11.8 - - depends_on_dgl: - specific: - - output_types: [conda] - matrices: - - matrix: {cuda: "12.*"} - packages: - - "dglteam/label/th21_cu121::dgl" - - matrix: {cuda: "11.*"} - packages: - - "dglteam/label/th21_cu118::dgl" - - {matrix: null, packages: ["dglteam/label/th23_cu121::dgl"]} - - depends_on_pyg: - specific: - - output_types: [conda] - matrices: - - matrix: {cuda: "12.*"} - packages: - - "pyg::pyg" - - matrix: {cuda: "11.*"} - packages: - - "pyg::pyg" - - {matrix: null, packages: ["pyg::pyg"]} - - depends_on_pylibwholegraph: - common: - - output_types: conda - packages: - - &pylibwholegraph_unsuffixed pylibwholegraph==24.12.*,>=0.0.0a0 - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - pylibwholegraph-cu12==24.12.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - pylibwholegraph-cu11==24.12.*,>=0.0.0a0 - - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} depends_on_rmm: common: @@ -706,3 +670,55 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + + depends_on_dgl: + specific: + - output_types: [conda] + matrices: + - matrix: {cuda: "12.1"} + packages: + - "dglteam/label/th23_cu121::dgl" + - matrix: {cuda: "12.4"} + packages: + - "dglteam/label/th23_cu124::dgl" + - matrix: {cuda: "11.*"} + packages: + - "dglteam/label/th23_cu118::dgl" + - {matrix: null, packages: ["dglteam/label/th23_cu121::dgl"]} + + depends_on_pyg: + specific: + - output_types: [conda] + matrices: + - matrix: {cuda: "12.*"} + packages: + - "pyg::pyg" + - matrix: {cuda: "11.*"} + packages: + - "pyg::pyg" + - {matrix: null, packages: ["pyg::pyg"]} + + depends_on_pylibwholegraph: + common: + - output_types: conda + packages: + - &pylibwholegraph_unsuffixed pylibwholegraph==24.12.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - pylibwholegraph-cu12==24.12.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - pylibwholegraph-cu11==24.12.*,>=0.0.0a0 + - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} diff --git a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml index 9f0070f..632157f 100644 --- a/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml +++ b/python/cugraph-dgl/conda/cugraph_dgl_dev_cuda-118.yaml @@ -9,8 +9,8 @@ channels: - nvidia dependencies: - cugraph==24.12.*,>=0.0.0a0 -- dgl>=1.1.0.cu* -- dglteam/label/th21_cu118::dgl +- dgl>=2.4.0.cu* +- dglteam/label/th23_cu118::dgl - pandas - pre-commit - pydantic @@ -18,10 +18,9 @@ dependencies: - pytest - pytest-benchmark - pytest-cov +- pytest-forked - pytest-xdist -- pytorch-cuda=11.8 -- pytorch::pytorch>=2.0 -- pytorch::pytorch>=2.0,<2.4.0a0 +- pytorch::pytorch>=2.3,<2.4.0a0 - tensordict>=0.1.2 - torchdata name: cugraph_dgl_dev_cuda-118 diff --git a/python/cugraph-dgl/cugraph_dgl/_version.py b/python/cugraph-dgl/cugraph_dgl/_version.py index f2210fd..e8adcc3 100644 --- a/python/cugraph-dgl/cugraph_dgl/_version.py +++ b/python/cugraph-dgl/cugraph_dgl/_version.py @@ -15,12 +15,17 @@ import importlib.resources -# Read VERSION file from the module that is symlinked to VERSION file -# in the root of the repo at build time or copied to the moudle at -# installation. VERSION is a separate file that allows CI build-time scripts -# to update version info (including commit hashes) without modifying -# source files. __version__ = ( - importlib.resources.files("cugraph_dgl").joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py b/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py index 4ec513c..ecc5100 100644 --- a/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py +++ b/python/cugraph-dgl/cugraph_dgl/dataloading/neighbor_sampler.py @@ -197,10 +197,8 @@ def sample( if g.is_homogeneous: indices = torch.concat(list(indices)) - ds.sample_from_nodes(indices.long(), batch_size=batch_size) - return HomogeneousSampleReader( - ds.get_reader(), self.output_format, self.edge_dir - ) + reader = ds.sample_from_nodes(indices.long(), batch_size=batch_size) + return HomogeneousSampleReader(reader, self.output_format, self.edge_dir) raise ValueError( "Sampling heterogeneous graphs is currently" diff --git a/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py b/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py index a60db97..2b9f46d 100644 --- a/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py +++ b/python/cugraph-dgl/cugraph_dgl/tests/test_graph.py @@ -53,10 +53,11 @@ def test_graph_make_homogeneous_graph(direction): graph.nodes() == torch.arange(num_nodes, dtype=torch.int64, device="cuda") ).all() - assert graph.nodes[None]["x"] is not None - assert (graph.nodes[None]["x"] == torch.as_tensor(node_x, device="cuda")).all() + emb = graph.nodes[None]["x"] + assert emb is not None + assert (emb() == torch.as_tensor(node_x, device="cuda")).all() assert ( - graph.nodes[None]["num"] + graph.nodes[None]["num"]() == torch.arange(num_nodes, dtype=torch.int64, device="cuda") ).all() @@ -64,7 +65,7 @@ def test_graph_make_homogeneous_graph(direction): graph.edges("eid", device="cuda") == torch.arange(len(df), dtype=torch.int64, device="cuda") ).all() - assert (graph.edges[None]["weight"] == torch.as_tensor(wgt, device="cuda")).all() + assert (graph.edges[None]["weight"]() == torch.as_tensor(wgt, device="cuda")).all() plc_expected_graph = pylibcugraph.SGGraph( pylibcugraph.ResourceHandle(), diff --git a/python/cugraph-dgl/cugraph_dgl/tests/test_version.py b/python/cugraph-dgl/cugraph_dgl/tests/test_version.py new file mode 100644 index 0000000..343e4fb --- /dev/null +++ b/python/cugraph-dgl/cugraph_dgl/tests/test_version.py @@ -0,0 +1,12 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import cugraph_dgl + + +def test_version_constants_are_populated(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(cugraph_dgl.__git_commit__, str) + + # __version__ should always be non-empty + assert isinstance(cugraph_dgl.__version__, str) + assert len(cugraph_dgl.__version__) > 0 diff --git a/python/cugraph-dgl/cugraph_dgl/view.py b/python/cugraph-dgl/cugraph_dgl/view.py index dbc53e7..7c4d95f 100644 --- a/python/cugraph-dgl/cugraph_dgl/view.py +++ b/python/cugraph-dgl/cugraph_dgl/view.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings from collections import defaultdict from collections.abc import MutableMapping @@ -20,11 +21,53 @@ import cugraph_dgl from cugraph_dgl.typing import TensorType +from cugraph_dgl.utils.cugraph_conversion_utils import _cast_to_torch_tensor torch = import_optional("torch") dgl = import_optional("dgl") +class EmbeddingView: + def __init__(self, storage: "dgl.storages.base.FeatureStorage", ld: int): + self.__ld = ld + self.__storage = storage + + def __getitem__(self, u: TensorType) -> "torch.Tensor": + u = _cast_to_torch_tensor(u) + try: + return self.__storage.fetch( + u, + "cuda", + ) + except RuntimeError as ex: + warnings.warn( + "Got error accessing data, trying again with index on device: " + + str(ex) + ) + return self.__storage.fetch( + u.cuda(), + "cuda", + ) + + def __call__(self): + warnings.warn( + "Getting an entire embedding tensor is not recommended " + " as it wastes memory. Consider indexing to get only the " + "required elements of the embedding tensor." + ) + return self[torch.arange(self.__ld, dtype=torch.int64)] + + @property + def shape(self) -> "torch.Size": + try: + f = self.__storage.fetch(torch.tensor([0]), "cpu") + except RuntimeError: + f = self.__storage.fetch(torch.tensor([0], device="cuda"), "cuda") + sz = [s for s in f.shape] + sz[0] = self.__ld + return torch.Size(tuple(sz)) + + class HeteroEdgeDataView(MutableMapping): """ Duck-typed version of DGL's HeteroEdgeDataView. diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 0ca636d..8af2524 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -6,8 +6,10 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0", "ninja", + "rapids-build-backend>=0.3.1,<0.4.0.dev0", + "setuptools", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -build-backend = "setuptools.build_meta" +build-backend = "rapids_build_backend.build" [project] name = "cugraph-dgl" @@ -37,7 +39,10 @@ test = [ "pytest", "pytest-benchmark", "pytest-cov", + "pytest-forked", "pytest-xdist", + "tensordict>=0.1.2", + "torch>=2.3,<2.4.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] @@ -54,3 +59,8 @@ version = {file = "cugraph_dgl/VERSION"} include = [ "cugraph_dgl*", ] + +[tool.rapids-build-backend] +build-backend = "setuptools.build_meta" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" diff --git a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml index a9f520c..5609cb5 100644 --- a/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml +++ b/python/cugraph-pyg/conda/cugraph_pyg_dev_cuda-118.yaml @@ -12,15 +12,14 @@ dependencies: - pandas - pre-commit - pydantic -- pyg>=2.5,<2.6 - pylibcugraphops==24.12.*,>=0.0.0a0 - pytest - pytest-benchmark - pytest-cov +- pytest-forked - pytest-xdist -- pytorch-cuda=11.8 -- pytorch::pytorch>=2.0 -- pytorch::pytorch>=2.0,<2.4.0a0 +- pytorch::pytorch>=2.3,<2.4.0a0 +- pytorch_geometric>=2.5,<2.6 - tensordict>=0.1.2 - torchdata name: cugraph_pyg_dev_cuda-118 diff --git a/python/cugraph-pyg/cugraph_pyg/tests/test_version.py b/python/cugraph-pyg/cugraph_pyg/tests/test_version.py new file mode 100644 index 0000000..4ea0f98 --- /dev/null +++ b/python/cugraph-pyg/cugraph_pyg/tests/test_version.py @@ -0,0 +1,12 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import cugraph_pyg + + +def test_version_constants_are_populated(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(cugraph_pyg.__git_commit__, str) + + # __version__ should always be non-empty + assert isinstance(cugraph_pyg.__version__, str) + assert len(cugraph_pyg.__version__) > 0 diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index d0bfe58..f096b58 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -6,7 +6,10 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0", "ninja", + "rapids-build-backend>=0.3.1,<0.4.0.dev0", + "setuptools", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "rapids_build_backend.build" [tool.pytest.ini_options] testpaths = ["cugraph_pyg/tests"] @@ -45,7 +48,10 @@ test = [ "pytest", "pytest-benchmark", "pytest-cov", + "pytest-forked", "pytest-xdist", + "tensordict>=0.1.2", + "torch>=2.3,<2.4.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.setuptools] @@ -59,3 +65,8 @@ include = [ "cugraph_pyg*", "cugraph_pyg.*", ] + +[tool.rapids-build-backend] +build-backend = "setuptools.build_meta" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" diff --git a/python/pylibwholegraph/pyproject.toml b/python/pylibwholegraph/pyproject.toml index 3eed9dc..f1bcfb3 100644 --- a/python/pylibwholegraph/pyproject.toml +++ b/python/pylibwholegraph/pyproject.toml @@ -58,6 +58,7 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0", "ninja", + "setuptools", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.scikit-build]