From f364fdcd44540b6d5403f1d08acbebfff4e78bd4 Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Fri, 19 Jul 2024 14:56:13 -0400 Subject: [PATCH 001/270] DOC v24.10 Updates [skip ci] --- .../cuda11.8-conda/devcontainer.json | 6 +-- .devcontainer/cuda11.8-pip/devcontainer.json | 6 +-- .../cuda12.5-conda/devcontainer.json | 6 +-- .devcontainer/cuda12.5-pip/devcontainer.json | 6 +-- .github/workflows/build.yaml | 20 ++++----- .github/workflows/pandas-tests.yaml | 2 +- .github/workflows/pr.yaml | 44 +++++++++---------- .github/workflows/test.yaml | 22 +++++----- README.md | 2 +- VERSION | 2 +- ci/test_wheel_cudf_polars.sh | 2 +- .../all_cuda-118_arch-x86_64.yaml | 10 ++--- .../all_cuda-125_arch-x86_64.yaml | 10 ++--- cpp/examples/versions.cmake | 2 +- dependencies.yaml | 32 +++++++------- java/ci/README.md | 4 +- java/pom.xml | 2 +- python/cudf/pyproject.toml | 4 +- python/cudf_kafka/pyproject.toml | 2 +- python/cudf_polars/docs/overview.md | 2 +- python/cudf_polars/pyproject.toml | 2 +- python/custreamz/pyproject.toml | 4 +- python/dask_cudf/pyproject.toml | 6 +-- 23 files changed, 99 insertions(+), 99 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 8423fe21c29..7a1361e52c5 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 4945d6cf753..64d7cd54130 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index fadce01d060..c1924243506 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.08-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 026eb540952..beab2940176 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.5-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.8": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.10": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2e5959338b0..2fc39c06fad 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -101,7 +101,7 @@ jobs: wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -111,7 +111,7 @@ jobs: wheel-build-cudf-polars: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -123,7 +123,7 @@ jobs: wheel-publish-cudf-polars: needs: wheel-build-cudf-polars secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pandas-tests.yaml b/.github/workflows/pandas-tests.yaml index 5a937b2f362..cf0c2b377dd 100644 --- a/.github/workflows/pandas-tests.yaml +++ b/.github/workflows/pandas-tests.yaml @@ -17,7 +17,7 @@ jobs: pandas-tests: # run the Pandas unit tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) build_type: nightly diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d5dfc9e1ff5..c2e7f64f952 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,41 +34,41 @@ jobs: - pandas-tests - pandas-tests-diff secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: pull-request enable_check_symbols: true conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request conda-python-cudf-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -76,14 +76,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -93,7 +93,7 @@ jobs: static-configure: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request # Use the wheel container so we can skip conda solves and since our @@ -103,7 +103,7 @@ jobs: conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -113,7 +113,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -123,21 +123,21 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -146,7 +146,7 @@ jobs: wheel-tests-cudf-polars: needs: wheel-build-cudf-polars secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -157,7 +157,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -166,7 +166,7 @@ jobs: wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -174,7 +174,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 with: arch: '["amd64"]' cuda: '["12.5"]' @@ -185,7 +185,7 @@ jobs: unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request @@ -194,7 +194,7 @@ jobs: # run the Pandas unit tests using PR branch needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) build_type: pull-request @@ -204,7 +204,7 @@ jobs: pandas-tests-diff: # diff the results of running the Pandas unit tests and publish a job summary needs: pandas-tests - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: node_type: cpu4 build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 36c9088d93c..9feea050b19 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -25,7 +25,7 @@ jobs: enable_check_symbols: true conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -33,7 +33,7 @@ jobs: sha: ${{ inputs.sha }} conda-cpp-memcheck-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -45,7 +45,7 @@ jobs: run_script: "ci/test_cpp_memcheck.sh" static-configure: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request # Use the wheel container so we can skip conda solves and since our @@ -54,7 +54,7 @@ jobs: run_script: "ci/configure_cpp_static.sh" conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -64,7 +64,7 @@ jobs: conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -73,7 +73,7 @@ jobs: script: "ci/test_python_other.sh" conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -85,7 +85,7 @@ jobs: run_script: "ci/test_java.sh" conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -97,7 +97,7 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -106,7 +106,7 @@ jobs: script: ci/test_wheel_cudf.sh wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -117,7 +117,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh unit-tests-cudf-pandas: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/README.md b/README.md index 1ab6a2d7457..fd8b0365807 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ cuDF can be installed with conda (via [miniconda](https://docs.conda.io/projects ```bash conda install -c rapidsai -c conda-forge -c nvidia \ - cudf=24.08 python=3.11 cuda-version=12.5 + cudf=24.10 python=3.11 cuda-version=12.5 ``` We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD diff --git a/VERSION b/VERSION index ec8489fda92..7c7ba04436f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.08.00 +24.10.00 diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 900acd5d473..cc9f5788685 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -10,7 +10,7 @@ set -eou pipefail # files in cudf_polars/pylibcudf", rather than "are there changes # between upstream and this branch which touch cudf_polars/pylibcudf" # TODO: is the target branch exposed anywhere in an environment variable? -if [ -n "$(git diff --name-only origin/branch-24.08...HEAD -- python/cudf_polars/ python/cudf/cudf/_lib/pylibcudf/)" ]; +if [ -n "$(git diff --name-only origin/branch-24.10...HEAD -- python/cudf_polars/ python/cudf/cudf/_lib/pylibcudf/)" ]; then HAS_CHANGES=1 else diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index b8d73a01f96..b1a1cc3c68e 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -26,7 +26,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.3 -- dask-cuda==24.8.*,>=0.0.0a0 +- dask-cuda==24.10.*,>=0.0.0a0 - dlpack>=0.8,<1.0 - doxygen=1.9.1 - fastavro>=0.22.9 @@ -43,10 +43,10 @@ dependencies: - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 - libcurand=10.3.0.86 -- libkvikio==24.8.*,>=0.0.0a0 +- libkvikio==24.10.*,>=0.0.0a0 - libparquet==16.1.0.* - librdkafka>=1.9.0,<1.10.0a0 -- librmm==24.8.*,>=0.0.0a0 +- librmm==24.10.*,>=0.0.0a0 - make - moto>=4.0.8 - msgpack-python @@ -77,9 +77,9 @@ dependencies: - python>=3.9,<3.12 - pytorch>=2.1.0 - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.8.*,>=0.0.0a0 +- rapids-dask-dependency==24.10.*,>=0.0.0a0 - rich -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - s3fs>=2022.3.0 - scikit-build-core>=0.7.0 - scipy diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 3f5fae49cbb..1017b11779c 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -27,7 +27,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.3 -- dask-cuda==24.8.*,>=0.0.0a0 +- dask-cuda==24.10.*,>=0.0.0a0 - dlpack>=0.8,<1.0 - doxygen=1.9.1 - fastavro>=0.22.9 @@ -42,10 +42,10 @@ dependencies: - libarrow==16.1.0.* - libcufile-dev - libcurand-dev -- libkvikio==24.8.*,>=0.0.0a0 +- libkvikio==24.10.*,>=0.0.0a0 - libparquet==16.1.0.* - librdkafka>=1.9.0,<1.10.0a0 -- librmm==24.8.*,>=0.0.0a0 +- librmm==24.10.*,>=0.0.0a0 - make - moto>=4.0.8 - msgpack-python @@ -75,9 +75,9 @@ dependencies: - python>=3.9,<3.12 - pytorch>=2.1.0 - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==24.8.*,>=0.0.0a0 +- rapids-dask-dependency==24.10.*,>=0.0.0a0 - rich -- rmm==24.8.*,>=0.0.0a0 +- rmm==24.10.*,>=0.0.0a0 - s3fs>=2022.3.0 - scikit-build-core>=0.7.0 - scipy diff --git a/cpp/examples/versions.cmake b/cpp/examples/versions.cmake index 144b3d3721b..44493011673 100644 --- a/cpp/examples/versions.cmake +++ b/cpp/examples/versions.cmake @@ -12,4 +12,4 @@ # the License. # ============================================================================= -set(CUDF_TAG branch-24.08) +set(CUDF_TAG branch-24.10) diff --git a/dependencies.yaml b/dependencies.yaml index a19574b7658..a90ac64387b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -287,8 +287,8 @@ dependencies: - output_types: conda packages: - fmt>=10.1.1,<11 - - librmm==24.8.*,>=0.0.0a0 - - libkvikio==24.8.*,>=0.0.0a0 + - librmm==24.10.*,>=0.0.0a0 + - libkvikio==24.10.*,>=0.0.0a0 - librdkafka>=1.9.0,<1.10.0a0 # Align nvcomp version with rapids-cmake - nvcomp==3.0.6 @@ -329,7 +329,7 @@ dependencies: common: - output_types: conda packages: - - &rmm_conda rmm==24.8.*,>=0.0.0a0 + - &rmm_conda rmm==24.10.*,>=0.0.0a0 - pip - pip: - git+https://github.com/python-streamz/streamz.git@master @@ -345,10 +345,10 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: &build_python_packages_cu12 - - rmm-cu12==24.8.*,>=0.0.0a0 + - rmm-cu12==24.10.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: &build_python_packages_cu11 - - rmm-cu11==24.8.*,>=0.0.0a0 + - rmm-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_conda] } libarrow_build: common: @@ -505,7 +505,7 @@ dependencies: - output_types: [conda] packages: - breathe>=4.35.0 - - dask-cuda==24.8.*,>=0.0.0a0 + - dask-cuda==24.10.*,>=0.0.0a0 - *doxygen - make - myst-nb @@ -597,11 +597,11 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - rmm-cu12==24.8.*,>=0.0.0a0 + - rmm-cu12==24.10.*,>=0.0.0a0 - pynvjitlink-cu12>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - rmm-cu11==24.8.*,>=0.0.0a0 + - rmm-cu11==24.10.*,>=0.0.0a0 - cubinlinker-cu11 - ptxcompiler-cu11 - {matrix: null, packages: [cubinlinker, ptxcompiler, *rmm_conda]} @@ -614,7 +614,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - rapids-dask-dependency==24.8.*,>=0.0.0a0 + - rapids-dask-dependency==24.10.*,>=0.0.0a0 run_custreamz: common: - output_types: conda @@ -700,13 +700,13 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - dask-cuda==24.8.*,>=0.0.0a0 + - dask-cuda==24.10.*,>=0.0.0a0 - *numba depends_on_cudf: common: - output_types: conda packages: - - &cudf_conda cudf==24.8.*,>=0.0.0a0 + - &cudf_conda cudf==24.10.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -718,16 +718,16 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - cudf-cu12==24.8.*,>=0.0.0a0 + - cudf-cu12==24.10.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - cudf-cu11==24.8.*,>=0.0.0a0 + - cudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_conda]} depends_on_cudf_kafka: common: - output_types: conda packages: - - &cudf_kafka_conda cudf_kafka==24.8.*,>=0.0.0a0 + - &cudf_kafka_conda cudf_kafka==24.10.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -739,10 +739,10 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - cudf_kafka-cu12==24.8.*,>=0.0.0a0 + - cudf_kafka-cu12==24.10.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - cudf_kafka-cu11==24.8.*,>=0.0.0a0 + - cudf_kafka-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_kafka_conda]} depends_on_cupy: common: diff --git a/java/ci/README.md b/java/ci/README.md index 49481efab6b..ccb9efb50b6 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -34,7 +34,7 @@ nvidia-docker run -it cudf-build:11.8.0-devel-rocky8 bash You can download the cuDF repo in the docker container or you can mount it into the container. Here I choose to download again in the container. ```bash -git clone --recursive https://github.com/rapidsai/cudf.git -b branch-24.08 +git clone --recursive https://github.com/rapidsai/cudf.git -b branch-24.10 ``` ### Build cuDF jar with devtoolset @@ -47,4 +47,4 @@ scl enable gcc-toolset-11 "java/ci/build-in-docker.sh" ### The output -You can find the cuDF jar in java/target/ like cudf-24.08.0-SNAPSHOT-cuda11.jar. +You can find the cuDF jar in java/target/ like cudf-24.10.0-SNAPSHOT-cuda11.jar. diff --git a/java/pom.xml b/java/pom.xml index 70230e6bc71..9694e741f16 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -21,7 +21,7 @@ ai.rapids cudf - 24.08.0-SNAPSHOT + 24.10.0-SNAPSHOT cudfjni diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index dcb33b1fc1a..da57622dec7 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "ptxcompiler", "pyarrow>=16.1.0,<16.2.0a0", "rich", - "rmm==24.8.*,>=0.0.0a0", + "rmm==24.10.*,>=0.0.0a0", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -126,7 +126,7 @@ requires = [ "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", - "rmm==24.8.*,>=0.0.0a0", + "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.scikit-build] diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index badfdf06d15..bff1a9b8493 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -18,7 +18,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cudf==24.8.*,>=0.0.0a0", + "cudf==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.optional-dependencies] diff --git a/python/cudf_polars/docs/overview.md b/python/cudf_polars/docs/overview.md index 874bb849747..6cd36136bf8 100644 --- a/python/cudf_polars/docs/overview.md +++ b/python/cudf_polars/docs/overview.md @@ -8,7 +8,7 @@ You will need: preferred configuration. Or else, use [rustup](https://www.rust-lang.org/tools/install) 2. A [cudf development - environment](https://github.com/rapidsai/cudf/blob/branch-24.08/CONTRIBUTING.md#setting-up-your-build-environment). + environment](https://github.com/rapidsai/cudf/blob/branch-24.10/CONTRIBUTING.md#setting-up-your-build-environment). The combined devcontainer works, or whatever your favourite approach is. > ![NOTE] These instructions will get simpler as we merge code in. diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 0b559f7a8e9..393a7510c89 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -19,7 +19,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cudf==24.8.*,>=0.0.0a0", + "cudf==24.10.*,>=0.0.0a0", "polars>=1.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 7b99e041b54..59ce15ac4ef 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -20,8 +20,8 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "confluent-kafka>=1.9.0,<1.10.0a0", - "cudf==24.8.*,>=0.0.0a0", - "cudf_kafka==24.8.*,>=0.0.0a0", + "cudf==24.10.*,>=0.0.0a0", + "cudf_kafka==24.10.*,>=0.0.0a0", "streamz", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 9b2e3a5a7b1..4968ff0b076 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -19,12 +19,12 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cudf==24.8.*,>=0.0.0a0", + "cudf==24.10.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numpy>=1.23,<2.0a0", "pandas>=2.0,<2.2.3dev0", - "rapids-dask-dependency==24.8.*,>=0.0.0a0", + "rapids-dask-dependency==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -45,7 +45,7 @@ cudf = "dask_cudf.backends:CudfDXBackendEntrypoint" [project.optional-dependencies] test = [ - "dask-cuda==24.8.*,>=0.0.0a0", + "dask-cuda==24.10.*,>=0.0.0a0", "numba>=0.57", "pytest-cov", "pytest-xdist", From 29ce5c529ea9ea18edc32ab905f1ef076f266008 Mon Sep 17 00:00:00 2001 From: Michael Schellenberger Costa Date: Thu, 25 Jul 2024 01:29:41 +0200 Subject: [PATCH 002/270] Fix some issues with deprecated / removed cccl facilities (#16377) `cub::If` has been deprecated and should not be used. There is a better alternative in `cuda::std::conditional_t` `thrust::{binary, unary}_function` has been deprecated and does not serve a purpose similar to the removed `std::{binary, unary}_function` Rather than relying on the type aliases one should use the `std::invoke` machinery Authors: - Michael Schellenberger Costa (https://github.com/miscco) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) - Bernhard Manfred Gruber (https://github.com/bernhardmgruber) URL: https://github.com/rapidsai/cudf/pull/16377 --- cpp/benchmarks/common/generate_input.cu | 2 +- cpp/include/cudf/detail/gather.cuh | 2 +- cpp/src/io/fst/agent_dfa.cuh | 2 +- cpp/src/reductions/minmax.cu | 3 +-- java/src/main/native/src/aggregation128_utils.cu | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/benchmarks/common/generate_input.cu b/cpp/benchmarks/common/generate_input.cu index 6df2cb44adc..0970003deb2 100644 --- a/cpp/benchmarks/common/generate_input.cu +++ b/cpp/benchmarks/common/generate_input.cu @@ -718,7 +718,7 @@ std::unique_ptr create_random_column(data_profi } template -struct clamp_down : public thrust::unary_function { +struct clamp_down { T max; clamp_down(T max) : max(max) {} __host__ __device__ T operator()(T x) const { return min(x, max); } diff --git a/cpp/include/cudf/detail/gather.cuh b/cpp/include/cudf/detail/gather.cuh index d3e9fc4974d..e8e95380815 100644 --- a/cpp/include/cudf/detail/gather.cuh +++ b/cpp/include/cudf/detail/gather.cuh @@ -518,7 +518,7 @@ struct column_gatherer_impl { * Positive indices are unchanged by this transformation. */ template -struct index_converter : public thrust::unary_function { +struct index_converter { index_converter(size_type n_rows) : n_rows(n_rows) {} __device__ map_type operator()(map_type in) const { return ((in % n_rows) + n_rows) % n_rows; } diff --git a/cpp/src/io/fst/agent_dfa.cuh b/cpp/src/io/fst/agent_dfa.cuh index bc5b94e2718..0e70984b39c 100644 --- a/cpp/src/io/fst/agent_dfa.cuh +++ b/cpp/src/io/fst/agent_dfa.cuh @@ -791,7 +791,7 @@ __launch_bounds__(int32_t(AgentDFAPolicy::BLOCK_THREADS)) CUDF_KERNEL can_use_smem_cache; using DFASimulationCallbackWrapperT = - typename cub::If::Type; + cuda::std::conditional_t; // Stage 1: Compute the state-transition vector if (IS_TRANS_VECTOR_PASS || IS_SINGLE_PASS) { diff --git a/cpp/src/reductions/minmax.cu b/cpp/src/reductions/minmax.cu index 2c1181972c5..6cb58786971 100644 --- a/cpp/src/reductions/minmax.cu +++ b/cpp/src/reductions/minmax.cu @@ -107,8 +107,7 @@ rmm::device_scalar reduce_device(InputIterator d_in, * respectively of the minimums and maximums of the input pairs. */ template -struct minmax_binary_op - : public thrust::binary_function, minmax_pair, minmax_pair> { +struct minmax_binary_op { __device__ minmax_pair operator()(minmax_pair const& lhs, minmax_pair const& rhs) const { return minmax_pair{thrust::min(lhs.min_val, rhs.min_val), diff --git a/java/src/main/native/src/aggregation128_utils.cu b/java/src/main/native/src/aggregation128_utils.cu index a32e7d27085..631df58b017 100644 --- a/java/src/main/native/src/aggregation128_utils.cu +++ b/java/src/main/native/src/aggregation128_utils.cu @@ -34,7 +34,7 @@ namespace { // Functor to reassemble a 128-bit value from four 64-bit chunks with overflow detection. -class chunk_assembler : public thrust::unary_function { +class chunk_assembler { public: chunk_assembler(bool* overflows, uint64_t const* chunks0, From 5a3399bec868f44d13c003f172c665919096d8e8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 19:26:12 -0500 Subject: [PATCH 003/270] fix [tool.setuptools] reference in custreamz config (#16365) Noticed this warning in logs from #16183 > _/python3.10/site-packages/setuptools/config/pyprojecttoml.py:70: _ToolsTypoInMetadata: Ignoring [tools.setuptools] in pyproject.toml, did you mean [tool.setuptools]?_ This fixes that. ## Notes for Reviewers Intentionally targeting this at 24.10. This misconfiguration has been in `custreamz` since the 23.04 release ([git blame link](https://github.com/rapidsai/cudf/blame/e6d412cba7c23df7ee500c28257ed9281cea49b9/python/custreamz/pyproject.toml#L60)). I think the only effect might be that some test files are included in wheels when we don't want to. I don't think the fix for it needs to be rushed into 24.08. I searched across RAPIDS in case this was copied from somewhere else... don't see any other instances of this typo that need to be fixed. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16365 --- python/custreamz/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 59ce15ac4ef..4be94aa3368 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -57,7 +57,7 @@ zip-safe = false [tool.setuptools.dynamic] version = {file = "custreamz/VERSION"} -[tools.setuptools.packages.find] +[tool.setuptools.packages.find] include = [ "custreamz", "custreamz.*", From 473dec55abd1a3d9d540c541443f831d18ebb532 Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Fri, 26 Jul 2024 14:45:12 -0700 Subject: [PATCH 004/270] Add query 10 to the TPC-H suite (#16392) Adds Q10 to the TPC-H benchmark suite Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/16392 --- cpp/examples/tpch/CMakeLists.txt | 4 + cpp/examples/tpch/q1.cpp | 2 +- cpp/examples/tpch/q10.cpp | 166 +++++++++++++++++++++++++++++++ cpp/examples/tpch/q5.cpp | 20 ++-- cpp/examples/tpch/q6.cpp | 2 +- 5 files changed, 182 insertions(+), 12 deletions(-) create mode 100644 cpp/examples/tpch/q10.cpp diff --git a/cpp/examples/tpch/CMakeLists.txt b/cpp/examples/tpch/CMakeLists.txt index 1b91d07e148..373a6d72d56 100644 --- a/cpp/examples/tpch/CMakeLists.txt +++ b/cpp/examples/tpch/CMakeLists.txt @@ -30,3 +30,7 @@ target_compile_features(tpch_q6 PRIVATE cxx_std_17) add_executable(tpch_q9 q9.cpp) target_link_libraries(tpch_q9 PRIVATE cudf::cudf) target_compile_features(tpch_q9 PRIVATE cxx_std_17) + +add_executable(tpch_q10 q10.cpp) +target_link_libraries(tpch_q10 PRIVATE cudf::cudf) +target_compile_features(tpch_q10 PRIVATE cxx_std_17) diff --git a/cpp/examples/tpch/q1.cpp b/cpp/examples/tpch/q1.cpp index 1bdf039da4a..fe03320b888 100644 --- a/cpp/examples/tpch/q1.cpp +++ b/cpp/examples/tpch/q1.cpp @@ -124,7 +124,7 @@ int main(int argc, char const** argv) auto shipdate_upper = cudf::timestamp_scalar(days_since_epoch(1998, 9, 2), true); auto const shipdate_upper_literal = cudf::ast::literal(shipdate_upper); - auto lineitem_pred = std::make_unique( + auto const lineitem_pred = std::make_unique( cudf::ast::ast_operator::LESS_EQUAL, shipdate_ref, shipdate_upper_literal); // Read out the `lineitem` table from parquet file diff --git a/cpp/examples/tpch/q10.cpp b/cpp/examples/tpch/q10.cpp new file mode 100644 index 00000000000..94da46f6930 --- /dev/null +++ b/cpp/examples/tpch/q10.cpp @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../utilities/timer.hpp" +#include "utils.hpp" + +#include +#include +#include + +/** + * @file q10.cpp + * @brief Implement query 10 of the TPC-H benchmark. + * + * create view customer as select * from '/tables/scale-1/customer.parquet'; + * create view orders as select * from '/tables/scale-1/orders.parquet'; + * create view lineitem as select * from '/tables/scale-1/lineitem.parquet'; + * create view nation as select * from '/tables/scale-1/nation.parquet'; + * + * select + * c_custkey, + * c_name, + * sum(l_extendedprice * (1 - l_discount)) as revenue, + * c_acctbal, + * n_name, + * c_address, + * c_phone, + * c_comment + * from + * customer, + * orders, + * lineitem, + * nation + * where + * c_custkey = o_custkey + * and l_orderkey = o_orderkey + * and o_orderdate >= date '1993-10-01' + * and o_orderdate < date '1994-01-01' + * and l_returnflag = 'R' + * and c_nationkey = n_nationkey + * group by + * c_custkey, + * c_name, + * c_acctbal, + * c_phone, + * n_name, + * c_address, + * c_comment + * order by + * revenue desc; + */ + +/** + * @brief Calculate the revenue column + * + * @param extendedprice The extended price column + * @param discount The discount column + * @param stream The CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory. + */ +[[nodiscard]] std::unique_ptr calc_revenue( + cudf::column_view const& extendedprice, + cudf::column_view const& discount, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) +{ + auto const one = cudf::numeric_scalar(1); + auto const one_minus_discount = + cudf::binary_operation(one, discount, cudf::binary_operator::SUB, discount.type(), stream, mr); + auto const revenue_type = cudf::data_type{cudf::type_id::FLOAT64}; + auto revenue = cudf::binary_operation(extendedprice, + one_minus_discount->view(), + cudf::binary_operator::MUL, + revenue_type, + stream, + mr); + return revenue; +} +int main(int argc, char const** argv) +{ + auto const args = parse_args(argc, argv); + + // Use a memory pool + auto resource = create_memory_resource(args.memory_resource_type); + rmm::mr::set_current_device_resource(resource.get()); + + cudf::examples::timer timer; + + // Define the column projection and filter predicate for the `orders` table + std::vector const orders_cols = {"o_custkey", "o_orderkey", "o_orderdate"}; + auto const o_orderdate_ref = cudf::ast::column_reference(std::distance( + orders_cols.begin(), std::find(orders_cols.begin(), orders_cols.end(), "o_orderdate"))); + auto o_orderdate_lower = + cudf::timestamp_scalar(days_since_epoch(1993, 10, 1), true); + auto const o_orderdate_lower_limit = cudf::ast::literal(o_orderdate_lower); + auto const o_orderdate_pred_lower = cudf::ast::operation( + cudf::ast::ast_operator::GREATER_EQUAL, o_orderdate_ref, o_orderdate_lower_limit); + auto o_orderdate_upper = + cudf::timestamp_scalar(days_since_epoch(1994, 1, 1), true); + auto const o_orderdate_upper_limit = cudf::ast::literal(o_orderdate_upper); + auto const o_orderdate_pred_upper = + cudf::ast::operation(cudf::ast::ast_operator::LESS, o_orderdate_ref, o_orderdate_upper_limit); + auto const orders_pred = std::make_unique( + cudf::ast::ast_operator::LOGICAL_AND, o_orderdate_pred_lower, o_orderdate_pred_upper); + + auto const l_returnflag_ref = cudf::ast::column_reference(3); + auto r_scalar = cudf::string_scalar("R"); + auto const r_literal = cudf::ast::literal(r_scalar); + auto const lineitem_pred = std::make_unique( + cudf::ast::ast_operator::EQUAL, l_returnflag_ref, r_literal); + + // Read out the tables from parquet files + // while pushing down the column projections and filter predicates + auto const customer = read_parquet( + args.dataset_dir + "/customer.parquet", + {"c_custkey", "c_name", "c_nationkey", "c_acctbal", "c_address", "c_phone", "c_comment"}); + auto const orders = + read_parquet(args.dataset_dir + "/orders.parquet", orders_cols, std::move(orders_pred)); + auto const lineitem = + read_parquet(args.dataset_dir + "/lineitem.parquet", + {"l_extendedprice", "l_discount", "l_orderkey", "l_returnflag"}, + std::move(lineitem_pred)); + auto const nation = read_parquet(args.dataset_dir + "/nation.parquet", {"n_name", "n_nationkey"}); + + // Perform the joins + auto const join_a = apply_inner_join(customer, nation, {"c_nationkey"}, {"n_nationkey"}); + auto const join_b = apply_inner_join(lineitem, orders, {"l_orderkey"}, {"o_orderkey"}); + auto const joined_table = apply_inner_join(join_a, join_b, {"c_custkey"}, {"o_custkey"}); + + // Calculate and append the `revenue` column + auto revenue = + calc_revenue(joined_table->column("l_extendedprice"), joined_table->column("l_discount")); + (*joined_table).append(revenue, "revenue"); + + // Perform the groupby operation + auto const groupedby_table = apply_groupby( + joined_table, + groupby_context_t{ + {"c_custkey", "c_name", "c_acctbal", "c_phone", "n_name", "c_address", "c_comment"}, + { + {"revenue", {{cudf::aggregation::Kind::SUM, "revenue"}}}, + }}); + + // Perform the order by operation + auto const orderedby_table = + apply_orderby(groupedby_table, {"revenue"}, {cudf::order::DESCENDING}); + + timer.print_elapsed_millis(); + + // Write query result to a parquet file + orderedby_table->to_parquet("q10.parquet"); + return 0; +} diff --git a/cpp/examples/tpch/q5.cpp b/cpp/examples/tpch/q5.cpp index e56850b94d6..89396a6c968 100644 --- a/cpp/examples/tpch/q5.cpp +++ b/cpp/examples/tpch/q5.cpp @@ -44,14 +44,14 @@ * region * where * c_custkey = o_custkey - * and l_orderkey = o_orderkey - * and l_suppkey = s_suppkey - * and c_nationkey = s_nationkey - * and s_nationkey = n_nationkey - * and n_regionkey = r_regionkey - * and r_name = 'ASIA' - * and o_orderdate >= date '1994-01-01' - * and o_orderdate < date '1995-01-01' + * and l_orderkey = o_orderkey + * and l_suppkey = s_suppkey + * and c_nationkey = s_nationkey + * and s_nationkey = n_nationkey + * and n_regionkey = r_regionkey + * and r_name = 'ASIA' + * and o_orderdate >= date '1994-01-01' + * and o_orderdate < date '1995-01-01' * group by * n_name * order by @@ -109,7 +109,7 @@ int main(int argc, char const** argv) auto const o_orderdate_upper_limit = cudf::ast::literal(o_orderdate_upper); auto const o_orderdate_pred_upper = cudf::ast::operation(cudf::ast::ast_operator::LESS, o_orderdate_ref, o_orderdate_upper_limit); - auto orders_pred = std::make_unique( + auto const orders_pred = std::make_unique( cudf::ast::ast_operator::LOGICAL_AND, o_orderdate_pred_lower, o_orderdate_pred_upper); // Define the column projection and filter predicate for the `region` table @@ -118,7 +118,7 @@ int main(int argc, char const** argv) region_cols.begin(), std::find(region_cols.begin(), region_cols.end(), "r_name"))); auto r_name_value = cudf::string_scalar("ASIA"); auto const r_name_literal = cudf::ast::literal(r_name_value); - auto region_pred = std::make_unique( + auto const region_pred = std::make_unique( cudf::ast::ast_operator::EQUAL, r_name_ref, r_name_literal); // Read out the tables from parquet files diff --git a/cpp/examples/tpch/q6.cpp b/cpp/examples/tpch/q6.cpp index f11b3d6ab3b..405b2ac73ca 100644 --- a/cpp/examples/tpch/q6.cpp +++ b/cpp/examples/tpch/q6.cpp @@ -84,7 +84,7 @@ int main(int argc, char const** argv) cudf::ast::ast_operator::GREATER_EQUAL, shipdate_ref, shipdate_lower_literal); auto const shipdate_pred_b = cudf::ast::operation(cudf::ast::ast_operator::LESS, shipdate_ref, shipdate_upper_literal); - auto lineitem_pred = std::make_unique( + auto const lineitem_pred = std::make_unique( cudf::ast::ast_operator::LOGICAL_AND, shipdate_pred_a, shipdate_pred_b); auto lineitem = read_parquet(args.dataset_dir + "/lineitem.parquet", lineitem_cols, std::move(lineitem_pred)); From 24997fda194d5b8af34048a8bf275830cabbff8c Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Fri, 26 Jul 2024 18:37:30 -0700 Subject: [PATCH 005/270] Deduplicate decimal32/decimal64 to decimal128 conversion function (#16236) Closes #16194 This PR deduplicates the `convert_data_to_decimal128` function from `to_arrow.cu`, `writer_impl.cu` and `to_arrow_device.cu` to a common location. Authors: - Muhammad Haseeb (https://github.com/mhaseeb123) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Nghia Truong (https://github.com/ttnghia) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16236 --- cpp/CMakeLists.txt | 1 + .../interop/decimal_conversion_utilities.cu | 70 +++++++++++++++++ .../interop/decimal_conversion_utilities.cuh | 44 +++++++++++ cpp/src/interop/to_arrow.cu | 8 +- cpp/src/interop/to_arrow_device.cu | 5 +- cpp/src/interop/to_arrow_host.cu | 40 +--------- cpp/src/io/parquet/writer_impl.cu | 60 ++++----------- cpp/tests/interop/to_arrow_device_test.cpp | 77 +++++++++++++++++++ 8 files changed, 220 insertions(+), 85 deletions(-) create mode 100644 cpp/src/interop/decimal_conversion_utilities.cu create mode 100644 cpp/src/interop/decimal_conversion_utilities.cuh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 95c509efc5b..310bc99b279 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -365,6 +365,7 @@ add_library( src/interop/dlpack.cpp src/interop/from_arrow.cu src/interop/arrow_utilities.cpp + src/interop/decimal_conversion_utilities.cu src/interop/to_arrow.cu src/interop/to_arrow_device.cu src/interop/to_arrow_host.cu diff --git a/cpp/src/interop/decimal_conversion_utilities.cu b/cpp/src/interop/decimal_conversion_utilities.cu new file mode 100644 index 00000000000..2f81c754a30 --- /dev/null +++ b/cpp/src/interop/decimal_conversion_utilities.cu @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "decimal_conversion_utilities.cuh" + +#include +#include +#include + +#include + +#include + +#include + +namespace cudf { +namespace detail { + +template +std::unique_ptr convert_decimals_to_decimal128( + cudf::column_view const& column, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) +{ + static_assert(std::is_same_v or std::is_same_v, + "Only int32 and int64 decimal types can be converted to decimal128."); + + constexpr size_type BIT_WIDTH_RATIO = sizeof(__int128_t) / sizeof(DecimalType); + auto buf = std::make_unique(column.size() * sizeof(__int128_t), stream, mr); + + thrust::for_each(rmm::exec_policy_nosync(stream, mr), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(column.size()), + [in = column.begin(), + out = reinterpret_cast(buf->data()), + BIT_WIDTH_RATIO] __device__(auto in_idx) { + auto const out_idx = in_idx * BIT_WIDTH_RATIO; + // the lowest order bits are the value, the remainder + // simply matches the sign bit to satisfy the two's + // complement integer representation of negative numbers. + out[out_idx] = in[in_idx]; +#pragma unroll BIT_WIDTH_RATIO - 1 + for (auto i = 1; i < BIT_WIDTH_RATIO; ++i) { + out[out_idx + i] = in[in_idx] < 0 ? -1 : 0; + } + }); + + return buf; +} + +// Instantiate templates for int32_t and int64_t decimal types +template std::unique_ptr convert_decimals_to_decimal128( + cudf::column_view const& column, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); + +template std::unique_ptr convert_decimals_to_decimal128( + cudf::column_view const& column, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); + +} // namespace detail +} // namespace cudf diff --git a/cpp/src/interop/decimal_conversion_utilities.cuh b/cpp/src/interop/decimal_conversion_utilities.cuh new file mode 100644 index 00000000000..41263147404 --- /dev/null +++ b/cpp/src/interop/decimal_conversion_utilities.cuh @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include + +#include + +namespace cudf::detail { + +/** + * @brief Convert decimal32 and decimal64 numeric data to decimal128 and return the device vector + * + * @tparam DecimalType to convert from + * + * @param column A view of the input columns + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource to use for device memory allocation + * + * @return A device vector containing the converted decimal128 data + */ +template +std::unique_ptr convert_decimals_to_decimal128( + cudf::column_view const& input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); + +} // namespace cudf::detail diff --git a/cpp/src/interop/to_arrow.cu b/cpp/src/interop/to_arrow.cu index 6b163e3441e..3d41f856f4f 100644 --- a/cpp/src/interop/to_arrow.cu +++ b/cpp/src/interop/to_arrow.cu @@ -15,6 +15,7 @@ */ #include "arrow_utilities.hpp" +#include "decimal_conversion_utilities.cuh" #include "detail/arrow_allocator.hpp" #include @@ -158,8 +159,11 @@ std::shared_ptr unsupported_decimals_to_arrow(column_view input, arrow::MemoryPool* ar_mr, rmm::cuda_stream_view stream) { - auto buf = - detail::decimals_to_arrow(input, stream, rmm::mr::get_current_device_resource()); + auto buf = detail::convert_decimals_to_decimal128( + input, stream, rmm::mr::get_current_device_resource()); + + // Synchronize stream here to ensure the decimal128 buffer is ready. + stream.synchronize(); auto const buf_size_in_bytes = buf->size(); auto data_buffer = allocate_arrow_buffer(buf_size_in_bytes, ar_mr); diff --git a/cpp/src/interop/to_arrow_device.cu b/cpp/src/interop/to_arrow_device.cu index 2eb9b912054..cea7cdebcba 100644 --- a/cpp/src/interop/to_arrow_device.cu +++ b/cpp/src/interop/to_arrow_device.cu @@ -15,6 +15,7 @@ */ #include "arrow_utilities.hpp" +#include "decimal_conversion_utilities.cuh" #include #include @@ -141,7 +142,9 @@ int construct_decimals(cudf::column_view input, nanoarrow::UniqueArray tmp; NANOARROW_RETURN_NOT_OK(initialize_array(tmp.get(), NANOARROW_TYPE_DECIMAL128, input)); - auto buf = detail::decimals_to_arrow(input, stream, mr); + auto buf = detail::convert_decimals_to_decimal128(input, stream, mr); + // Synchronize stream here to ensure the decimal128 buffer is ready. + stream.synchronize(); NANOARROW_RETURN_NOT_OK(set_buffer(std::move(buf), fixed_width_data_buffer_idx, tmp.get())); ArrowArrayMove(tmp.get(), out); diff --git a/cpp/src/interop/to_arrow_host.cu b/cpp/src/interop/to_arrow_host.cu index c9e53ebaab7..193b3a3b5a2 100644 --- a/cpp/src/interop/to_arrow_host.cu +++ b/cpp/src/interop/to_arrow_host.cu @@ -15,6 +15,7 @@ */ #include "arrow_utilities.hpp" +#include "decimal_conversion_utilities.cuh" #include #include @@ -50,41 +51,6 @@ namespace cudf { namespace detail { -template -std::unique_ptr decimals_to_arrow(cudf::column_view input, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - constexpr size_type BIT_WIDTH_RATIO = sizeof(__int128_t) / sizeof(DeviceType); - auto buf = std::make_unique(input.size() * sizeof(__int128_t), stream, mr); - - auto count = thrust::counting_iterator(0); - thrust::for_each(rmm::exec_policy(stream, mr), - count, - count + input.size(), - [in = input.begin(), - out = reinterpret_cast(buf->data()), - BIT_WIDTH_RATIO] __device__(auto in_idx) { - auto const out_idx = in_idx * BIT_WIDTH_RATIO; - // the lowest order bits are the value, the remainder - // simply matches the sign bit to satisfy the two's - // complement integer representation of negative numbers. - out[out_idx] = in[in_idx]; -#pragma unroll BIT_WIDTH_RATIO - 1 - for (auto i = 1; i < BIT_WIDTH_RATIO; ++i) { - out[out_idx + i] = in[in_idx] < 0 ? -1 : 0; - } - }); - - return buf; -} - -template std::unique_ptr decimals_to_arrow( - cudf::column_view input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); - -template std::unique_ptr decimals_to_arrow( - cudf::column_view input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); - namespace { struct dispatch_to_arrow_host { @@ -156,7 +122,9 @@ struct dispatch_to_arrow_host { NANOARROW_RETURN_NOT_OK(initialize_array(tmp.get(), NANOARROW_TYPE_DECIMAL128, column)); NANOARROW_RETURN_NOT_OK(populate_validity_bitmap(ArrowArrayValidityBitmap(tmp.get()))); - auto buf = detail::decimals_to_arrow(column, stream, mr); + auto buf = detail::convert_decimals_to_decimal128(column, stream, mr); + // No need to synchronize stream here as populate_data_buffer uses the same stream to copy data + // to host. NANOARROW_RETURN_NOT_OK( populate_data_buffer(device_span<__int128_t const>( reinterpret_cast(buf->data()), column.size()), diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 2df71b77301..36a1d8377bf 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -22,6 +22,7 @@ #include "arrow_schema_writer.hpp" #include "compact_protocol_reader.hpp" #include "compact_protocol_writer.hpp" +#include "interop/decimal_conversion_utilities.cuh" #include "io/comp/nvcomp_adapter.hpp" #include "io/parquet/parquet.hpp" #include "io/parquet/parquet_gpu.hpp" @@ -1601,50 +1602,12 @@ size_t column_index_buffer_size(EncColumnChunk* ck, return ck->ck_stat_size * num_pages + column_index_truncate_length + padding + size_struct_size; } -/** - * @brief Convert decimal32 and decimal64 data to decimal128 and return the device vector - * - * @tparam DecimalType to convert from - * - * @param column A view of the input columns - * @param stream CUDA stream used for device memory operations and kernel launches - * - * @return A device vector containing the converted decimal128 data - */ -template -rmm::device_uvector<__int128_t> convert_data_to_decimal128(column_view const& column, - rmm::cuda_stream_view stream) -{ - size_type constexpr BIT_WIDTH_RATIO = sizeof(__int128_t) / sizeof(DecimalType); - - rmm::device_uvector<__int128_t> d128_buffer(column.size(), stream); - - thrust::for_each(rmm::exec_policy_nosync(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(column.size()), - [in = column.begin(), - out = reinterpret_cast(d128_buffer.data()), - BIT_WIDTH_RATIO] __device__(auto in_idx) { - auto const out_idx = in_idx * BIT_WIDTH_RATIO; - // The lowest order bits are the value, the remainder - // simply matches the sign bit to satisfy the two's - // complement integer representation of negative numbers. - out[out_idx] = in[in_idx]; -#pragma unroll BIT_WIDTH_RATIO - 1 - for (auto i = 1; i < BIT_WIDTH_RATIO; ++i) { - out[out_idx + i] = in[in_idx] < 0 ? -1 : 0; - } - }); - - return d128_buffer; -} - /** * @brief Function to convert decimal32 and decimal64 columns to decimal128 data, * update the input table metadata, and return a new vector of column views. * * @param[in,out] table_meta The table metadata - * @param[in,out] d128_vectors Vector containing the computed decimal128 data buffers. + * @param[in,out] d128_buffers Buffers containing the converted decimal128 data. * @param input The input table * @param stream CUDA stream used for device memory operations and kernel launches * @@ -1652,7 +1615,7 @@ rmm::device_uvector<__int128_t> convert_data_to_decimal128(column_view const& co */ std::vector convert_decimal_columns_and_metadata( table_input_metadata& table_meta, - std::vector>& d128_vectors, + std::vector>& d128_buffers, table_view const& table, rmm::cuda_stream_view stream) { @@ -1673,28 +1636,30 @@ std::vector convert_decimal_columns_and_metadata( switch (column.type().id()) { case type_id::DECIMAL32: // Convert data to decimal128 type - d128_vectors.emplace_back(convert_data_to_decimal128(column, stream)); + d128_buffers.emplace_back(cudf::detail::convert_decimals_to_decimal128( + column, stream, rmm::mr::get_current_device_resource())); // Update metadata metadata.set_decimal_precision(MAX_DECIMAL32_PRECISION); metadata.set_type_length(size_of(data_type{type_id::DECIMAL128, column.type().scale()})); // Create a new column view from the d128 data vector return {data_type{type_id::DECIMAL128, column.type().scale()}, column.size(), - d128_vectors.back().data(), + d128_buffers.back()->data(), column.null_mask(), column.null_count(), column.offset(), converted_children}; case type_id::DECIMAL64: // Convert data to decimal128 type - d128_vectors.emplace_back(convert_data_to_decimal128(column, stream)); + d128_buffers.emplace_back(cudf::detail::convert_decimals_to_decimal128( + column, stream, rmm::mr::get_current_device_resource())); // Update metadata metadata.set_decimal_precision(MAX_DECIMAL64_PRECISION); metadata.set_type_length(size_of(data_type{type_id::DECIMAL128, column.type().scale()})); // Create a new column view from the d128 data vector return {data_type{type_id::DECIMAL128, column.type().scale()}, column.size(), - d128_vectors.back().data(), + d128_buffers.back()->data(), column.null_mask(), column.null_count(), column.offset(), @@ -1722,6 +1687,9 @@ std::vector convert_decimal_columns_and_metadata( std::back_inserter(converted_column_views), [&](auto elem) { return convert_column(thrust::get<0>(elem), thrust::get<1>(elem)); }); + // Synchronize stream here to ensure all decimal128 buffers are ready. + stream.synchronize(); + return converted_column_views; } @@ -1780,13 +1748,13 @@ auto convert_table_to_parquet_data(table_input_metadata& table_meta, rmm::cuda_stream_view stream) { // Container to store decimal128 converted data if needed - std::vector> d128_vectors; + std::vector> d128_buffers; // Convert decimal32/decimal64 data to decimal128 if writing arrow schema // and initialize LinkedColVector auto vec = table_to_linked_columns( (write_arrow_schema) - ? table_view({convert_decimal_columns_and_metadata(table_meta, d128_vectors, input, stream)}) + ? table_view({convert_decimal_columns_and_metadata(table_meta, d128_buffers, input, stream)}) : input); auto schema_tree = construct_parquet_schema_tree( diff --git a/cpp/tests/interop/to_arrow_device_test.cpp b/cpp/tests/interop/to_arrow_device_test.cpp index 77da4039103..51216a8512c 100644 --- a/cpp/tests/interop/to_arrow_device_test.cpp +++ b/cpp/tests/interop/to_arrow_device_test.cpp @@ -710,6 +710,83 @@ TEST_F(ToArrowDeviceTest, StructColumn) template using fp_wrapper = cudf::test::fixed_point_column_wrapper; +TEST_F(ToArrowDeviceTest, FixedPoint32Table) +{ + using namespace numeric; + + for (auto const scale : {6, 4, 2, 0, -1, -3, -5}) { + auto const expect_data = + std::vector{-1000, -1, -1, -1, 2400, 0, 0, 0, -3456, -1, -1, -1, + 4650, 0, 0, 0, 5154, 0, 0, 0, 6800, 0, 0, 0}; + auto col = fp_wrapper({-1000, 2400, -3456, 4650, 5154, 6800}, scale_type{scale}); + std::vector> table_cols; + table_cols.emplace_back(col.release()); + auto input = cudf::table(std::move(table_cols)); + + nanoarrow::UniqueSchema expected_schema; + ArrowSchemaInit(expected_schema.get()); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetTypeStruct(expected_schema.get(), 1)); + ArrowSchemaInit(expected_schema->children[0]); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetTypeDecimal(expected_schema->children[0], + NANOARROW_TYPE_DECIMAL128, + cudf::detail::max_precision(), + -scale)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(expected_schema->children[0], "a")); + expected_schema->children[0]->flags = 0; + + auto got_arrow_schema = + cudf::to_arrow_schema(input.view(), std::vector{{"a"}}); + compare_schemas(expected_schema.get(), got_arrow_schema.get()); + + auto result_dev_data = std::make_unique>( + expect_data.size(), cudf::get_default_stream()); + cudaMemcpy(result_dev_data->data(), + expect_data.data(), + sizeof(int32_t) * expect_data.size(), + cudaMemcpyHostToDevice); + + cudf::get_default_stream().synchronize(); + nanoarrow::UniqueArray expected_array; + NANOARROW_THROW_NOT_OK( + ArrowArrayInitFromSchema(expected_array.get(), expected_schema.get(), nullptr)); + expected_array->length = input.num_rows(); + + expected_array->children[0]->length = input.num_rows(); + NANOARROW_THROW_NOT_OK( + ArrowBufferSetAllocator(ArrowArrayBuffer(expected_array->children[0], 0), noop_alloc)); + ArrowArrayValidityBitmap(expected_array->children[0])->buffer.data = + const_cast(reinterpret_cast(input.view().column(0).null_mask())); + + auto data_ptr = reinterpret_cast(result_dev_data->data()); + NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator( + ArrowArrayBuffer(expected_array->children[0], 1), + ArrowBufferDeallocator( + [](ArrowBufferAllocator* alloc, uint8_t*, int64_t) { + auto buf = + reinterpret_cast>*>(alloc->private_data); + delete buf; + }, + new std::unique_ptr>(std::move(result_dev_data))))); + ArrowArrayBuffer(expected_array->children[0], 1)->data = data_ptr; + NANOARROW_THROW_NOT_OK( + ArrowArrayFinishBuilding(expected_array.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); + + auto got_arrow_array = cudf::to_arrow_device(input.view()); + ASSERT_EQ(rmm::get_current_cuda_device().value(), got_arrow_array->device_id); + ASSERT_EQ(ARROW_DEVICE_CUDA, got_arrow_array->device_type); + ASSERT_CUDA_SUCCEEDED( + cudaEventSynchronize(*reinterpret_cast(got_arrow_array->sync_event))); + compare_arrays(expected_schema.get(), expected_array.get(), &got_arrow_array->array); + + got_arrow_array = cudf::to_arrow_device(std::move(input)); + ASSERT_EQ(rmm::get_current_cuda_device().value(), got_arrow_array->device_id); + ASSERT_EQ(ARROW_DEVICE_CUDA, got_arrow_array->device_type); + ASSERT_CUDA_SUCCEEDED( + cudaEventSynchronize(*reinterpret_cast(got_arrow_array->sync_event))); + compare_arrays(expected_schema.get(), expected_array.get(), &got_arrow_array->array); + } +} + TEST_F(ToArrowDeviceTest, FixedPoint64Table) { using namespace numeric; From 18c1465b597284d8b558964cc0ca48de7da60a17 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jul 2024 06:06:07 -1000 Subject: [PATCH 006/270] Align ewm APIs with pandas 2.x (#16413) These all currently are not implemented and raise a `NotImplementedError` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16413 --- python/cudf/cudf/core/window/ewm.py | 52 ++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/core/window/ewm.py b/python/cudf/cudf/core/window/ewm.py index bb153d4b549..1203a840076 100644 --- a/python/cudf/cudf/core/window/ewm.py +++ b/python/cudf/cudf/core/window/ewm.py @@ -114,23 +114,57 @@ def __init__( self.adjust = adjust self.com = get_center_of_mass(com, span, halflife, alpha) - def mean(self): + def online(self, engine: str = "numba", engine_kwargs=None): + """ + Return an ``OnlineExponentialMovingWindow`` object to calculate + exponentially moving window aggregations in an online method. + + Currently not supported. + """ + raise NotImplementedError("online is currently not supported.") + + def mean( + self, numeric_only: bool = False, engine=None, engine_kwargs=None + ): """ Calculate the ewm (exponential weighted moment) mean. """ + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is currently not supported." + ) + if engine is not None: + raise NotImplementedError( + "engine is non-functional and added for compatibility with pandas." + ) + if engine_kwargs is not None: + raise NotImplementedError( + "engine_kwargs is non-functional and added for compatibility with pandas." + ) return self._apply_agg("ewma") - def var(self, bias): - raise NotImplementedError("ewmvar not yet supported.") + def sum(self, numeric_only: bool = False, engine=None, engine_kwargs=None): + raise NotImplementedError("sum not yet supported.") - def std(self, bias): - raise NotImplementedError("ewmstd not yet supported.") + def var(self, bias: bool = False, numeric_only: bool = False): + raise NotImplementedError("var not yet supported.") - def corr(self, other): - raise NotImplementedError("ewmcorr not yet supported.") + def std(self, bias: bool = False, numeric_only: bool = False): + raise NotImplementedError("std not yet supported.") - def cov(self, other): - raise NotImplementedError("ewmcov not yet supported.") + def corr( + self, other, pairwise: bool | None = None, numeric_only: bool = False + ): + raise NotImplementedError("corr not yet supported.") + + def cov( + self, + other, + pairwise: bool | None = None, + bias: bool = False, + numeric_only: bool = False, + ): + raise NotImplementedError("cov not yet supported.") def _apply_agg_series(self, sr, agg_name): if not is_numeric_dtype(sr.dtype): From 58f47242fe04b1e25fd42e1e45e8c15417140777 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jul 2024 06:09:21 -1000 Subject: [PATCH 007/270] Align groupby APIs with pandas 2.x (#16403) The following breaking APIs are affected: * `apply` * `transform` * `describe` The rest of the APIs are non-breaking and generally will raise a `NotImplementedError` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16403 --- .../source/user_guide/api_docs/groupby.rst | 3 +- python/cudf/cudf/core/groupby/groupby.py | 629 ++++++++++++++---- python/cudf/cudf/core/resample.py | 6 +- python/cudf/cudf/tests/test_groupby.py | 25 + 4 files changed, 514 insertions(+), 149 deletions(-) diff --git a/docs/cudf/source/user_guide/api_docs/groupby.rst b/docs/cudf/source/user_guide/api_docs/groupby.rst index 80811efa33f..ca29087cbf9 100644 --- a/docs/cudf/source/user_guide/api_docs/groupby.rst +++ b/docs/cudf/source/user_guide/api_docs/groupby.rst @@ -68,7 +68,6 @@ Computations / descriptive stats GroupBy.std GroupBy.sum GroupBy.var - GroupBy.corr GroupBy.cov The following methods are available in both ``SeriesGroupBy`` and @@ -81,6 +80,7 @@ application to columns of a specific data type. :toctree: api/ DataFrameGroupBy.bfill + DataFrameGroupBy.corr DataFrameGroupBy.count DataFrameGroupBy.cumcount DataFrameGroupBy.cummax @@ -102,5 +102,6 @@ The following methods are available only for ``SeriesGroupBy`` objects. .. autosummary:: :toctree: api/ + SeriesGroupBy.corr SeriesGroupBy.nunique SeriesGroupBy.unique diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 1646c5042fd..3cfbd1d736a 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -8,7 +8,7 @@ import warnings from collections import abc from functools import cached_property -from typing import TYPE_CHECKING, Any, Iterable +from typing import TYPE_CHECKING, Any, Iterable, Literal import cupy as cp import numpy as np @@ -306,6 +306,18 @@ def __iter__(self): grouped_values[offsets[i] : offsets[i + 1]], ) + def __len__(self) -> int: + return self.ngroups + + @property + def ngroups(self) -> int: + _, offsets, _, _ = self._grouped() + return len(offsets) - 1 + + @property + def ndim(self) -> int: + return self.obj.ndim + @property def dtypes(self): """ @@ -457,10 +469,20 @@ def size(self): ) @_performance_tracking - def cumcount(self): + def cumcount(self, ascending: bool = True): """ Return the cumulative count of keys in each group. + + Parameters + ---------- + ascending : bool, default True + If False, number in reverse, from length of group - 1 to 0. + Currently not supported """ + if ascending is not True: + raise NotImplementedError( + "ascending is currently not implemented." + ) return ( cudf.Series( cudf.core.column.column_empty( @@ -527,7 +549,7 @@ def _groupby(self): ) @_performance_tracking - def agg(self, func): + def agg(self, func, *args, engine=None, engine_kwargs=None, **kwargs): """ Apply aggregation(s) to the groups. @@ -615,6 +637,22 @@ def agg(self, func): 1 1.5 1.75 2.0 2.0 2 3.0 3.00 1.0 1.0 """ + if engine is not None: + raise NotImplementedError( + "engine is non-functional and added for compatibility with pandas" + ) + if engine_kwargs is not None: + raise NotImplementedError( + "engine_kwargs is non-functional added for compatibility with pandas" + ) + if args: + raise NotImplementedError( + "Passing args to func is currently not supported." + ) + if kwargs: + raise NotImplementedError( + "Passing kwargs to func is currently not supported." + ) column_names, columns, normalized_aggs = self._normalize_aggs(func) orig_dtypes = tuple(c.dtype for c in columns) @@ -935,12 +973,13 @@ def tail(self, n: int = 5, *, preserve_order: bool = True): ) @_performance_tracking - def nth(self, n): + def nth(self, n, dropna: Literal["any", "all", None] = None): """ Return the nth row from each group. """ - - self.obj["__groupbynth_order__"] = range(0, len(self.obj)) + if dropna is not None: + raise NotImplementedError("dropna is not currently supported.") + self.obj["__groupbynth_order__"] = range(0, len(self.obj)) # type: ignore[index] # We perform another groupby here to have the grouping columns # be a part of dataframe columns. result = self.obj.groupby(self.grouping.keys).agg(lambda x: x.nth(n)) @@ -1423,13 +1462,13 @@ def _post_process_chunk_results( @_performance_tracking def apply( - self, function, *args, engine="auto", include_groups: bool = True + self, func, *args, engine="auto", include_groups: bool = True, **kwargs ): """Apply a python transformation function over the grouped chunk. Parameters ---------- - function : callable + func : callable The python transformation function that will be applied on the grouped chunk. args : tuple @@ -1452,6 +1491,9 @@ def apply( When True, will attempt to apply ``func`` to the groupings in the case that they are columns of the DataFrame. In the future, this will default to ``False``. + kwargs : dict + Optional keyword arguments to pass to the function. + Currently not supported Examples -------- @@ -1528,13 +1570,17 @@ def mult(df): dtype: int64 """ + if kwargs: + raise NotImplementedError( + "Passing kwargs to func is currently not supported." + ) if self.obj.empty: - if function in {"count", "size", "idxmin", "idxmax"}: + if func in {"count", "size", "idxmin", "idxmax"}: res = cudf.Series([], dtype="int64") else: res = self.obj.copy(deep=True) res.index = self.grouping.keys - if function in {"sum", "product"}: + if func in {"sum", "product"}: # For `sum` & `product`, boolean types # will need to result in `int64` type. for name, col in res._data.items(): @@ -1542,20 +1588,20 @@ def mult(df): res._data[name] = col.astype("int") return res - if not callable(function): - raise TypeError(f"type {type(function)} is not callable") + if not callable(func): + raise TypeError(f"type {type(func)} is not callable") group_names, offsets, group_keys, grouped_values = self._grouped( include_groups=include_groups ) if engine == "auto": - if _can_be_jitted(grouped_values, function, args): + if _can_be_jitted(grouped_values, func, args): engine = "jit" else: engine = "cudf" if engine == "jit": result = self._jit_groupby_apply( - function, + func, group_names, offsets, group_keys, @@ -1564,7 +1610,7 @@ def mult(df): ) elif engine == "cudf": result = self._iterative_groupby_apply( - function, + func, group_names, offsets, group_keys, @@ -1744,12 +1790,14 @@ def _broadcast(self, values: cudf.Series) -> cudf.Series: return values @_performance_tracking - def transform(self, function): + def transform( + self, func, *args, engine=None, engine_kwargs=None, **kwargs + ): """Apply an aggregation, then broadcast the result to the group size. Parameters ---------- - function: str or callable + func: str or callable Aggregation to apply to each group. Note that the set of operations currently supported by `transform` is identical to that supported by the `agg` method. @@ -1778,18 +1826,35 @@ def transform(self, function): -------- agg """ - if not (isinstance(function, str) or callable(function)): + if engine is not None: + raise NotImplementedError( + "engine is non-functional and added for compatibility with pandas" + ) + if engine_kwargs is not None: + raise NotImplementedError( + "engine_kwargs is non-functional added for compatibility with pandas" + ) + if args: + raise NotImplementedError( + "Passing args to func is currently not supported." + ) + if kwargs: + raise NotImplementedError( + "Passing kwargs to func is currently not supported." + ) + + if not (isinstance(func, str) or callable(func)): raise TypeError( "Aggregation must be a named aggregation or a callable" ) try: - result = self.agg(function) + result = self.agg(func) except TypeError as e: raise NotImplementedError( "Currently, `transform()` supports only aggregations." ) from e # If the aggregation is a scan, don't broadcast - if libgroupby._is_all_scan_aggregate([[function]]): + if libgroupby._is_all_scan_aggregate([[func]]): if len(result) != len(self.obj): raise AssertionError( "Unexpected result length for scan transform" @@ -1824,7 +1889,7 @@ def func(x): return self.agg(func) @_performance_tracking - def describe(self, include=None, exclude=None): + def describe(self, percentiles=None, include=None, exclude=None): """ Generate descriptive statistics that summarizes the central tendency, dispersion and shape of a dataset's distribution, excluding NaN values. @@ -1833,6 +1898,10 @@ def describe(self, include=None, exclude=None): Parameters ---------- + percentiles : list-like of numbers, optional + The percentiles to include in the output. + Currently not supported. + include: 'all', list-like of dtypes or None (default), optional list of data types to include in the result. Ignored for Series. @@ -1869,8 +1938,12 @@ def describe(self, include=None, exclude=None): 90 1 24.0 24.0 24.0 24.0 24.0 24.0 """ - if exclude is not None and include is not None: - raise NotImplementedError + if percentiles is not None: + raise NotImplementedError("percentiles is currently not supported") + if exclude is not None: + raise NotImplementedError("exclude is currently not supported") + if include is not None: + raise NotImplementedError("include is currently not supported") res = self.agg( [ @@ -1896,69 +1969,7 @@ def describe(self, include=None, exclude=None): return res @_performance_tracking - def corr(self, method="pearson", min_periods=1): - """ - Compute pairwise correlation of columns, excluding NA/null values. - - Parameters - ---------- - method: {"pearson", "kendall", "spearman"} or callable, - default "pearson". Currently only the pearson correlation - coefficient is supported. - - min_periods: int, optional - Minimum number of observations required per pair of columns - to have a valid result. - - Returns - ------- - DataFrame - Correlation matrix. - - Examples - -------- - >>> import cudf - >>> gdf = cudf.DataFrame({ - ... "id": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], - ... "val1": [5, 4, 6, 4, 8, 7, 4, 5, 2], - ... "val2": [4, 5, 6, 1, 2, 9, 8, 5, 1], - ... "val3": [4, 5, 6, 1, 2, 9, 8, 5, 1]}) - >>> gdf - id val1 val2 val3 - 0 a 5 4 4 - 1 a 4 5 5 - 2 a 6 6 6 - 3 b 4 1 1 - 4 b 8 2 2 - 5 b 7 9 9 - 6 c 4 8 8 - 7 c 5 5 5 - 8 c 2 1 1 - >>> gdf.groupby("id").corr(method="pearson") - val1 val2 val3 - id - a val1 1.000000 0.500000 0.500000 - val2 0.500000 1.000000 1.000000 - val3 0.500000 1.000000 1.000000 - b val1 1.000000 0.385727 0.385727 - val2 0.385727 1.000000 1.000000 - val3 0.385727 1.000000 1.000000 - c val1 1.000000 0.714575 0.714575 - val2 0.714575 1.000000 1.000000 - val3 0.714575 1.000000 1.000000 - """ - - if method.lower() not in ("pearson",): - raise NotImplementedError( - "Only pearson correlation is currently supported" - ) - - return self._cov_or_corr( - lambda x: x.corr(method, min_periods), "Correlation" - ) - - @_performance_tracking - def cov(self, min_periods=0, ddof=1): + def cov(self, min_periods=0, ddof=1, numeric_only: bool = False): """ Compute the pairwise covariance among the columns of a DataFrame, excluding NA/null values. @@ -2042,6 +2053,10 @@ def cov(self, min_periods=0, ddof=1): val2 3.833333 12.333333 12.333333 val3 3.833333 12.333333 12.333333 """ + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is currently not supported." + ) return self._cov_or_corr( lambda x: x.cov(min_periods, ddof), "Covariance" @@ -2137,7 +2152,13 @@ def _cov_or_corr(self, func, method_name): return res @_performance_tracking - def var(self, ddof=1): + def var( + self, + ddof=1, + engine=None, + engine_kwargs=None, + numeric_only: bool = False, + ): """Compute the column-wise variance of the values in each group. Parameters @@ -2146,6 +2167,18 @@ def var(self, ddof=1): The delta degrees of freedom. N - ddof is the divisor used to normalize the variance. """ + if engine is not None: + raise NotImplementedError( + "engine is non-functional and added for compatibility with pandas" + ) + if engine_kwargs is not None: + raise NotImplementedError( + "engine_kwargs is non-functional added for compatibility with pandas" + ) + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is currently not supported." + ) def func(x): return getattr(x, "var")(ddof=ddof) @@ -2153,7 +2186,13 @@ def func(x): return self.agg(func) @_performance_tracking - def std(self, ddof=1): + def std( + self, + ddof=1, + engine=None, + engine_kwargs=None, + numeric_only: bool = False, + ): """Compute the column-wise std of the values in each group. Parameters @@ -2162,6 +2201,18 @@ def std(self, ddof=1): The delta degrees of freedom. N - ddof is the divisor used to normalize the standard deviation. """ + if engine is not None: + raise NotImplementedError( + "engine is non-functional and added for compatibility with pandas" + ) + if engine_kwargs is not None: + raise NotImplementedError( + "engine_kwargs is non-functional added for compatibility with pandas" + ) + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is currently not supported." + ) def func(x): return getattr(x, "std")(ddof=ddof) @@ -2169,7 +2220,9 @@ def func(x): return self.agg(func) @_performance_tracking - def quantile(self, q=0.5, interpolation="linear"): + def quantile( + self, q=0.5, interpolation="linear", numeric_only: bool = False + ): """Compute the column-wise quantiles of the values in each group. Parameters @@ -2179,7 +2232,14 @@ def quantile(self, q=0.5, interpolation="linear"): interpolation : {"linear", "lower", "higher", "midpoint", "nearest"} The interpolation method to use when the desired quantile lies between two data points. Defaults to "linear". + numeric_only : bool, default False + Include only `float`, `int` or `boolean` data. + Currently not supported """ + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is not currently supported." + ) def func(x): return getattr(x, "quantile")(q=q, interpolation=interpolation) @@ -2333,7 +2393,14 @@ def fillna( ) @_performance_tracking - def shift(self, periods=1, freq=None, axis=0, fill_value=None): + def shift( + self, + periods=1, + freq=None, + axis=0, + fill_value=None, + suffix: str | None = None, + ): """ Shift each group by ``periods`` positions. @@ -2355,6 +2422,10 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=None): the list. The length of the list should match the number of columns shifted. Each value should match the data type of the column to fill. + suffix : str, optional + A string to add to each shifted column if there are multiple periods. + Ignored otherwise. + Currently not supported. Returns ------- @@ -2374,6 +2445,9 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=None): if not axis == 0: raise NotImplementedError("Only axis=0 is supported.") + if suffix is not None: + raise NotImplementedError("shift is not currently supported.") + values = self.grouping.values if is_list_like(fill_value): if len(fill_value) != len(values._data): @@ -2473,6 +2547,142 @@ def pct_change( shifted = fill_grp.shift(periods=periods, freq=freq) return (filled / shifted) - 1 + def _mimic_pandas_order( + self, result: DataFrameOrSeries + ) -> DataFrameOrSeries: + """Given a groupby result from libcudf, reconstruct the row orders + matching that of pandas. This also adds appropriate indices. + """ + # TODO: copy metadata after this method is a common pattern, should + # merge in this method. + + # This function is used to reorder the results of scan-based + # groupbys which have the same output size as input size. + # However, if the grouping key has NAs and dropna=True, the + # result coming back from libcudf has null_count few rows than + # the input, so we must produce an ordering from the full + # input range. + _, _, (ordering,) = self._groupby.groups( + [as_column(range(0, len(self.obj)))] + ) + if self._dropna and any( + c.has_nulls(include_nan=True) > 0 + for c in self.grouping._key_columns + ): + # Scan aggregations with null/nan keys put nulls in the + # corresponding output rows in pandas, to do that here + # expand the result by reindexing. + ri = cudf.RangeIndex(0, len(self.obj)) + result.index = cudf.Index(ordering) + # This reorders and expands + result = result.reindex(ri) + else: + # Just reorder according to the groupings + result = result.take(ordering.argsort()) + # Now produce the actual index we first thought of + result.index = self.obj.index + return result + + def ohlc(self): + """ + Compute open, high, low and close values of a group, excluding missing values. + + Currently not implemented. + """ + raise NotImplementedError("ohlc is currently not implemented") + + @property + def plot(self): + """ + Make plots of a grouped Series or DataFrame. + + Currently not implemented. + """ + raise NotImplementedError("plot is currently not implemented") + + def resample(self, rule, *args, include_groups: bool = True, **kwargs): + """ + Provide resampling when using a TimeGrouper. + + Currently not implemented. + """ + raise NotImplementedError("resample is currently not implemented") + + def take(self, indices): + """ + Return the elements in the given *positional* indices in each group. + + Currently not implemented. + """ + raise NotImplementedError("take is currently not implemented") + + def filter(self, func, dropna: bool = True, *args, **kwargs): + """ + Filter elements from groups that don't satisfy a criterion. + + Currently not implemented. + """ + raise NotImplementedError("filter is currently not implemented") + + def expanding(self, *args, **kwargs): + """ + Return an expanding grouper, providing expanding + functionality per group. + + Currently not implemented. + """ + raise NotImplementedError("expanding is currently not implemented") + + def ewm(self, *args, **kwargs): + """ + Return an ewm grouper, providing ewm functionality per group. + + Currently not implemented. + """ + raise NotImplementedError("expanding is currently not implemented") + + def any(self, skipna: bool = True): + """ + Return True if any value in the group is truthful, else False. + + Currently not implemented. + """ + raise NotImplementedError("any is currently not implemented") + + def all(self, skipna: bool = True): + """ + Return True if all values in the group are truthful, else False. + + Currently not implemented. + """ + raise NotImplementedError("all is currently not implemented") + + +class DataFrameGroupBy(GroupBy, GetAttrGetItemMixin): + obj: "cudf.core.dataframe.DataFrame" + + _PROTECTED_KEYS = frozenset(("obj",)) + + def _reduce_numeric_only(self, op: str): + columns = list( + name + for name in self.obj._data.names + if ( + is_numeric_dtype(self.obj._data[name].dtype) + and name not in self.grouping.names + ) + ) + return self[columns].agg(op) + + def __getitem__(self, key): + return self.obj[key].groupby( + by=self.grouping.keys, + dropna=self._dropna, + sort=self._sort, + group_keys=self._group_keys, + as_index=self._as_index, + ) + def value_counts( self, subset=None, @@ -2637,68 +2847,112 @@ def value_counts( return result - def _mimic_pandas_order( - self, result: DataFrameOrSeries - ) -> DataFrameOrSeries: - """Given a groupby result from libcudf, reconstruct the row orders - matching that of pandas. This also adds appropriate indices. + @_performance_tracking + def corr( + self, method="pearson", min_periods=1, numeric_only: bool = False + ): """ - # TODO: copy metadata after this method is a common pattern, should - # merge in this method. + Compute pairwise correlation of columns, excluding NA/null values. - # This function is used to reorder the results of scan-based - # groupbys which have the same output size as input size. - # However, if the grouping key has NAs and dropna=True, the - # result coming back from libcudf has null_count few rows than - # the input, so we must produce an ordering from the full - # input range. - _, _, (ordering,) = self._groupby.groups( - [as_column(range(0, len(self.obj)))] - ) - if self._dropna and any( - c.has_nulls(include_nan=True) > 0 - for c in self.grouping._key_columns - ): - # Scan aggregations with null/nan keys put nulls in the - # corresponding output rows in pandas, to do that here - # expand the result by reindexing. - ri = cudf.RangeIndex(0, len(self.obj)) - result.index = cudf.Index(ordering) - # This reorders and expands - result = result.reindex(ri) - else: - # Just reorder according to the groupings - result = result.take(ordering.argsort()) - # Now produce the actual index we first thought of - result.index = self.obj.index - return result + Parameters + ---------- + method: {"pearson", "kendall", "spearman"} or callable, + default "pearson". Currently only the pearson correlation + coefficient is supported. + min_periods: int, optional + Minimum number of observations required per pair of columns + to have a valid result. -class DataFrameGroupBy(GroupBy, GetAttrGetItemMixin): - obj: "cudf.core.dataframe.DataFrame" + Returns + ------- + DataFrame + Correlation matrix. - _PROTECTED_KEYS = frozenset(("obj",)) + Examples + -------- + >>> import cudf + >>> gdf = cudf.DataFrame({ + ... "id": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + ... "val1": [5, 4, 6, 4, 8, 7, 4, 5, 2], + ... "val2": [4, 5, 6, 1, 2, 9, 8, 5, 1], + ... "val3": [4, 5, 6, 1, 2, 9, 8, 5, 1]}) + >>> gdf + id val1 val2 val3 + 0 a 5 4 4 + 1 a 4 5 5 + 2 a 6 6 6 + 3 b 4 1 1 + 4 b 8 2 2 + 5 b 7 9 9 + 6 c 4 8 8 + 7 c 5 5 5 + 8 c 2 1 1 + >>> gdf.groupby("id").corr(method="pearson") + val1 val2 val3 + id + a val1 1.000000 0.500000 0.500000 + val2 0.500000 1.000000 1.000000 + val3 0.500000 1.000000 1.000000 + b val1 1.000000 0.385727 0.385727 + val2 0.385727 1.000000 1.000000 + val3 0.385727 1.000000 1.000000 + c val1 1.000000 0.714575 0.714575 + val2 0.714575 1.000000 1.000000 + val3 0.714575 1.000000 1.000000 + """ - def _reduce_numeric_only(self, op: str): - columns = list( - name - for name in self.obj._data.names - if ( - is_numeric_dtype(self.obj._data[name].dtype) - and name not in self.grouping.names + if method != "pearson": + raise NotImplementedError( + "Only pearson correlation is currently supported" + ) + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is currently not supported." ) - ) - return self[columns].agg(op) - def __getitem__(self, key): - return self.obj[key].groupby( - by=self.grouping.keys, - dropna=self._dropna, - sort=self._sort, - group_keys=self._group_keys, - as_index=self._as_index, + return self._cov_or_corr( + lambda x: x.corr(method, min_periods), "Correlation" ) + def hist( + self, + column=None, + by=None, + grid: bool = True, + xlabelsize: int | None = None, + xrot: float | None = None, + ylabelsize: int | None = None, + yrot: float | None = None, + ax=None, + sharex: bool = False, + sharey: bool = False, + figsize: tuple[float, float] | None = None, + layout: tuple[int, int] | None = None, + bins: int | abc.Sequence[int] = 10, + backend: str | None = None, + legend: bool = False, + **kwargs, + ): + raise NotImplementedError("hist is not currently implemented") + + def boxplot( + self, + subplots: bool = True, + column=None, + fontsize: int | None = None, + rot: int = 0, + grid: bool = True, + ax=None, + figsize: tuple[float, float] | None = None, + layout=None, + sharex: bool = False, + sharey: bool = True, + backend=None, + **kwargs, + ): + raise NotImplementedError("boxplot is not currently implemented") + DataFrameGroupBy.__doc__ = groupby_doc_template.format(ret="") @@ -2706,8 +2960,10 @@ def __getitem__(self, key): class SeriesGroupBy(GroupBy): obj: "cudf.core.series.Series" - def agg(self, func): - result = super().agg(func) + def agg(self, func, *args, engine=None, engine_kwargs=None, **kwargs): + result = super().agg( + func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs + ) # downcast the result to a Series: if len(result._data): @@ -2722,14 +2978,95 @@ def agg(self, func): aggregate = agg - def apply(self, func, *args): - result = super().apply(func, *args) + def apply(self, func, *args, **kwargs): + result = super().apply(func, *args, **kwargs) # apply Series name to result result.name = self.obj.name return result + @property + def dtype(self) -> pd.Series: + raise NotImplementedError("dtype is currently not implemented.") + + def hist( + self, + by=None, + ax=None, + grid: bool = True, + xlabelsize: int | None = None, + xrot: float | None = None, + ylabelsize: int | None = None, + yrot: float | None = None, + figsize: tuple[float, float] | None = None, + bins: int | abc.Sequence[int] = 10, + backend: str | None = None, + legend: bool = False, + **kwargs, + ): + raise NotImplementedError("hist is currently not implemented.") + + @property + def is_monotonic_increasing(self) -> cudf.Series: + """ + Return whether each group's values are monotonically increasing. + + Currently not implemented + """ + raise NotImplementedError( + "is_monotonic_increasing is currently not implemented." + ) + + @property + def is_monotonic_decreasing(self) -> cudf.Series: + """ + Return whether each group's values are monotonically decreasing. + + Currently not implemented + """ + raise NotImplementedError( + "is_monotonic_decreasing is currently not implemented." + ) + + def nlargest( + self, n: int = 5, keep: Literal["first", "last", "all"] = "first" + ) -> cudf.Series: + """ + Return the largest n elements. + + Currently not implemented + """ + raise NotImplementedError("nlargest is currently not implemented.") + + def nsmallest( + self, n: int = 5, keep: Literal["first", "last", "all"] = "first" + ) -> cudf.Series: + """ + Return the smallest n elements. + + Currently not implemented + """ + raise NotImplementedError("nsmallest is currently not implemented.") + + def value_counts( + self, + normalize: bool = False, + sort: bool = True, + ascending: bool = False, + bins=None, + dropna: bool = True, + ) -> cudf.Series | cudf.DataFrame: + raise NotImplementedError("value_counts is currently not implemented.") + + def corr( + self, + other: cudf.Series, + method: str = "pearson", + min_periods: int | None = None, + ) -> cudf.Series: + raise NotImplementedError("corr is currently not implemented.") + SeriesGroupBy.__doc__ = groupby_doc_template.format(ret="") diff --git a/python/cudf/cudf/core/resample.py b/python/cudf/cudf/core/resample.py index 4e0c5bd86b9..715bbf89b15 100644 --- a/python/cudf/cudf/core/resample.py +++ b/python/cudf/cudf/core/resample.py @@ -43,8 +43,10 @@ def __init__(self, obj, by, axis=None, kind=None): by = _ResampleGrouping(obj, by) super().__init__(obj, by=by) - def agg(self, func): - result = super().agg(func) + def agg(self, func, *args, engine=None, engine_kwargs=None, **kwargs): + result = super().agg( + func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs + ) if len(self.grouping.bin_labels) != len(result): index = cudf.core.index.Index( self.grouping.bin_labels, name=self.grouping.names[0] diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 826a0e52f57..74f04c0584f 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -3885,3 +3885,28 @@ def test_group_by_raises_category_error(op): with pytest.raises(TypeError): df.groupby(df.a).agg(op) + + +def test_ngroups(): + pdf = pd.DataFrame({"a": [1, 1, 3], "b": range(3)}) + gdf = cudf.DataFrame.from_pandas(pdf) + + pgb = pdf.groupby("a") + ggb = gdf.groupby("a") + assert pgb.ngroups == ggb.ngroups + assert len(pgb) == len(ggb) + + +def test_ndim(): + pdf = pd.DataFrame({"a": [1, 1, 3], "b": range(3)}) + gdf = cudf.DataFrame.from_pandas(pdf) + + pgb = pdf.groupby("a") + ggb = gdf.groupby("a") + assert pgb.ndim == ggb.ndim + + pser = pd.Series(range(3)) + gser = cudf.Series.from_pandas(pser) + pgb = pser.groupby([0, 0, 1]) + ggb = gser.groupby(cudf.Series([0, 0, 1])) + assert pgb.ndim == ggb.ndim From 6e7624d6b31c93b0547590929ac63ed8e3a48d24 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:06:51 -0400 Subject: [PATCH 008/270] Add stream parameter to reshape APIs (#16410) Adds `stream` parameter to reshape APIs: - `cudf::interleave_columns` - `cudf::tile` - `cudf::byte_cast` Found while working #15983 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16410 --- cpp/include/cudf/detail/reshape.hpp | 4 --- cpp/include/cudf/reshape.hpp | 17 ++++++---- cpp/src/reshape/byte_cast.cu | 11 ++----- cpp/src/reshape/interleave_columns.cu | 3 +- cpp/src/reshape/tile.cu | 3 +- cpp/tests/CMakeLists.txt | 1 + cpp/tests/streams/reshape_test.cpp | 47 +++++++++++++++++++++++++++ 7 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 cpp/tests/streams/reshape_test.cpp diff --git a/cpp/include/cudf/detail/reshape.hpp b/cpp/include/cudf/detail/reshape.hpp index 30f8b88b116..68a856373bf 100644 --- a/cpp/include/cudf/detail/reshape.hpp +++ b/cpp/include/cudf/detail/reshape.hpp @@ -28,8 +28,6 @@ namespace CUDF_EXPORT cudf { namespace detail { /** * @copydoc cudf::tile - * - * @param stream CUDA stream used for device memory operations and kernel launches */ std::unique_ptr tile(table_view const& input, size_type count, @@ -38,8 +36,6 @@ std::unique_ptr
tile(table_view const& input, /** * @copydoc cudf::interleave_columns - * - * @param stream CUDA stream used for device memory operations and kernel launches */ std::unique_ptr interleave_columns(table_view const& input, rmm::cuda_stream_view, diff --git a/cpp/include/cudf/reshape.hpp b/cpp/include/cudf/reshape.hpp index a0a7fe694bb..07aaf6488ad 100644 --- a/cpp/include/cudf/reshape.hpp +++ b/cpp/include/cudf/reshape.hpp @@ -47,13 +47,14 @@ namespace CUDF_EXPORT cudf { * @throws cudf::logic_error if input contains no columns. * @throws cudf::logic_error if input columns dtypes are not identical. * - * @param[in] input Table containing columns to interleave - * @param[in] mr Device memory resource used to allocate the returned column's device memory - * + * @param input Table containing columns to interleave + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory * @return The interleaved columns as a single column */ std::unique_ptr interleave_columns( table_view const& input, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -68,15 +69,17 @@ std::unique_ptr interleave_columns( * return = [[8, 4, 7, 8, 4, 7], [5, 2, 3, 5, 2, 3]] * ``` * - * @param[in] input Table containing rows to be repeated - * @param[in] count Number of times to tile "rows". Must be non-negative - * @param[in] mr Device memory resource used to allocate the returned table's device memory + * @param input Table containing rows to be repeated + * @param count Number of times to tile "rows". Must be non-negative + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned table's device memory * * @return The table containing the tiled "rows" */ std::unique_ptr
tile( table_view const& input, size_type count, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -95,6 +98,7 @@ enum class flip_endianness : bool { NO, YES }; * * @param input_column Column to be converted to lists of bytes * @param endian_configuration Whether to retain or flip the endianness of the elements + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory * * @return The column containing the lists of bytes @@ -102,6 +106,7 @@ enum class flip_endianness : bool { NO, YES }; std::unique_ptr byte_cast( column_view const& input_column, flip_endianness endian_configuration, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @} */ // end of group diff --git a/cpp/src/reshape/byte_cast.cu b/cpp/src/reshape/byte_cast.cu index 3dfa0b65814..2a03a5504c1 100644 --- a/cpp/src/reshape/byte_cast.cu +++ b/cpp/src/reshape/byte_cast.cu @@ -167,11 +167,6 @@ struct byte_list_conversion_fn byte_cast(column_view const& input, flip_endianness endian_configuration, rmm::cuda_stream_view stream, @@ -183,15 +178,13 @@ std::unique_ptr byte_cast(column_view const& input, } // namespace detail -/** - * @copydoc cudf::byte_cast(column_view const&, flip_endianness, rmm::device_async_resource_ref) - */ std::unique_ptr byte_cast(column_view const& input, flip_endianness endian_configuration, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::byte_cast(input, endian_configuration, cudf::get_default_stream(), mr); + return detail::byte_cast(input, endian_configuration, stream, mr); } } // namespace cudf diff --git a/cpp/src/reshape/interleave_columns.cu b/cpp/src/reshape/interleave_columns.cu index 79124508b11..7473b6045af 100644 --- a/cpp/src/reshape/interleave_columns.cu +++ b/cpp/src/reshape/interleave_columns.cu @@ -264,10 +264,11 @@ std::unique_ptr interleave_columns(table_view const& input, } // namespace detail std::unique_ptr interleave_columns(table_view const& input, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::interleave_columns(input, cudf::get_default_stream(), mr); + return detail::interleave_columns(input, stream, mr); } } // namespace cudf diff --git a/cpp/src/reshape/tile.cu b/cpp/src/reshape/tile.cu index 29996aa2152..3d4fb73c000 100644 --- a/cpp/src/reshape/tile.cu +++ b/cpp/src/reshape/tile.cu @@ -64,10 +64,11 @@ std::unique_ptr
tile(table_view const& in, std::unique_ptr
tile(table_view const& in, size_type count, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::tile(in, count, cudf::get_default_stream(), mr); + return detail::tile(in, count, stream, mr); } } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 22827484f9a..4dffcb41ba2 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -704,6 +704,7 @@ ConfigureTest(STREAM_PARQUETIO_TEST streams/io/parquet_test.cpp STREAM_MODE test ConfigureTest(STREAM_POOL_TEST streams/pool_test.cu STREAM_MODE testing) ConfigureTest(STREAM_REDUCTION_TEST streams/reduction_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_REPLACE_TEST streams/replace_test.cpp STREAM_MODE testing) +ConfigureTest(STREAM_RESHAPE_TEST streams/reshape_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_ROLLING_TEST streams/rolling_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_SEARCH_TEST streams/search_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_SORTING_TEST streams/sorting_test.cpp STREAM_MODE testing) diff --git a/cpp/tests/streams/reshape_test.cpp b/cpp/tests/streams/reshape_test.cpp new file mode 100644 index 00000000000..d7c5da91bca --- /dev/null +++ b/cpp/tests/streams/reshape_test.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include + +class ReshapeTest : public cudf::test::BaseFixture {}; + +TEST_F(ReshapeTest, InterleaveColumns) +{ + auto a = cudf::test::fixed_width_column_wrapper({0, 3, 6}); + auto b = cudf::test::fixed_width_column_wrapper({1, 4, 7}); + auto c = cudf::test::fixed_width_column_wrapper({2, 5, 8}); + cudf::table_view in(std::vector{a, b, c}); + cudf::interleave_columns(in, cudf::test::get_default_stream()); +} + +TEST_F(ReshapeTest, Tile) +{ + auto a = cudf::test::fixed_width_column_wrapper({-1, 0, 1}); + cudf::table_view in(std::vector{a}); + cudf::tile(in, 2, cudf::test::get_default_stream()); +} + +TEST_F(ReshapeTest, ByteCast) +{ + auto a = cudf::test::fixed_width_column_wrapper({0, 100, -100, 1000, 1000}); + cudf::byte_cast(a, cudf::flip_endianness::YES, cudf::test::get_default_stream()); + cudf::byte_cast(a, cudf::flip_endianness::NO, cudf::test::get_default_stream()); +} From 35796057b64e258713d4d89ba368837d30a1a9c5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jul 2024 08:33:23 -1000 Subject: [PATCH 009/270] Align misc DataFrame and MultiIndex methods with pandas 2.x (#16402) The API changes in this PR are mostly adding implementations or adding missing keyword argument (although they might not be implemented). The APIs affected are: * `DataFrame.insert` * `DataFrame.melt` * `DataFrame.merge` * `DataFrame.quantile` * `DataFrame.cov` * `DataFrame.corr` * `DataFrame.median` * `DataFrame.rolling` * `DataFrame.resample` * `DataFrame.dropna` * `MultiIndex.from_tuple` * `MultiIndex.from_frame` * `MultiIndex.from_product` Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16402 --- python/cudf/cudf/core/dataframe.py | 106 +++++++++++++++++------- python/cudf/cudf/core/indexed_frame.py | 81 +++++++++++------- python/cudf/cudf/core/multiindex.py | 38 +++++++-- python/cudf/cudf/core/reshape.py | 3 + python/cudf/cudf/core/window/ewm.py | 23 +++-- python/cudf/cudf/core/window/rolling.py | 27 +++++- python/cudf/cudf/tests/test_dropna.py | 9 ++ 7 files changed, 211 insertions(+), 76 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 1d7136e61e3..6ea11fe9f64 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -3215,26 +3215,37 @@ def reset_index( ) @_performance_tracking - def insert(self, loc, name, value, nan_as_null=no_default): + def insert( + self, + loc, + column, + value, + allow_duplicates: bool = False, + nan_as_null=no_default, + ): """Add a column to DataFrame at the index specified by loc. Parameters ---------- loc : int location to insert by index, cannot be greater then num columns + 1 - name : number or string - name or label of column to be inserted + column : number or string + column or label of column to be inserted value : Series or array-like nan_as_null : bool, Default None If ``None``/``True``, converts ``np.nan`` values to ``null`` values. If ``False``, leaves ``np.nan`` values as is. """ + if allow_duplicates is not False: + raise NotImplementedError( + "allow_duplicates is currently not implemented." + ) if nan_as_null is no_default: nan_as_null = not cudf.get_option("mode.pandas_compatible") return self._insert( loc=loc, - name=name, + name=column, value=value, nan_as_null=nan_as_null, ignore_index=False, @@ -4097,7 +4108,15 @@ def transpose(self): T = property(transpose, doc=transpose.__doc__) @_performance_tracking - def melt(self, **kwargs): + def melt( + self, + id_vars=None, + value_vars=None, + var_name=None, + value_name="value", + col_level=None, + ignore_index: bool = True, + ): """Unpivots a DataFrame from wide format to long format, optionally leaving identifier variables set. @@ -4124,23 +4143,30 @@ def melt(self, **kwargs): """ from cudf.core.reshape import melt - return melt(self, **kwargs) + return melt( + self, + id_vars=id_vars, + value_vars=value_vars, + var_name=var_name, + value_name=value_name, + col_level=col_level, + ignore_index=ignore_index, + ) @_performance_tracking def merge( self, right, + how="inner", on=None, left_on=None, right_on=None, left_index=False, right_index=False, - how="inner", sort=False, - lsuffix=None, - rsuffix=None, - indicator=False, suffixes=("_x", "_y"), + indicator=False, + validate=None, ): """Merge GPU DataFrame objects by performing a database-style join operation by columns or indexes. @@ -4241,17 +4267,8 @@ def merge( raise NotImplementedError( "Only indicator=False is currently supported" ) - - if lsuffix or rsuffix: - raise ValueError( - "The lsuffix and rsuffix keywords have been replaced with the " - "``suffixes=`` keyword. " - "Please provide the following instead: \n\n" - " suffixes=('%s', '%s')" - % (lsuffix or "_x", rsuffix or "_y") - ) - else: - lsuffix, rsuffix = suffixes + if validate is not None: + raise NotImplementedError("validate is currently not supported.") lhs, rhs = self, right merge_cls = Merge @@ -5952,9 +5969,9 @@ def quantile( axis=0, numeric_only=True, interpolation=None, + method="single", columns=None, exact=True, - method="single", ): """ Return values at the given quantile. @@ -5980,14 +5997,14 @@ def quantile( * higher: `j`. * nearest: `i` or `j` whichever is nearest. * midpoint: (`i` + `j`) / 2. - columns : list of str - List of column names to include. - exact : boolean - Whether to use approximate or exact quantile algorithm. method : {'single', 'table'}, default `'single'` Whether to compute quantiles per-column ('single') or over all columns ('table'). When 'table', the only allowed interpolation methods are 'nearest', 'lower', and 'higher'. + columns : list of str + List of column names to include. + exact : boolean + Whether to use approximate or exact quantile algorithm. Returns ------- @@ -7309,25 +7326,47 @@ def unnamed_group_generator(): return result @_performance_tracking - def cov(self, **kwargs): + def cov(self, min_periods=None, ddof: int = 1, numeric_only: bool = False): """Compute the covariance matrix of a DataFrame. Parameters ---------- - **kwargs - Keyword arguments to be passed to cupy.cov + min_periods : int, optional + Minimum number of observations required per pair of columns to + have a valid result. + Currently not supported. + + ddof : int, default 1 + Delta degrees of freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + + numeric_only : bool, default False + Include only `float`, `int` or `boolean` data. + Currently not supported. Returns ------- cov : DataFrame """ - cov = cupy.cov(self.values, rowvar=False) + if min_periods is not None: + raise NotImplementedError( + "min_periods is currently not supported." + ) + + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is currently not supported." + ) + + cov = cupy.cov(self.values, ddof=ddof, rowvar=False) cols = self._data.to_pandas_index() df = DataFrame(cupy.asfortranarray(cov)).set_index(cols) df._set_columns_like(self._data) return df - def corr(self, method="pearson", min_periods=None): + def corr( + self, method="pearson", min_periods=None, numeric_only: bool = False + ): """Compute the correlation matrix of a DataFrame. Parameters @@ -7357,6 +7396,11 @@ def corr(self, method="pearson", min_periods=None): if min_periods is not None: raise NotImplementedError("Unsupported argument 'min_periods'") + if numeric_only is not False: + raise NotImplementedError( + "numeric_only is currently not supported." + ) + corr = cupy.corrcoef(values, rowvar=False) cols = self._data.to_pandas_index() df = DataFrame(cupy.asfortranarray(corr)).set_index(cols) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index e14f8923c25..0678ebfdd81 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1495,9 +1495,7 @@ def mean(self, axis=0, skipna=True, numeric_only=False, **kwargs): **kwargs, ) - def median( - self, axis=None, skipna=True, level=None, numeric_only=None, **kwargs - ): + def median(self, axis=None, skipna=True, numeric_only=None, **kwargs): """ Return the median of the values for the requested axis. @@ -1857,7 +1855,16 @@ def mask( @_performance_tracking @copy_docstring(Rolling) def rolling( - self, window, min_periods=None, center=False, axis=0, win_type=None + self, + window, + min_periods=None, + center: bool = False, + win_type: str | None = None, + on=None, + axis=0, + closed: str | None = None, + step: int | None = None, + method: str = "single", ): return Rolling( self, @@ -1865,7 +1872,11 @@ def rolling( min_periods=min_periods, center=center, axis=axis, + on=on, win_type=win_type, + closed=closed, + step=step, + method=method, ) @copy_docstring(ExponentialMovingWindow) @@ -1880,6 +1891,7 @@ def ewm( ignore_na: bool = False, axis: int = 0, times: str | np.ndarray | None = None, + method: Literal["single", "table"] = "single", ): return ExponentialMovingWindow( self, @@ -1892,6 +1904,7 @@ def ewm( ignore_na=ignore_na, axis=axis, times=times, + method=method, ) @_performance_tracking @@ -3943,16 +3956,15 @@ def resample( self, rule, axis=0, - closed=None, - label=None, - convention="start", + closed: Literal["right", "left"] | None = None, + label: Literal["right", "left"] | None = None, + convention: Literal["start", "end", "s", "e"] = "start", kind=None, - loffset=None, - base=None, on=None, level=None, origin="start_day", offset=None, + group_keys: bool = False, ): """ Convert the frequency of ("resample") the given time series data. @@ -4090,26 +4102,27 @@ def resample( "deprecated and will be removed in a future version. ", FutureWarning, ) - if (axis, convention, kind, loffset, base, origin, offset) != ( - 0, - "start", - None, - None, - None, - "start_day", - None, - ): - raise NotImplementedError( - "The following arguments are not " - "currently supported by resample:\n\n" - "- axis\n" - "- convention\n" - "- kind\n" - "- loffset\n" - "- base\n" - "- origin\n" - "- offset" + raise NotImplementedError("kind is currently not supported.") + if axis != 0: + warnings.warn( + "The 'axis' keyword in is " + "deprecated and will be removed in a future version. ", + FutureWarning, ) + raise NotImplementedError("axis is currently not supported.") + if convention != "start": + warnings.warn( + "The 'convention' keyword in is " + "deprecated and will be removed in a future version. ", + FutureWarning, + ) + raise NotImplementedError("convention is currently not supported.") + if origin != "start_day": + raise NotImplementedError("origin is currently not supported.") + if offset is not None: + raise NotImplementedError("offset is currently not supported.") + if group_keys is not False: + raise NotImplementedError("group_keys is currently not supported.") by = cudf.Grouper( key=on, freq=rule, closed=closed, label=label, level=level ) @@ -4120,7 +4133,13 @@ def resample( ) def dropna( - self, axis=0, how="any", thresh=None, subset=None, inplace=False + self, + axis=0, + how="any", + thresh=None, + subset=None, + inplace=False, + ignore_index: bool = False, ): """ Drop rows (or columns) containing nulls from a Column. @@ -4144,6 +4163,8 @@ def dropna( columns, subset is a list of rows to consider. inplace : bool, default False If True, do operation inplace and return None. + ignore_index : bool, default ``False`` + If ``True``, the resulting axis will be labeled 0, 1, …, n - 1. Returns ------- @@ -4220,6 +4241,8 @@ def dropna( """ if axis == 0: result = self._drop_na_rows(how=how, subset=subset, thresh=thresh) + if ignore_index: + result.index = RangeIndex(len(result)) else: result = self._drop_na_columns( how=how, subset=subset, thresh=thresh diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index dfc596bf279..0e1fddd7ed5 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -524,8 +524,10 @@ def codes(self): col.values for col in self._codes ) - def get_slice_bound(self, label, side, kind=None): - raise NotImplementedError() + def get_slice_bound(self, label, side): + raise NotImplementedError( + "get_slice_bound is not currently implemented." + ) @property # type: ignore @_performance_tracking @@ -1108,7 +1110,7 @@ def _concat(cls, objs): @classmethod @_performance_tracking - def from_tuples(cls, tuples, names=None): + def from_tuples(cls, tuples, sortorder: int | None = None, names=None): """ Convert list of tuples to MultiIndex. @@ -1116,6 +1118,9 @@ def from_tuples(cls, tuples, names=None): ---------- tuples : list / sequence of tuple-likes Each tuple is the index of one row/column. + sortorder : int or None + Level of sortedness (must be lexicographically sorted by that + level). names : list / sequence of str, optional Names for the levels in the index. @@ -1142,7 +1147,9 @@ def from_tuples(cls, tuples, names=None): names=['number', 'color']) """ # Use Pandas for handling Python host objects - pdi = pd.MultiIndex.from_tuples(tuples, names=names) + pdi = pd.MultiIndex.from_tuples( + tuples, sortorder=sortorder, names=names + ) return cls.from_pandas(pdi) @_performance_tracking @@ -1215,7 +1222,12 @@ def values(self): @classmethod @_performance_tracking - def from_frame(cls, df: pd.DataFrame | cudf.DataFrame, names=None): + def from_frame( + cls, + df: pd.DataFrame | cudf.DataFrame, + sortorder: int | None = None, + names=None, + ): """ Make a MultiIndex from a DataFrame. @@ -1223,6 +1235,9 @@ def from_frame(cls, df: pd.DataFrame | cudf.DataFrame, names=None): ---------- df : DataFrame DataFrame to be converted to MultiIndex. + sortorder : int, optional + Level of sortedness (must be lexicographically sorted by that + level). names : list-like, optional If no names are provided, use the column names, or tuple of column names if the columns is a MultiIndex. If a sequence, overwrite @@ -1273,11 +1288,13 @@ def from_frame(cls, df: pd.DataFrame | cudf.DataFrame, names=None): else: source_data = df names = names if names is not None else source_data._column_names - return cls.from_arrays(source_data._columns, names=names) + return cls.from_arrays( + source_data._columns, sortorder=sortorder, names=names + ) @classmethod @_performance_tracking - def from_product(cls, arrays, names=None): + def from_product(cls, iterables, sortorder: int | None = None, names=None): """ Make a MultiIndex from the cartesian product of multiple iterables. @@ -1285,6 +1302,9 @@ def from_product(cls, arrays, names=None): ---------- iterables : list / sequence of iterables Each iterable has unique labels for each level of the index. + sortorder : int or None + Level of sortedness (must be lexicographically sorted by that + level). names : list / sequence of str, optional Names for the levels in the index. If not explicitly provided, names will be inferred from the @@ -1314,7 +1334,9 @@ def from_product(cls, arrays, names=None): names=['number', 'color']) """ # Use Pandas for handling Python host objects - pdi = pd.MultiIndex.from_product(arrays, names=names) + pdi = pd.MultiIndex.from_product( + iterables, sortorder=sortorder, names=names + ) return cls.from_pandas(pdi) @classmethod diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index a542c5f5969..e7248977b1d 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -502,6 +502,7 @@ def melt( var_name=None, value_name="value", col_level=None, + ignore_index: bool = True, ): """Unpivots a DataFrame from wide format to long format, optionally leaving identifier variables set. @@ -566,6 +567,8 @@ def melt( """ if col_level is not None: raise NotImplementedError("col_level != None is not supported yet.") + if ignore_index is not True: + raise NotImplementedError("ignore_index is currently not supported.") # Arg cleaning diff --git a/python/cudf/cudf/core/window/ewm.py b/python/cudf/cudf/core/window/ewm.py index 1203a840076..ef0f6958aeb 100644 --- a/python/cudf/cudf/core/window/ewm.py +++ b/python/cudf/cudf/core/window/ewm.py @@ -1,7 +1,9 @@ # Copyright (c) 2022-2024, NVIDIA CORPORATION. - from __future__ import annotations +import warnings +from typing import Literal + import numpy as np from cudf._lib.reduce import scan @@ -103,13 +105,24 @@ def __init__( ignore_na: bool = False, axis: int = 0, times: str | np.ndarray | None = None, + method: Literal["single", "table"] = "single", ): - if (min_periods, ignore_na, axis, times) != (0, False, 0, None): + if min_periods != 0: raise NotImplementedError( - "The parameters `min_periods`, `ignore_na`, " - "`axis`, and `times` are not yet supported." + "min_periods is currently not supported." ) - + if ignore_na is not False: + raise NotImplementedError("ignore_na is currently not supported.") + if axis != 0: + warnings.warn( + "axis is deprecated with will be removed in a future version. " + "Transpose the DataFrame first instead." + ) + raise NotImplementedError("axis is currently not supported.") + if times is not None: + raise NotImplementedError("times is currently not supported.") + if method != "single": + raise NotImplementedError("method is currently not supported.") self.obj = obj self.adjust = adjust self.com = get_center_of_mass(com, span, halflife, alpha) diff --git a/python/cudf/cudf/core/window/rolling.py b/python/cudf/cudf/core/window/rolling.py index 29391c68471..043a41145e5 100644 --- a/python/cudf/cudf/core/window/rolling.py +++ b/python/cudf/cudf/core/window/rolling.py @@ -1,4 +1,7 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION +from __future__ import annotations + +import warnings import numba import pandas as pd @@ -196,17 +199,26 @@ def __init__( obj, window, min_periods=None, - center=False, + center: bool = False, + win_type: str | None = None, + on=None, axis=0, - win_type=None, + closed: str | None = None, + step: int | None = None, + method: str = "single", ): self.obj = obj self.window = window self.min_periods = min_periods self.center = center self._normalize() - self.agg_params = {} + # for var & std only? + self.agg_params: dict[str, int] = {} if axis != 0: + warnings.warn( + "axis is deprecated with will be removed in a future version. " + "Transpose the DataFrame first instead." + ) raise NotImplementedError("axis != 0 is not supported yet.") self.axis = axis @@ -217,6 +229,15 @@ def __init__( ) self.win_type = win_type + if on is not None: + raise NotImplementedError("on is currently not supported") + if closed not in (None, "right"): + raise NotImplementedError("closed is currently not supported") + if step is not None: + raise NotImplementedError("step is currently not supported") + if method != "single": + raise NotImplementedError("method is currently not supported") + def __getitem__(self, arg): if isinstance(arg, tuple): arg = list(arg) diff --git a/python/cudf/cudf/tests/test_dropna.py b/python/cudf/cudf/tests/test_dropna.py index ed0cf0053ea..5b1ee0ffac6 100644 --- a/python/cudf/cudf/tests/test_dropna.py +++ b/python/cudf/cudf/tests/test_dropna.py @@ -284,3 +284,12 @@ def test_dropna_multiindex_2(data, how): got = gi.dropna(how) assert_eq(expect, got) + + +def test_ignore_index(): + pser = pd.Series([1, 2, np.nan], index=[2, 4, 1]) + gser = cudf.from_pandas(pser) + + result = pser.dropna(ignore_index=True) + expected = gser.dropna(ignore_index=True) + assert_eq(result, expected) From 743e16426c564d0ed0d7e3d9be5f67e4605c4f32 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 29 Jul 2024 14:19:43 -0500 Subject: [PATCH 010/270] update some branch references in GitHub Actions configs (#16397) Fixes some lingering references to `branch-24.08` in the `pr_issue_status_automation` CI workflow. This was missed when new branches were cut because that file ends in `.yml` and `update-version.sh` was only modifying files ending in `.yaml`. The corresponding `update-version.sh` changes were made in #16183 and are already on 24.10 thanks to forward mergers. https://github.com/rapidsai/cudf/blob/dc05a01f3fc0742c5fbbddd86a0f2007bfdc2050/ci/release/update-version.sh#L78 ## Notes for Reviewers I checked like this, and don't see any other missed references: ```shell git grep -E '24\.8|24\.08|0\.39' ``` Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/cudf/pull/16397 --- .github/workflows/pr_issue_status_automation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_issue_status_automation.yml b/.github/workflows/pr_issue_status_automation.yml index 8ca971dc28d..45e5191eb54 100644 --- a/.github/workflows/pr_issue_status_automation.yml +++ b/.github/workflows/pr_issue_status_automation.yml @@ -23,7 +23,7 @@ on: jobs: get-project-id: - uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@branch-24.10 if: github.event.pull_request.state == 'open' secrets: inherit permissions: @@ -34,7 +34,7 @@ jobs: update-status: # This job sets the PR and its linked issues to "In Progress" status - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@branch-24.10 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: @@ -50,7 +50,7 @@ jobs: update-sprint: # This job sets the PR and its linked issues to the current "Weekly Sprint" - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@branch-24.10 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: From f8eb63e499f94d583d715f5c1f5e6f234589be57 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:39:19 -1000 Subject: [PATCH 011/270] Align Index APIs with pandas 2.x (#16361) Similar to https://github.com/rapidsai/cudf/pull/16310, the follow APIs have been modified to adjust/add parameters * `to_flat_index` * `isin` * `unique` * `transpose` Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16361 --- docs/cudf/source/conf.py | 5 ++++ python/cudf/cudf/core/_base_index.py | 25 ++++++++++++++++++-- python/cudf/cudf/core/index.py | 24 +++++++++++++++---- python/cudf/cudf/core/multiindex.py | 16 +++++++++++-- python/cudf/cudf/core/series.py | 8 ------- python/cudf/cudf/core/single_column_frame.py | 7 ++++++ python/cudf/cudf/tests/test_multiindex.py | 9 +++++++ 7 files changed, 78 insertions(+), 16 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index f544536fb31..7421d9be298 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -561,6 +561,11 @@ def on_missing_reference(app, env, node, contnode): ("py:class", "ScalarLike"), ("py:class", "ParentType"), ("py:class", "ColumnLike"), + ("py:class", "ColumnLike"), + ("py:obj", "cudf.Index.transpose"), + ("py:obj", "cudf.Index.T"), + ("py:obj", "cudf.Index.to_flat_index"), + ("py:obj", "cudf.MultiIndex.to_flat_index"), # TODO: Remove this when we figure out why typing_extensions doesn't seem # to map types correctly for intersphinx ("py:class", "typing_extensions.Self"), diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 8fad82c5c46..c91514202c5 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -868,6 +868,24 @@ def to_numpy(self): """Convert to a numpy array.""" raise NotImplementedError + def to_flat_index(self) -> Self: + """ + Identity method. + + This is implemented for compatibility with subclass implementations + when chaining. + + Returns + ------- + pd.Index + Caller. + + See Also + -------- + MultiIndex.to_flat_index : Subclass implementation. + """ + return self + def any(self): """ Return whether any elements is True in Index. @@ -945,7 +963,7 @@ def to_pandas(self, *, nullable: bool = False, arrow_type: bool = False): """ raise NotImplementedError - def isin(self, values): + def isin(self, values, level=None): """Return a boolean array where the index values are in values. Compute boolean array of whether each index value is found in @@ -956,6 +974,9 @@ def isin(self, values): ---------- values : set, list-like, Index Sought values. + level : str or int, optional + Name or position of the index level to use (if the index is a + `MultiIndex`). Returns ------- @@ -979,7 +1000,7 @@ def isin(self, values): # ColumnBase.isin). raise NotImplementedError - def unique(self): + def unique(self, level: int | None = None): """ Return unique values in the index. diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 1c48b8f4f2d..156cb973a9a 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -540,8 +540,12 @@ def memory_usage(self, deep: bool = False) -> int: ) return 0 - def unique(self) -> Self: + def unique(self, level: int | None = None) -> Self: # RangeIndex always has unique values + if level is not None and level > 0: + raise IndexError( + f"Too many levels: Index has only 1 level, not {level + 1}" + ) return self.copy() @_performance_tracking @@ -964,7 +968,11 @@ def _indices_of(self, value) -> cudf.core.column.NumericalColumn: i = [] return as_column(i, dtype=size_type_dtype) - def isin(self, values): + def isin(self, values, level=None): + if level is not None and level > 0: + raise IndexError( + f"Too many levels: Index has only 1 level, not {level + 1}" + ) if is_scalar(values): raise TypeError( "only list-like objects are allowed to be passed " @@ -1616,12 +1624,20 @@ def append(self, other): return self._concat(to_concat) - def unique(self): + def unique(self, level: int | None = None) -> Self: + if level is not None and level > 0: + raise IndexError( + f"Too many levels: Index has only 1 level, not {level + 1}" + ) return cudf.core.index._index_from_data( {self.name: self._values.unique()}, name=self.name ) - def isin(self, values): + def isin(self, values, level=None): + if level is not None and level > 0: + raise IndexError( + f"Too many levels: Index has only 1 level, not {level + 1}" + ) if is_scalar(values): raise TypeError( "only list-like objects are allowed to be passed " diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 0e1fddd7ed5..2788455aebf 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1156,6 +1156,15 @@ def from_tuples(cls, tuples, sortorder: int | None = None, names=None): def to_numpy(self): return self.values_host + def to_flat_index(self): + """ + Convert a MultiIndex to an Index of Tuples containing the level values. + + This is not currently implemented + """ + # TODO: Could implement as Index of ListDtype? + raise NotImplementedError("to_flat_index is not currently supported.") + @property # type: ignore @_performance_tracking def values_host(self): @@ -1734,8 +1743,11 @@ def fillna(self, value): return super().fillna(value=value) @_performance_tracking - def unique(self): - return self.drop_duplicates(keep="first") + def unique(self, level: int | None = None) -> Self | cudf.Index: + if level is None: + return self.drop_duplicates(keep="first") + else: + return self.get_level_values(level).unique() @_performance_tracking def nunique(self, dropna: bool = True) -> int: diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 8277ccf68fc..10ac1fdfc1e 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2775,14 +2775,6 @@ def cov(self, other, min_periods=None, ddof: int | None = None): f"{other.dtype}" ) - @_performance_tracking - def transpose(self): - """Return the transpose, which is by definition self.""" - - return self - - T = property(transpose, doc=transpose.__doc__) - @_performance_tracking def duplicated(self, keep="first"): """ diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index b93528f9693..a5ff1223791 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -389,3 +389,10 @@ def where(self, cond, other=None, inplace=False): result = cudf._lib.copying.copy_if_else(input_col, other, cond) return _make_categorical_like(result, self_column) + + @_performance_tracking + def transpose(self): + """Return the transpose, which is by definition self.""" + return self + + T = property(transpose, doc=transpose.__doc__) diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index 2c00d48266c..b7314a36e73 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -2170,3 +2170,12 @@ def test_bool_raises(): lfunc_args_and_kwargs=[[cudf.MultiIndex.from_arrays([range(1)])]], rfunc_args_and_kwargs=[[pd.MultiIndex.from_arrays([range(1)])]], ) + + +def test_unique_level(): + pd_mi = pd.MultiIndex.from_arrays([[1, 1, 2], [3, 3, 2]]) + cudf_mi = cudf.MultiIndex.from_pandas(pd_mi) + + result = pd_mi.unique(level=1) + expected = cudf_mi.unique(level=1) + assert_eq(result, expected) From 368a34ca9fd7db1b6cfb6e7817978e3e4fcfb00b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 29 Jul 2024 20:05:17 -0500 Subject: [PATCH 012/270] Use RMM adaptor constructors instead of factories. (#16414) This PR uses RMM memory resource adaptor constructors instead of factory functions. With CTAD, we do not need the factory and can use the constructor directly. The factory will be deprecated in https://github.com/rapidsai/rmm/pull/1626. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Nghia Truong (https://github.com/ttnghia) - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) URL: https://github.com/rapidsai/cudf/pull/16414 --- cpp/benchmarks/fixture/benchmark_fixture.hpp | 2 +- .../cudf_test/stream_checking_resource_adaptor.hpp | 12 ------------ cpp/include/cudf_test/testing_main.hpp | 2 +- java/src/main/native/src/RmmJni.cpp | 7 ------- 4 files changed, 2 insertions(+), 21 deletions(-) diff --git a/cpp/benchmarks/fixture/benchmark_fixture.hpp b/cpp/benchmarks/fixture/benchmark_fixture.hpp index 8c8d6756b00..8900899f9be 100644 --- a/cpp/benchmarks/fixture/benchmark_fixture.hpp +++ b/cpp/benchmarks/fixture/benchmark_fixture.hpp @@ -107,7 +107,7 @@ class memory_stats_logger { public: memory_stats_logger() : existing_mr(rmm::mr::get_current_device_resource()), - statistics_mr(rmm::mr::make_statistics_adaptor(existing_mr)) + statistics_mr(rmm::mr::statistics_resource_adaptor(existing_mr)) { rmm::mr::set_current_device_resource(&statistics_mr); } diff --git a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp index 4f3c723d195..417bbb3d9ab 100644 --- a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp +++ b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp @@ -156,16 +156,4 @@ class stream_checking_resource_adaptor final : public rmm::mr::device_memory_res // cudf::test::get_default_stream() is observed. }; -/** - * @brief Convenience factory to return a `stream_checking_resource_adaptor` around the - * upstream resource `upstream`. - * - * @param upstream Reference to the upstream resource - */ -inline stream_checking_resource_adaptor make_stream_checking_resource_adaptor( - rmm::device_async_resource_ref upstream, bool error_on_invalid_stream, bool check_default_stream) -{ - return stream_checking_resource_adaptor{upstream, error_on_invalid_stream, check_default_stream}; -} - } // namespace cudf::test diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index 9866253a9f8..ed83ddabb00 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -183,7 +183,7 @@ inline auto make_stream_mode_adaptor(cxxopts::ParseResult const& cmd_opts) auto const stream_error_mode = cmd_opts["stream_error_mode"].as(); auto const error_on_invalid_stream = (stream_error_mode == "error"); auto const check_default_stream = (stream_mode == "new_cudf_default"); - auto adaptor = cudf::test::make_stream_checking_resource_adaptor( + auto adaptor = cudf::test::stream_checking_resource_adaptor( resource, error_on_invalid_stream, check_default_stream); if ((stream_mode == "new_cudf_default") || (stream_mode == "new_testing_default")) { rmm::mr::set_current_device_resource(&adaptor); diff --git a/java/src/main/native/src/RmmJni.cpp b/java/src/main/native/src/RmmJni.cpp index 5842a980fc4..09c04a77590 100644 --- a/java/src/main/native/src/RmmJni.cpp +++ b/java/src/main/native/src/RmmJni.cpp @@ -154,13 +154,6 @@ class tracking_resource_adaptor final : public base_tracking_resource_adaptor { } }; -template -tracking_resource_adaptor* make_tracking_adaptor(Upstream* upstream, - std::size_t size_alignment) -{ - return new tracking_resource_adaptor{upstream, size_alignment}; -} - /** * @brief An RMM device memory resource adaptor that delegates to the wrapped resource * for most operations but will call Java to handle certain situations (e.g.: allocation failure). From d1be0b6dc06fddd0b69fb69731281b16894cb132 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:12:38 -1000 Subject: [PATCH 013/270] Align CategoricalIndex APIs with pandas 2.x (#16369) Mostly exposing methods that were available on the CategoricalColumn Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16369 --- python/cudf/cudf/core/column/categorical.py | 130 +++++++++++--------- python/cudf/cudf/core/index.py | 116 +++++++++++++++++ python/cudf/cudf/tests/test_categorical.py | 56 +++++++++ 3 files changed, 247 insertions(+), 55 deletions(-) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 9aaccca349d..9433a91b9c6 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -262,37 +262,10 @@ def add_categories(self, new_categories: Any) -> SeriesOrIndex | None: dtype: category Categories (2, int64): [1, 2] """ - old_categories = self._column.categories - new_categories = column.as_column( - new_categories, - dtype=old_categories.dtype if len(new_categories) == 0 else None, - ) - - if is_mixed_with_object_dtype(old_categories, new_categories): - raise TypeError( - f"cudf does not support adding categories with existing " - f"categories of dtype `{old_categories.dtype}` and new " - f"categories of dtype `{new_categories.dtype}`, please " - f"type-cast new_categories to the same type as " - f"existing categories." - ) - common_dtype = find_common_type( - [old_categories.dtype, new_categories.dtype] + return self._return_or_inplace( + self._column.add_categories(new_categories=new_categories) ) - new_categories = new_categories.astype(common_dtype) - old_categories = old_categories.astype(common_dtype) - - if old_categories.isin(new_categories).any(): - raise ValueError("new categories must not include old categories") - - new_categories = old_categories.append(new_categories) - out_col = self._column - if not out_col._categories_equal(new_categories): - out_col = out_col._set_categories(new_categories) - - return self._return_or_inplace(out_col) - def remove_categories( self, removals: Any, @@ -349,23 +322,9 @@ def remove_categories( dtype: category Categories (3, int64): [1, 2, 10] """ - - cats = self.categories.to_series() - removals = cudf.Series(removals, dtype=cats.dtype) - removals_mask = removals.isin(cats) - - # ensure all the removals are in the current categories - # list. If not, raise an error to match Pandas behavior - if not removals_mask.all(): - vals = removals[~removals_mask].to_numpy() - raise ValueError(f"removals must all be in old categories: {vals}") - - new_categories = cats[~cats.isin(removals)]._column - out_col = self._column - if not out_col._categories_equal(new_categories): - out_col = out_col._set_categories(new_categories) - - return self._return_or_inplace(out_col) + return self._return_or_inplace( + self._column.remove_categories(removals=removals) + ) def set_categories( self, @@ -1319,7 +1278,7 @@ def _set_categories( new_categories: Any, is_unique: bool = False, ordered: bool = False, - ) -> CategoricalColumn: + ) -> Self: """Returns a new CategoricalColumn with the categories set to the specified *new_categories*. @@ -1376,17 +1335,68 @@ def _set_categories( new_codes = df._data["new_codes"] # codes can't have masks, so take mask out before moving in - return column.build_categorical_column( - categories=new_cats, - codes=column.build_column( - new_codes.base_data, dtype=new_codes.dtype + return cast( + Self, + column.build_categorical_column( + categories=new_cats, + codes=column.build_column( + new_codes.base_data, dtype=new_codes.dtype + ), + mask=new_codes.base_mask, + size=new_codes.size, + offset=new_codes.offset, + ordered=ordered, ), - mask=new_codes.base_mask, - size=new_codes.size, - offset=new_codes.offset, - ordered=ordered, ) + def add_categories(self, new_categories: Any) -> Self: + old_categories = self.categories + new_categories = column.as_column( + new_categories, + dtype=old_categories.dtype if len(new_categories) == 0 else None, + ) + if is_mixed_with_object_dtype(old_categories, new_categories): + raise TypeError( + f"cudf does not support adding categories with existing " + f"categories of dtype `{old_categories.dtype}` and new " + f"categories of dtype `{new_categories.dtype}`, please " + f"type-cast new_categories to the same type as " + f"existing categories." + ) + common_dtype = find_common_type( + [old_categories.dtype, new_categories.dtype] + ) + + new_categories = new_categories.astype(common_dtype) + old_categories = old_categories.astype(common_dtype) + + if old_categories.isin(new_categories).any(): + raise ValueError("new categories must not include old categories") + + new_categories = old_categories.append(new_categories) + if not self._categories_equal(new_categories): + return self._set_categories(new_categories) + return self + + def remove_categories( + self, + removals: Any, + ) -> Self: + removals = column.as_column(removals).astype(self.categories.dtype) + removals_mask = removals.isin(self.categories) + + # ensure all the removals are in the current categories + # list. If not, raise an error to match Pandas behavior + if not removals_mask.all(): + raise ValueError("removals must all be in old categories") + + new_categories = self.categories.apply_boolean_mask( + self.categories.isin(removals).unary_operator("not") + ) + if not self._categories_equal(new_categories): + return self._set_categories(new_categories) + return self + def reorder_categories( self, new_categories: Any, @@ -1404,6 +1414,16 @@ def reorder_categories( ) return self._set_categories(new_categories, ordered=ordered) + def rename_categories(self, new_categories) -> CategoricalColumn: + raise NotImplementedError( + "rename_categories is currently not supported." + ) + + def remove_unused_categories(self) -> Self: + raise NotImplementedError( + "remove_unused_categories is currently not supported." + ) + def as_ordered(self, ordered: bool): if self.dtype.ordered == ordered: return self diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 156cb973a9a..8c3b091abec 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2721,6 +2721,10 @@ def __init__( data = data.as_ordered(ordered=False) super().__init__(data, name=name) + @property + def ordered(self) -> bool: + return self._column.ordered + @property # type: ignore @_performance_tracking def codes(self): @@ -2743,6 +2747,118 @@ def _is_boolean(self): def _is_categorical(self): return True + def add_categories(self, new_categories) -> Self: + """ + Add new categories. + + `new_categories` will be included at the last/highest place in the + categories and will be unused directly after this call. + """ + return type(self)._from_data( + {self.name: self._column.add_categories(new_categories)} + ) + + def as_ordered(self) -> Self: + """ + Set the Categorical to be ordered. + """ + return type(self)._from_data( + {self.name: self._column.as_ordered(ordered=True)} + ) + + def as_unordered(self) -> Self: + """ + Set the Categorical to be unordered. + """ + return type(self)._from_data( + {self.name: self._column.as_ordered(ordered=False)} + ) + + def remove_categories(self, removals) -> Self: + """ + Remove the specified categories. + + `removals` must be included in the old categories. + + Parameters + ---------- + removals : category or list of categories + The categories which should be removed. + """ + return type(self)._from_data( + {self.name: self._column.remove_categories(removals)} + ) + + def remove_unused_categories(self) -> Self: + """ + Remove categories which are not used. + + This method is currently not supported. + """ + return type(self)._from_data( + {self.name: self._column.remove_unused_categories()} + ) + + def rename_categories(self, new_categories) -> Self: + """ + Rename categories. + + This method is currently not supported. + """ + return type(self)._from_data( + {self.name: self._column.rename_categories(new_categories)} + ) + + def reorder_categories(self, new_categories, ordered=None) -> Self: + """ + Reorder categories as specified in new_categories. + + ``new_categories`` need to include all old categories and no new category + items. + + Parameters + ---------- + new_categories : Index-like + The categories in new order. + ordered : bool, optional + Whether or not the categorical is treated as a ordered categorical. + If not given, do not change the ordered information. + """ + return type(self)._from_data( + { + self.name: self._column.reorder_categories( + new_categories, ordered=ordered + ) + } + ) + + def set_categories( + self, new_categories, ordered=None, rename: bool = False + ) -> Self: + """ + Set the categories to the specified new_categories. + + Parameters + ---------- + new_categories : list-like + The categories in new order. + ordered : bool, default None + Whether or not the categorical is treated as + a ordered categorical. If not given, do + not change the ordered information. + rename : bool, default False + Whether or not the `new_categories` should be + considered as a rename of the old categories + or as reordered categories. + """ + return type(self)._from_data( + { + self.name: self._column.set_categories( + new_categories, ordered=ordered, rename=rename + ) + } + ) + @_performance_tracking def interval_range( diff --git a/python/cudf/cudf/tests/test_categorical.py b/python/cudf/cudf/tests/test_categorical.py index 9b6029582ce..ae58af8ebce 100644 --- a/python/cudf/cudf/tests/test_categorical.py +++ b/python/cudf/cudf/tests/test_categorical.py @@ -891,3 +891,59 @@ def test_categorical_maxima(op): result = getattr(ser.cat.as_ordered(), op)() result_pd = getattr(ser_pd.cat.as_ordered(), op)() assert_eq(result, result_pd) + + +@pytest.mark.parametrize("ordered", [True, False]) +def test_index_ordered(ordered): + pd_ci = pd.CategoricalIndex([1, 2, 3], ordered=ordered) + cudf_ci = cudf.from_pandas(pd_ci) + assert pd_ci.ordered == cudf_ci.ordered + + +@pytest.mark.parametrize("method", ["as_ordered", "as_unordered"]) +@pytest.mark.parametrize("ordered", [True, False]) +def test_index_as_ordered(method, ordered): + pd_ci = pd.CategoricalIndex([1, 2, 3], ordered=ordered) + cudf_ci = cudf.from_pandas(pd_ci) + + expected = getattr(pd_ci, method)() + result = getattr(cudf_ci, method)() + assert_eq(result, expected) + + +def test_index_add_categories(): + pd_ci = pd.CategoricalIndex([1, 2, 3]) + cudf_ci = cudf.from_pandas(pd_ci) + + expected = pd_ci.add_categories([4]) + result = cudf_ci.add_categories([4]) + assert_eq(result, expected) + + +def test_index_remove_categories(): + pd_ci = pd.CategoricalIndex([1, 2, 3], categories=[1, 2, 3, 4]) + cudf_ci = cudf.from_pandas(pd_ci) + + expected = pd_ci.remove_categories([4]) + result = cudf_ci.remove_categories([4]) + assert_eq(result, expected) + + +@pytest.mark.parametrize("ordered", [True, False]) +def test_index_reorder_categories(ordered): + pd_ci = pd.CategoricalIndex([1, 2, 3], categories=[1, 3, 2, 4]) + cudf_ci = cudf.from_pandas(pd_ci) + + expected = pd_ci.reorder_categories([1, 2, 3, 4], ordered=ordered) + result = cudf_ci.reorder_categories([1, 2, 3, 4], ordered=ordered) + assert_eq(result, expected) + + +@pytest.mark.parametrize("ordered", [True, False]) +def test_index_set_categories(ordered): + pd_ci = pd.CategoricalIndex([1, 2, 3]) + cudf_ci = cudf.from_pandas(pd_ci) + + expected = pd_ci.set_categories([1, 2, 3, 4], ordered=ordered) + result = cudf_ci.set_categories([1, 2, 3, 4], ordered=ordered) + assert_eq(result, expected) From 8def2ec1acac6a538002db011d977bb22cfbda82 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 30 Jul 2024 14:34:59 -0500 Subject: [PATCH 014/270] Add Java APIs to copy column data to host asynchronously (#16429) Adds Java methods to ColumnView to allow copying of column data to host memory asynchronously. This can be used to avoid many unnecessary stream synchronization when copying many columns to the host. Authors: - Jason Lowe (https://github.com/jlowe) Approvers: - Nghia Truong (https://github.com/ttnghia) - Robert (Bobby) Evans (https://github.com/revans2) URL: https://github.com/rapidsai/cudf/pull/16429 --- .../main/java/ai/rapids/cudf/ColumnView.java | 52 +++++++++++++------ .../java/ai/rapids/cudf/HostColumnVector.java | 4 ++ .../ai/rapids/cudf/HostColumnVectorCore.java | 4 +- .../ai/rapids/cudf/JCudfSerialization.java | 5 +- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 997ff77bae3..8ff2f0f0a73 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -5034,8 +5034,8 @@ private static NestedColumnVector createNestedColumnVector(DType type, long rows // DATA MOVEMENT ///////////////////////////////////////////////////////////////////////////// - private static HostColumnVectorCore copyToHostNestedHelper( - ColumnView deviceCvPointer, HostMemoryAllocator hostMemoryAllocator) { + private static HostColumnVectorCore copyToHostAsyncNestedHelper( + Cuda.Stream stream, ColumnView deviceCvPointer, HostMemoryAllocator hostMemoryAllocator) { if (deviceCvPointer == null) { return null; } @@ -5056,20 +5056,20 @@ private static HostColumnVectorCore copyToHostNestedHelper( currValidity = deviceCvPointer.getValid(); if (currData != null) { hostData = hostMemoryAllocator.allocate(currData.length); - hostData.copyFromDeviceBuffer(currData); + hostData.copyFromDeviceBufferAsync(currData, stream); } if (currValidity != null) { hostValid = hostMemoryAllocator.allocate(currValidity.length); - hostValid.copyFromDeviceBuffer(currValidity); + hostValid.copyFromDeviceBufferAsync(currValidity, stream); } if (currOffsets != null) { hostOffsets = hostMemoryAllocator.allocate(currOffsets.length); - hostOffsets.copyFromDeviceBuffer(currOffsets); + hostOffsets.copyFromDeviceBufferAsync(currOffsets, stream); } int numChildren = deviceCvPointer.getNumChildren(); for (int i = 0; i < numChildren; i++) { try(ColumnView childDevPtr = deviceCvPointer.getChildColumnView(i)) { - children.add(copyToHostNestedHelper(childDevPtr, hostMemoryAllocator)); + children.add(copyToHostAsyncNestedHelper(stream, childDevPtr, hostMemoryAllocator)); } } currNullCount = deviceCvPointer.getNullCount(); @@ -5103,11 +5103,20 @@ private static HostColumnVectorCore copyToHostNestedHelper( } } + /** Copy the data to the host synchronously. */ + public HostColumnVector copyToHost(HostMemoryAllocator hostMemoryAllocator) { + HostColumnVector result = copyToHostAsync(Cuda.DEFAULT_STREAM, hostMemoryAllocator); + Cuda.DEFAULT_STREAM.sync(); + return result; + } + /** - * Copy the data to the host. + * Copy the data to the host asynchronously. The caller MUST synchronize on the stream + * before examining the result. */ - public HostColumnVector copyToHost(HostMemoryAllocator hostMemoryAllocator) { - try (NvtxRange toHost = new NvtxRange("ensureOnHost", NvtxColor.BLUE)) { + public HostColumnVector copyToHostAsync(Cuda.Stream stream, + HostMemoryAllocator hostMemoryAllocator) { + try (NvtxRange toHost = new NvtxRange("toHostAsync", NvtxColor.BLUE)) { HostMemoryBuffer hostDataBuffer = null; HostMemoryBuffer hostValidityBuffer = null; HostMemoryBuffer hostOffsetsBuffer = null; @@ -5127,16 +5136,16 @@ public HostColumnVector copyToHost(HostMemoryAllocator hostMemoryAllocator) { if (!type.isNestedType()) { if (valid != null) { hostValidityBuffer = hostMemoryAllocator.allocate(valid.getLength()); - hostValidityBuffer.copyFromDeviceBuffer(valid); + hostValidityBuffer.copyFromDeviceBufferAsync(valid, stream); } if (offsets != null) { hostOffsetsBuffer = hostMemoryAllocator.allocate(offsets.length); - hostOffsetsBuffer.copyFromDeviceBuffer(offsets); + hostOffsetsBuffer.copyFromDeviceBufferAsync(offsets, stream); } // If a strings column is all null values there is no data buffer allocated if (data != null) { hostDataBuffer = hostMemoryAllocator.allocate(data.length); - hostDataBuffer.copyFromDeviceBuffer(data); + hostDataBuffer.copyFromDeviceBufferAsync(data, stream); } HostColumnVector ret = new HostColumnVector(type, rows, Optional.of(nullCount), hostDataBuffer, hostValidityBuffer, hostOffsetsBuffer); @@ -5145,21 +5154,21 @@ public HostColumnVector copyToHost(HostMemoryAllocator hostMemoryAllocator) { } else { if (data != null) { hostDataBuffer = hostMemoryAllocator.allocate(data.length); - hostDataBuffer.copyFromDeviceBuffer(data); + hostDataBuffer.copyFromDeviceBufferAsync(data, stream); } if (valid != null) { hostValidityBuffer = hostMemoryAllocator.allocate(valid.getLength()); - hostValidityBuffer.copyFromDeviceBuffer(valid); + hostValidityBuffer.copyFromDeviceBufferAsync(valid, stream); } if (offsets != null) { hostOffsetsBuffer = hostMemoryAllocator.allocate(offsets.getLength()); - hostOffsetsBuffer.copyFromDeviceBuffer(offsets); + hostOffsetsBuffer.copyFromDeviceBufferAsync(offsets, stream); } List children = new ArrayList<>(); for (int i = 0; i < getNumChildren(); i++) { try (ColumnView childDevPtr = getChildColumnView(i)) { - children.add(copyToHostNestedHelper(childDevPtr, hostMemoryAllocator)); + children.add(copyToHostAsyncNestedHelper(stream, childDevPtr, hostMemoryAllocator)); } } HostColumnVector ret = new HostColumnVector(type, rows, Optional.of(nullCount), @@ -5192,10 +5201,19 @@ public HostColumnVector copyToHost(HostMemoryAllocator hostMemoryAllocator) { } } + /** Copy the data to host memory synchronously */ public HostColumnVector copyToHost() { return copyToHost(DefaultHostMemoryAllocator.get()); } + /** + * Copy the data to the host asynchronously. The caller MUST synchronize on the stream + * before examining the result. + */ + public HostColumnVector copyToHostAsync(Cuda.Stream stream) { + return copyToHostAsync(stream, DefaultHostMemoryAllocator.get()); + } + /** * Calculate the total space required to copy the data to the host. This should be padded to * the alignment that the CPU requires. diff --git a/java/src/main/java/ai/rapids/cudf/HostColumnVector.java b/java/src/main/java/ai/rapids/cudf/HostColumnVector.java index 6b41d10fee3..61b11673957 100644 --- a/java/src/main/java/ai/rapids/cudf/HostColumnVector.java +++ b/java/src/main/java/ai/rapids/cudf/HostColumnVector.java @@ -92,6 +92,8 @@ public interface EventHandler { public HostColumnVector(DType type, long rows, Optional nullCount, HostMemoryBuffer hostDataBuffer, HostMemoryBuffer hostValidityBuffer, HostMemoryBuffer offsetBuffer, List nestedHcv) { + // NOTE: This constructor MUST NOT examine the contents of any host buffers, as they may be + // asynchronously written by the device. super(type, rows, nullCount, hostDataBuffer, hostValidityBuffer, offsetBuffer, nestedHcv); refCount = 0; incRefCountInternal(true); @@ -100,6 +102,8 @@ public HostColumnVector(DType type, long rows, Optional nullCount, HostColumnVector(DType type, long rows, Optional nullCount, HostMemoryBuffer hostDataBuffer, HostMemoryBuffer hostValidityBuffer, HostMemoryBuffer offsetBuffer) { + // NOTE: This constructor MUST NOT examine the contents of any host buffers, as they may be + // asynchronously written by the device. super(type, rows, nullCount, hostDataBuffer, hostValidityBuffer, offsetBuffer, new ArrayList<>()); assert !type.equals(DType.LIST) : "This constructor should not be used for list type"; if (nullCount.isPresent() && nullCount.get() > 0 && hostValidityBuffer == null) { diff --git a/java/src/main/java/ai/rapids/cudf/HostColumnVectorCore.java b/java/src/main/java/ai/rapids/cudf/HostColumnVectorCore.java index 95d209c0984..a225fbf34e1 100644 --- a/java/src/main/java/ai/rapids/cudf/HostColumnVectorCore.java +++ b/java/src/main/java/ai/rapids/cudf/HostColumnVectorCore.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,6 +47,8 @@ public class HostColumnVectorCore implements AutoCloseable { public HostColumnVectorCore(DType type, long rows, Optional nullCount, HostMemoryBuffer data, HostMemoryBuffer validity, HostMemoryBuffer offsets, List nestedChildren) { + // NOTE: This constructor MUST NOT examine the contents of any host buffers, as they may be + // asynchronously written by the device. this.offHeap = new OffHeapState(data, validity, offsets); MemoryCleaner.register(this, offHeap); this.type = type; diff --git a/java/src/main/java/ai/rapids/cudf/JCudfSerialization.java b/java/src/main/java/ai/rapids/cudf/JCudfSerialization.java index 666a8864003..89f363d2b29 100644 --- a/java/src/main/java/ai/rapids/cudf/JCudfSerialization.java +++ b/java/src/main/java/ai/rapids/cudf/JCudfSerialization.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -907,8 +907,9 @@ private static ColumnBufferProvider[] providersFrom(ColumnVector[] columns) { boolean success = false; try { for (int i = 0; i < columns.length; i++) { - onHost[i] = columns[i].copyToHost(); + onHost[i] = columns[i].copyToHostAsync(Cuda.DEFAULT_STREAM); } + Cuda.DEFAULT_STREAM.sync(); ColumnBufferProvider[] ret = providersFrom(onHost, true); success = true; return ret; From 79a1eed785fccbca2c20ff5cc844ec1a9e741ee5 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 31 Jul 2024 11:00:30 -0400 Subject: [PATCH 015/270] Remove checking for specific tests in memcheck script (#16412) Removes the checking for specific gtests in the `run_cudf_memcheck_ctests.sh` script. Each of those tests can check the `LIBCUDF_MEMCHECK_ENABLED` environment variable themselves. This simplifies the script logic and may help with replacing this with ctest logic in the future. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/16412 --- ci/run_cudf_memcheck_ctests.sh | 3 --- cpp/tests/error/error_handling_test.cu | 4 ++++ .../test_default_stream_identification.cu | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/run_cudf_memcheck_ctests.sh b/ci/run_cudf_memcheck_ctests.sh index aacd93e3b96..653829db419 100755 --- a/ci/run_cudf_memcheck_ctests.sh +++ b/ci/run_cudf_memcheck_ctests.sh @@ -15,9 +15,6 @@ export LIBCUDF_MEMCHECK_ENABLED=1 for gt in ./*_TEST ; do test_name=$(basename ${gt}) # Run gtests with compute-sanitizer - if [[ "$test_name" == "ERROR_TEST" ]] || [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then - continue - fi echo "Running compute-sanitizer on $test_name" compute-sanitizer --tool memcheck ${gt} "$@" done diff --git a/cpp/tests/error/error_handling_test.cu b/cpp/tests/error/error_handling_test.cu index 46d01ec14ff..1dfe45556c4 100644 --- a/cpp/tests/error/error_handling_test.cu +++ b/cpp/tests/error/error_handling_test.cu @@ -50,6 +50,8 @@ CUDF_KERNEL void test_kernel(int* data) { data[threadIdx.x] = threadIdx.x; } // calls. TEST(StreamCheck, FailedKernel) { + if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { GTEST_SKIP(); } + rmm::cuda_stream stream; int a; test_kernel<<<0, 0, 0, stream.value()>>>(&a); @@ -61,6 +63,8 @@ TEST(StreamCheck, FailedKernel) TEST(StreamCheck, CatchFailedKernel) { + if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { GTEST_SKIP(); } + rmm::cuda_stream stream; int a; test_kernel<<<0, 0, 0, stream.value()>>>(&a); diff --git a/cpp/tests/identify_stream_usage/test_default_stream_identification.cu b/cpp/tests/identify_stream_usage/test_default_stream_identification.cu index 268c7b37c81..c5fb75a7a8e 100644 --- a/cpp/tests/identify_stream_usage/test_default_stream_identification.cu +++ b/cpp/tests/identify_stream_usage/test_default_stream_identification.cu @@ -33,6 +33,7 @@ void test_cudaLaunchKernel() } catch (std::runtime_error&) { return; } + if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } throw std::runtime_error("No exception raised for kernel on default stream!"); } From 9336c172b1f61408e2392cbbd953e7f7e6e9ae3d Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 31 Jul 2024 16:27:26 +0100 Subject: [PATCH 016/270] Add upper bound pin for polars (#16442) This aligns the polars dependency with the most modern version supported by cudf-polars in this branch. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16442 --- dependencies.yaml | 2 +- python/cudf_polars/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 0fa32404156..aeb030313ed 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -630,7 +630,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - polars>=1.0 + - polars>=1.0,<1.3 run_dask_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index f8a1973bdbf..424c83a5199 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -20,7 +20,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cudf==24.10.*,>=0.0.0a0", - "polars>=1.0", + "polars>=1.0,<1.3", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", From 0f3b3808348debca8458bf73575745770b494ddc Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 31 Jul 2024 07:38:56 -1000 Subject: [PATCH 017/270] Add environment variable to log cudf.pandas fallback calls (#16161) Introduces a new environment variable `LOG_FAST_FALLBACK` which will create a structured log of the call that failed. An example of the log is ``` INFO:root:{"debug_type": "LOG_FAST_FALLBACK", "failed_call": "pandas._libs.interval.Interval(0,1)", "exception": "Exception", "exception_message": "Cannot transform _Unusable", "pandas_object": "pandas._libs.interval.Interval", "passed_args": "0,1,", "passed_kwargs": {}} ``` I could turn this into a warning instead, but I imagine we would want to first utilize this to parse the failures and see generalized failures in aggregate Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) - Matthew Murray (https://github.com/Matt711) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16161 --- python/cudf/cudf/pandas/_logger.py | 80 ++++++++++++++++++++++ python/cudf/cudf/pandas/fast_slow_proxy.py | 6 +- 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 python/cudf/cudf/pandas/_logger.py diff --git a/python/cudf/cudf/pandas/_logger.py b/python/cudf/cudf/pandas/_logger.py new file mode 100644 index 00000000000..68923c3e35c --- /dev/null +++ b/python/cudf/cudf/pandas/_logger.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import logging + +logging.basicConfig( + filename="cudf_pandas_unit_tests_debug.log", level=logging.INFO +) +logger = logging.getLogger() + + +class StructuredMessage: + # https://docs.python.org/3/howto/logging-cookbook.html#implementing-structured-logging + def __init__(self, debug_type: str, /, **kwargs) -> None: + self.debug_type = debug_type + self.kwargs = kwargs + + def __str__(self) -> str: + log = {"debug_type": self.debug_type} + return json.dumps({**log, **self.kwargs}) + + +def reprify(arg) -> str: + """Attempt to return arg's repr for logging.""" + try: + return repr(arg) + except Exception: + return "" + + +def log_fallback( + slow_args: tuple, slow_kwargs: dict, exception: Exception +) -> None: + """Log when a fast call falls back to the slow path.""" + caller = slow_args[0] + module = getattr(caller, "__module__", "") + obj_name = getattr(caller, "__qualname__", type(caller).__qualname__) + if module: + slow_object = f"{module}.{obj_name}" + else: + slow_object = obj_name + # TODO: Maybe use inspect.signature to map called args and kwargs + # to their keyword names, but a user calling an API incorrectly would + # break this. + caller_args = slow_args[1] + args_passed = ", ".join((reprify(arg) for arg in caller_args)) + args_types_passed = ", ".join((type(arg).__name__ for arg in caller_args)) + kwargs_passed = {} + kwargs_types_passed = "" + if len(slow_args) == 3: + caller_kwargs = slow_args[2] + if caller_kwargs: + fmt_kwargs = ", ".join( + f"{kwarg}={reprify(value)}" + for kwarg, value in caller_kwargs.items() + ) + kwargs_types_passed = ", ".join( + f"{kwarg}={type(value).__name__}" + for kwarg, value in caller_kwargs.items() + ) + args_passed = f"{args_passed}, {fmt_kwargs}" + kwargs_passed = { + kwarg: reprify(value) for kwarg, value in caller_kwargs.items() + } + message = StructuredMessage( + "LOG_FAST_FALLBACK", + failed_call=f"{slow_object}({args_passed})", + exception=type(exception).__name__, + exception_message=str(exception), + slow_object=slow_object, + args_passed=args_passed, + kwargs_passed=kwargs_passed, + args_types_passed=args_types_passed, + kwargs_types_passed=kwargs_types_passed, + ) + logger.info(message) diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index dfb729cae6b..bb678fd1efe 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -930,13 +930,17 @@ def _fast_slow_function_call( "Pandas debugging mode failed. " f"The exception was {e}." ) - except Exception: + except Exception as err: with nvtx.annotate( "EXECUTE_SLOW", color=_CUDF_PANDAS_NVTX_COLORS["EXECUTE_SLOW"], domain="cudf_pandas", ): slow_args, slow_kwargs = _slow_arg(args), _slow_arg(kwargs) + if _env_get_bool("LOG_FAST_FALLBACK", False): + from ._logger import log_fallback + + log_fallback(slow_args, slow_kwargs, err) with disable_module_accelerator(): result = func(*slow_args, **slow_kwargs) return _maybe_wrap_result(result, func, *args, **kwargs), fast From 5bcd8e062369a7d15222fa6d0bcc0b310553edbf Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:34:37 -1000 Subject: [PATCH 018/270] Align DatetimeIndex APIs with pandas 2.x (#16367) Mostly transferring methods that were defined on `Series.dt` methods to `DatetimeColumn` so it could be reused in `DatetimeIndex` Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Lawrence Mitchell (https://github.com/wence-) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16367 --- docs/cudf/source/conf.py | 2 + python/cudf/cudf/core/column/datetime.py | 56 ++++++ python/cudf/cudf/core/index.py | 211 ++++++++++++++++++++++- python/cudf/cudf/core/series.py | 43 ++--- python/cudf/cudf/tests/test_datetime.py | 107 ++++++++++++ 5 files changed, 385 insertions(+), 34 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 7421d9be298..7ebafc0da95 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -556,6 +556,8 @@ def on_missing_reference(app, env, node, contnode): ("py:class", "Dtype"), # The following are erroneously warned due to # https://github.com/sphinx-doc/sphinx/issues/11225 + ("py:obj", "cudf.DatetimeIndex.time"), + ("py:obj", "cudf.DatetimeIndex.date"), ("py:obj", "cudf.Index.values_host"), ("py:class", "pa.Array"), ("py:class", "ScalarLike"), diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 73902789c11..81fbb914842 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -286,6 +286,62 @@ def dayofyear(self) -> ColumnBase: def day_of_year(self) -> ColumnBase: return self.get_dt_field("day_of_year") + @property + def is_month_start(self) -> ColumnBase: + return (self.day == 1).fillna(False) + + @property + def is_month_end(self) -> ColumnBase: + last_day_col = libcudf.datetime.last_day_of_month(self) + return (self.day == last_day_col.day).fillna(False) + + @property + def is_quarter_end(self) -> ColumnBase: + last_month = self.month.isin([3, 6, 9, 12]) + return (self.is_month_end & last_month).fillna(False) + + @property + def is_quarter_start(self) -> ColumnBase: + first_month = self.month.isin([1, 4, 7, 10]) + return (self.is_month_start & first_month).fillna(False) + + @property + def is_year_end(self) -> ColumnBase: + day_of_year = self.day_of_year + leap_dates = libcudf.datetime.is_leap_year(self) + + leap = day_of_year == cudf.Scalar(366) + non_leap = day_of_year == cudf.Scalar(365) + return libcudf.copying.copy_if_else(leap, non_leap, leap_dates).fillna( + False + ) + + @property + def is_year_start(self) -> ColumnBase: + return (self.day_of_year == 1).fillna(False) + + @property + def days_in_month(self) -> ColumnBase: + return libcudf.datetime.days_in_month(self) + + @property + def day_of_week(self) -> ColumnBase: + raise NotImplementedError("day_of_week is currently not implemented.") + + @property + def is_normalized(self) -> bool: + raise NotImplementedError( + "is_normalized is currently not implemented." + ) + + def to_julian_date(self) -> ColumnBase: + raise NotImplementedError( + "to_julian_date is currently not implemented." + ) + + def normalize(self) -> ColumnBase: + raise NotImplementedError("normalize is currently not implemented.") + @property def values(self): """ diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 8c3b091abec..40a5d9ff259 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -61,6 +61,7 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable + from datetime import tzinfo def ensure_index(index_like: Any) -> BaseIndex: @@ -1680,7 +1681,7 @@ class DatetimeIndex(Index): copy : bool Make a copy of input. freq : str, optional - This is not yet supported + Frequency of the DatetimeIndex tz : pytz.timezone or dateutil.tz.tzfile This is not yet supported ambiguous : 'infer', bool-ndarray, 'NaT', default 'raise' @@ -1847,6 +1848,210 @@ def searchsorted( value, side=side, ascending=ascending, na_position=na_position ) + def as_unit(self, unit: str, round_ok: bool = True) -> Self: + """ + Convert to a dtype with the given unit resolution. + + Currently not implemented. + + Parameters + ---------- + unit : {'s', 'ms', 'us', 'ns'} + round_ok : bool, default True + If False and the conversion requires rounding, raise ValueError. + """ + raise NotImplementedError("as_unit is currently not implemented") + + def mean(self, *, skipna: bool = True, axis: int | None = 0): + return self._column.mean(skipna=skipna) + + def std(self, *, skipna: bool = True, axis: int | None = 0, ddof: int = 1): + return self._column.std(skipna=skipna, ddof=ddof) + + def strftime(self, date_format: str) -> Index: + """ + Convert to Index using specified date_format. + + Return an Index of formatted strings specified by date_format, which + supports the same string format as the python standard library. + + Parameters + ---------- + date_format : str + Date format string (e.g. "%Y-%m-%d"). + """ + return Index._from_data( + {self.name: self._column.strftime(date_format)} + ) + + @property + def asi8(self) -> cupy.ndarray: + return self._column.astype("int64").values + + @property + def inferred_freq(self) -> cudf.DateOffset | None: + raise NotImplementedError("inferred_freq is currently not implemented") + + @property + def freq(self) -> cudf.DateOffset | None: + return self._freq + + @freq.setter + def freq(self) -> None: + raise NotImplementedError("Setting freq is currently not supported.") + + @property + def freqstr(self) -> str: + raise NotImplementedError("freqstr is currently not implemented") + + @property + def resolution(self) -> str: + """ + Returns day, hour, minute, second, millisecond or microsecond + """ + raise NotImplementedError("resolution is currently not implemented") + + @property + def unit(self) -> str: + return self._column.time_unit + + @property + def tz(self) -> tzinfo | None: + """ + Return the timezone. + + Returns + ------- + datetime.tzinfo or None + Returns None when the array is tz-naive. + """ + return getattr(self.dtype, "tz", None) + + @property + def tzinfo(self) -> tzinfo | None: + """ + Alias for tz attribute + """ + return self.tz + + def to_pydatetime(self) -> np.ndarray: + """ + Return an ndarray of ``datetime.datetime`` objects. + + Returns + ------- + numpy.ndarray + An ndarray of ``datetime.datetime`` objects. + """ + return self.to_pandas().to_pydatetime() + + def to_julian_date(self) -> Index: + return Index._from_data({self.name: self._column.to_julian_date()}) + + def to_period(self, freq) -> pd.PeriodIndex: + return self.to_pandas().to_period(freq=freq) + + def normalize(self) -> Self: + """ + Convert times to midnight. + + Currently not implemented. + """ + return type(self)._from_data({self.name: self._column.normalize()}) + + @property + def time(self) -> np.ndarray: + """ + Returns numpy array of ``datetime.time`` objects. + + The time part of the Timestamps. + """ + return self.to_pandas().time + + @property + def timetz(self) -> np.ndarray: + """ + Returns numpy array of ``datetime.time`` objects with timezones. + + The time part of the Timestamps. + """ + return self.to_pandas().timetz + + @property + def date(self) -> np.ndarray: + """ + Returns numpy array of python ``datetime.date`` objects. + + Namely, the date part of Timestamps without time and + timezone information. + """ + return self.to_pandas().date + + @property + def is_month_start(self) -> cupy.ndarray: + """ + Booleans indicating if dates are the first day of the month. + """ + return self._column.is_month_start.values + + @property + def is_month_end(self) -> cupy.ndarray: + """ + Booleans indicating if dates are the last day of the month. + """ + return self._column.is_month_end.values + + @property + def is_quarter_end(self) -> cupy.ndarray: + """ + Booleans indicating if dates are the last day of the quarter. + """ + return self._column.is_quarter_end.values + + @property + def is_quarter_start(self) -> cupy.ndarray: + """ + Booleans indicating if dates are the start day of the quarter. + """ + return self._column.is_quarter_start.values + + @property + def is_year_end(self) -> cupy.ndarray: + """ + Booleans indicating if dates are the last day of the year. + """ + return self._column.is_year_end.values + + @property + def is_year_start(self) -> cupy.ndarray: + """ + Booleans indicating if dates are the first day of the year. + """ + return self._column.is_year_start.values + + @property + def is_normalized(self) -> bool: + """ + Returns True if all of the dates are at midnight ("no time") + """ + return self._column.is_normalized + + @property + def days_in_month(self) -> Index: + """ + Get the total number of days in the month that the date falls on. + """ + return Index._from_data({self.name: self._column.days_in_month}) + + daysinmonth = days_in_month + + @property + def day_of_week(self) -> Index: + """ + Get the day of week that the date falls on. + """ + return Index._from_data({self.name: self._column.day_of_week}) + @property # type: ignore @_performance_tracking def year(self): @@ -3391,9 +3596,11 @@ def _get_nearest_indexer( return indexer -def _validate_freq(freq: Any) -> cudf.DateOffset: +def _validate_freq(freq: Any) -> cudf.DateOffset | None: if isinstance(freq, str): return cudf.DateOffset._from_freqstr(freq) + elif freq is None: + return freq elif freq is not None and not isinstance(freq, cudf.DateOffset): raise ValueError(f"Invalid frequency: {freq}") return cast(cudf.DateOffset, freq) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 10ac1fdfc1e..929af5cd981 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -4415,7 +4415,9 @@ def is_month_start(self) -> Series: """ Booleans indicating if dates are the first day of the month. """ - return (self.day == 1).fillna(False) + return self._return_result_like_self( + self.series._column.is_month_start + ) @property # type: ignore @_performance_tracking @@ -4462,9 +4464,7 @@ def days_in_month(self) -> Series: 11 31 dtype: int16 """ - return self._return_result_like_self( - libcudf.datetime.days_in_month(self.series._column) - ) + return self._return_result_like_self(self.series._column.days_in_month) @property # type: ignore @_performance_tracking @@ -4505,9 +4505,7 @@ def is_month_end(self) -> Series: 8 False dtype: bool """ # noqa: E501 - last_day_col = libcudf.datetime.last_day_of_month(self.series._column) - last_day = self._return_result_like_self(last_day_col) - return (self.day == last_day.dt.day).fillna(False) + return self._return_result_like_self(self.series._column.is_month_end) @property # type: ignore @_performance_tracking @@ -4546,14 +4544,10 @@ def is_quarter_start(self) -> Series: 7 False dtype: bool """ - day = self.series._column.get_dt_field("day") - first_month = self.series._column.get_dt_field("month").isin( - [1, 4, 7, 10] + return self._return_result_like_self( + self.series._column.is_quarter_start ) - result = ((day == cudf.Scalar(1)) & first_month).fillna(False) - return self._return_result_like_self(result) - @property # type: ignore @_performance_tracking def is_quarter_end(self) -> Series: @@ -4591,16 +4585,10 @@ def is_quarter_end(self) -> Series: 7 False dtype: bool """ - day = self.series._column.get_dt_field("day") - last_day = libcudf.datetime.last_day_of_month(self.series._column) - last_day = last_day.get_dt_field("day") - last_month = self.series._column.get_dt_field("month").isin( - [3, 6, 9, 12] + return self._return_result_like_self( + self.series._column.is_quarter_end ) - result = ((day == last_day) & last_month).fillna(False) - return self._return_result_like_self(result) - @property # type: ignore @_performance_tracking def is_year_start(self) -> Series: @@ -4627,10 +4615,7 @@ def is_year_start(self) -> Series: 2 True dtype: bool """ - outcol = self.series._column.get_dt_field( - "day_of_year" - ) == cudf.Scalar(1) - return self._return_result_like_self(outcol.fillna(False)) + return self._return_result_like_self(self.series._column.is_year_start) @property # type: ignore @_performance_tracking @@ -4658,13 +4643,7 @@ def is_year_end(self) -> Series: 2 False dtype: bool """ - day_of_year = self.series._column.get_dt_field("day_of_year") - leap_dates = libcudf.datetime.is_leap_year(self.series._column) - - leap = day_of_year == cudf.Scalar(366) - non_leap = day_of_year == cudf.Scalar(365) - result = cudf._lib.copying.copy_if_else(leap, non_leap, leap_dates) - return self._return_result_like_self(result.fillna(False)) + return self._return_result_like_self(self.series._column.is_year_end) @_performance_tracking def _get_dt_field(self, field: str) -> Series: diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 7ab9ff2ef23..6bc775d2a2c 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -7,6 +7,7 @@ import cupy as cp import numpy as np import pandas as pd +import pandas._testing as tm import pyarrow as pa import pytest @@ -2429,3 +2430,109 @@ def test_day_month_name_locale_not_implemented(meth, klass): obj = obj.dt with pytest.raises(NotImplementedError): getattr(obj, meth)(locale="pt_BR.utf8") + + +@pytest.mark.parametrize( + "attr", + [ + "is_month_start", + "is_month_end", + "is_quarter_end", + "is_quarter_start", + "is_year_end", + "is_year_start", + "days_in_month", + "timetz", + "time", + "date", + ], +) +def test_dti_datetime_attributes(attr): + data = [ + "2020-01-01", + "2020-01-31", + "2020-03-01", + "2020-03-31", + "2020-03-31", + "2020-12-31", + None, + ] + pd_dti = pd.DatetimeIndex(data, name="foo") + cudf_dti = cudf.from_pandas(pd_dti) + + result = getattr(cudf_dti, attr) + expected = getattr(pd_dti, attr) + if isinstance(result, np.ndarray): + # numpy doesn't assert object arrays with NaT correctly + tm.assert_numpy_array_equal(result, expected) + else: + assert_eq(result, expected) + + +@pytest.mark.parametrize("attr", ["freq", "unit"]) +def test_dti_properties(attr): + pd_dti = pd.DatetimeIndex( + ["2020-01-01", "2020-01-02"], dtype="datetime64[ns]" + ) + cudf_dti = cudf.DatetimeIndex( + ["2020-01-01", "2020-01-02"], dtype="datetime64[ns]" + ) + + result = getattr(cudf_dti, attr) + expected = getattr(pd_dti, attr) + assert result == expected + + +def test_dti_asi8(): + pd_dti = pd.DatetimeIndex(["2020-01-01", "2020-12-31"], name="foo") + cudf_dti = cudf.from_pandas(pd_dti) + + result = pd_dti.asi8 + expected = cudf_dti.asi8 + assert_eq(result, expected) + + +@pytest.mark.parametrize( + "method, kwargs", + [ + ["mean", {}], + pytest.param( + "std", + {}, + marks=pytest.mark.xfail( + reason="https://github.com/rapidsai/cudf/issues/16444" + ), + ), + pytest.param( + "std", + {"ddof": 0}, + marks=pytest.mark.xfail( + reason="https://github.com/rapidsai/cudf/issues/16444" + ), + ), + ], +) +def test_dti_reduction(method, kwargs): + pd_dti = pd.DatetimeIndex(["2020-01-01", "2020-12-31"], name="foo") + cudf_dti = cudf.from_pandas(pd_dti) + + result = getattr(cudf_dti, method)(**kwargs) + expected = getattr(pd_dti, method)(**kwargs) + assert result == expected + + +@pytest.mark.parametrize( + "method, kwargs", + [ + ["to_pydatetime", {}], + ["to_period", {"freq": "D"}], + ["strftime", {"date_format": "%Y-%m-%d"}], + ], +) +def test_dti_methods(method, kwargs): + pd_dti = pd.DatetimeIndex(["2020-01-01", "2020-12-31"], name="foo") + cudf_dti = cudf.from_pandas(pd_dti) + + result = getattr(cudf_dti, method)(**kwargs) + expected = getattr(pd_dti, method)(**kwargs) + assert_eq(result, expected) From e2d45d6f24adbeb3a21081e078a6c2776d550a06 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:36:08 -1000 Subject: [PATCH 019/270] Align TimedeltaIndex APIs with pandas 2.x (#16368) Mostly exposing methods that were available on the `TimedeltaColumn` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16368 --- python/cudf/cudf/core/column/timedelta.py | 12 +++ python/cudf/cudf/core/index.py | 92 +++++++++++++++++++++++ python/cudf/cudf/tests/test_timedelta.py | 39 ++++++++++ 3 files changed, 143 insertions(+) diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index 59ea1cc002c..47c8ed6fd95 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -251,6 +251,18 @@ def normalize_binop_value(self, other) -> ColumnBinaryOperand: def time_unit(self) -> str: return np.datetime_data(self.dtype)[0] + def total_seconds(self) -> ColumnBase: + raise NotImplementedError("total_seconds is currently not implemented") + + def ceil(self, freq: str) -> ColumnBase: + raise NotImplementedError("ceil is currently not implemented") + + def floor(self, freq: str) -> ColumnBase: + raise NotImplementedError("floor is currently not implemented") + + def round(self, freq: str) -> ColumnBase: + raise NotImplementedError("round is currently not implemented") + def as_numerical_column( self, dtype: Dtype ) -> "cudf.core.column.NumericalColumn": diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 40a5d9ff259..888ea25cdae 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2759,6 +2759,98 @@ def __getitem__(self, index): return pd.Timedelta(value) return value + def as_unit(self, unit: str, round_ok: bool = True) -> Self: + """ + Convert to a dtype with the given unit resolution. + + Currently not implemented. + + Parameters + ---------- + unit : {'s', 'ms', 'us', 'ns'} + round_ok : bool, default True + If False and the conversion requires rounding, raise ValueError. + """ + raise NotImplementedError("as_unit is currently not implemented") + + @property + def freq(self) -> cudf.DateOffset | None: + raise NotImplementedError("freq is currently not implemented") + + @property + def freqstr(self) -> str: + raise NotImplementedError("freqstr is currently not implemented") + + @property + def resolution(self) -> str: + """ + Returns day, hour, minute, second, millisecond or microsecond + """ + raise NotImplementedError("resolution is currently not implemented") + + @property + def unit(self) -> str: + return self._column.time_unit + + def to_pytimedelta(self) -> np.ndarray: + """ + Return an ndarray of ``datetime.timedelta`` objects. + + Returns + ------- + numpy.ndarray + An ndarray of ``datetime.timedelta`` objects. + """ + return self.to_pandas().to_pytimedelta() + + @property + def asi8(self) -> cupy.ndarray: + return self._column.astype("int64").values + + def sum(self, *, skipna: bool = True, axis: int | None = 0): + return self._column.sum(skipna=skipna) + + def mean(self, *, skipna: bool = True, axis: int | None = 0): + return self._column.mean(skipna=skipna) + + def median(self, *, skipna: bool = True, axis: int | None = 0): + return self._column.median(skipna=skipna) + + def std(self, *, skipna: bool = True, axis: int | None = 0, ddof: int = 1): + return self._column.std(skipna=skipna, ddof=ddof) + + def total_seconds(self) -> cupy.ndarray: + """ + Return total duration of each element expressed in seconds. + + This method is currently not implemented. + """ + return self._column.total_seconds().values + + def ceil(self, freq: str) -> Self: + """ + Ceil to the specified resolution. + + This method is currently not implemented. + """ + return type(self)._from_data({self.name: self._column.ceil(freq)}) + + def floor(self, freq: str) -> Self: + """ + Floor to the specified resolution. + + This method is currently not implemented. + """ + return type(self)._from_data({self.name: self._column.floor(freq)}) + + def round(self, freq: str) -> Self: + """ + Round to the specified resolution. + + This method is currently not implemented. + """ + return type(self)._from_data({self.name: self._column.round(freq)}) + @property # type: ignore @_performance_tracking def days(self): diff --git a/python/cudf/cudf/tests/test_timedelta.py b/python/cudf/cudf/tests/test_timedelta.py index c4a2349f535..d622ff6b94e 100644 --- a/python/cudf/cudf/tests/test_timedelta.py +++ b/python/cudf/cudf/tests/test_timedelta.py @@ -1467,3 +1467,42 @@ def test_timedelta_series_cmpops_pandas_compatibility(data1, data2, op): got = op(gsr1, gsr2) assert_eq(expect, got) + + +@pytest.mark.parametrize( + "method, kwargs", + [ + ["sum", {}], + ["mean", {}], + ["median", {}], + ["std", {}], + ["std", {"ddof": 0}], + ], +) +def test_tdi_reductions(method, kwargs): + pd_tdi = pd.TimedeltaIndex(["1 day", "2 days", "3 days"]) + cudf_tdi = cudf.from_pandas(pd_tdi) + + result = getattr(pd_tdi, method)(**kwargs) + expected = getattr(cudf_tdi, method)(**kwargs) + assert result == expected + + +def test_tdi_asi8(): + pd_tdi = pd.TimedeltaIndex(["1 day", "2 days", "3 days"]) + cudf_tdi = cudf.from_pandas(pd_tdi) + + result = pd_tdi.asi8 + expected = cudf_tdi.asi8 + assert_eq(result, expected) + + +def test_tdi_unit(): + pd_tdi = pd.TimedeltaIndex( + ["1 day", "2 days", "3 days"], dtype="timedelta64[ns]" + ) + cudf_tdi = cudf.from_pandas(pd_tdi) + + result = pd_tdi.unit + expected = cudf_tdi.unit + assert result == expected From dab8660df7ba823dcef8cb8276a3867c2bb27cc7 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:37:48 -1000 Subject: [PATCH 020/270] Align IntervalIndex APIs with pandas 2.x (#16371) Implemented the relatively straightforward, missing APIs and raised `NotImplementedError` for the others Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16371 --- docs/cudf/source/conf.py | 15 ++- python/cudf/cudf/core/column/interval.py | 64 ++++++++- python/cudf/cudf/core/index.py | 123 ++++++++++++++++++ .../cudf/cudf/tests/indexes/test_interval.py | 33 +++++ 4 files changed, 229 insertions(+), 6 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 7ebafc0da95..43e2d6031bc 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -559,15 +559,20 @@ def on_missing_reference(app, env, node, contnode): ("py:obj", "cudf.DatetimeIndex.time"), ("py:obj", "cudf.DatetimeIndex.date"), ("py:obj", "cudf.Index.values_host"), - ("py:class", "pa.Array"), - ("py:class", "ScalarLike"), - ("py:class", "ParentType"), - ("py:class", "ColumnLike"), - ("py:class", "ColumnLike"), ("py:obj", "cudf.Index.transpose"), ("py:obj", "cudf.Index.T"), ("py:obj", "cudf.Index.to_flat_index"), ("py:obj", "cudf.MultiIndex.to_flat_index"), + ("py:meth", "pyarrow.Table.to_pandas"), + ("py:class", "pa.Array"), + ("py:class", "ScalarLike"), + ("py:class", "ParentType"), + ("py:class", "pyarrow.lib.DataType"), + ("py:class", "pyarrow.lib.Table"), + ("py:class", "pyarrow.lib.Scalar"), + ("py:class", "pyarrow.lib.ChunkedArray"), + ("py:class", "pyarrow.lib.Array"), + ("py:class", "ColumnLike"), # TODO: Remove this when we figure out why typing_extensions doesn't seem # to map types correctly for intersphinx ("py:class", "typing_extensions.Self"), diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index d09a1f66539..b2f79ef0c65 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -1,11 +1,18 @@ # Copyright (c) 2018-2024, NVIDIA CORPORATION. +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal + import pandas as pd import pyarrow as pa import cudf -from cudf.core.column import StructColumn +from cudf.core.column import StructColumn, as_column from cudf.core.dtypes import IntervalDtype +if TYPE_CHECKING: + from cudf.core.column import ColumnBase + class IntervalColumn(StructColumn): def __init__( @@ -85,6 +92,61 @@ def copy(self, deep=True): children=struct_copy.base_children, ) + @property + def is_empty(self) -> ColumnBase: + left_equals_right = (self.right == self.left).fillna(False) + not_closed_both = as_column( + self.dtype.closed != "both", length=len(self) + ) + return left_equals_right & not_closed_both + + @property + def is_non_overlapping_monotonic(self) -> bool: + raise NotImplementedError( + "is_overlapping is currently not implemented." + ) + + @property + def is_overlapping(self) -> bool: + raise NotImplementedError( + "is_overlapping is currently not implemented." + ) + + @property + def length(self) -> ColumnBase: + return self.right - self.left + + @property + def left(self) -> ColumnBase: + return self.children[0] + + @property + def mid(self) -> ColumnBase: + try: + return 0.5 * (self.left + self.right) + except TypeError: + # datetime safe version + return self.left + 0.5 * self.length + + @property + def right(self) -> ColumnBase: + return self.children[1] + + def overlaps(other) -> ColumnBase: + raise NotImplementedError("overlaps is not currently implemented.") + + def set_closed( + self, closed: Literal["left", "right", "both", "neither"] + ) -> IntervalColumn: + return IntervalColumn( + size=self.size, + dtype=IntervalDtype(self.dtype.fields["left"], closed), + mask=self.base_mask, + offset=self.offset, + null_count=self.null_count, + children=self.base_children, + ) + def as_interval_column(self, dtype): if isinstance(dtype, IntervalDtype): return IntervalColumn( diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 888ea25cdae..cd879d559cd 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3429,6 +3429,31 @@ def from_breaks( ) return IntervalIndex(interval_col, name=name, closed=closed) + @classmethod + def from_arrays( + cls, + left, + right, + closed: Literal["left", "right", "both", "neither"] = "right", + copy: bool = False, + dtype=None, + ) -> Self: + raise NotImplementedError("from_arrays is currently not supported.") + + @classmethod + def from_tuples( + cls, + data, + closed: Literal["left", "right", "both", "neither"] = "right", + name=None, + copy: bool = False, + dtype=None, + ) -> IntervalIndex: + piidx = pd.IntervalIndex.from_tuples( + data, closed=closed, name=name, copy=copy, dtype=dtype + ) + return cls.from_pandas(piidx) + def __getitem__(self, index): raise NotImplementedError( "Getting a scalar from an IntervalIndex is not yet supported" @@ -3443,6 +3468,104 @@ def _is_boolean(self): def _clean_nulls_from_index(self): return self + @property + def is_empty(self) -> cupy.ndarray: + """ + Indicates if an interval is empty, meaning it contains no points. + """ + return self._column.is_empty.values + + @property + def is_non_overlapping_monotonic(self) -> bool: + """ + Return a True if the IntervalIndex is non-overlapping and monotonic. + """ + return self._column.is_non_overlapping_monotonic + + @property + def is_overlapping(self) -> bool: + """ + Return True if the IntervalIndex has overlapping intervals, else False. + + Currently not implemented + """ + return self._column.is_overlapping + + @property + def length(self) -> Index: + """ + Return an Index with entries denoting the length of each Interval. + """ + return _index_from_data({None: self._column.length}) + + @property + def left(self) -> Index: + """ + Return left bounds of the intervals in the IntervalIndex. + + The left bounds of each interval in the IntervalIndex are + returned as an Index. The datatype of the left bounds is the + same as the datatype of the endpoints of the intervals. + """ + return _index_from_data({None: self._column.left}) + + @property + def mid(self) -> Index: + """ + Return the midpoint of each interval in the IntervalIndex as an Index. + + Each midpoint is calculated as the average of the left and right bounds + of each interval. + """ + return _index_from_data({None: self._column.mid}) + + @property + def right(self) -> Index: + """ + Return right bounds of the intervals in the IntervalIndex. + + The right bounds of each interval in the IntervalIndex are + returned as an Index. The datatype of the right bounds is the + same as the datatype of the endpoints of the intervals. + """ + return _index_from_data({None: self._column.right}) + + def overlaps(self, other) -> cupy.ndarray: + """ + Check elementwise if an Interval overlaps the values in the IntervalIndex. + + Currently not supported. + """ + return self._column.overlaps(other).values + + def set_closed( + self, closed: Literal["left", "right", "both", "neither"] + ) -> Self: + """ + Return an identical IntervalArray closed on the specified side. + + Parameters + ---------- + closed : {'left', 'right', 'both', 'neither'} + Whether the intervals are closed on the left-side, right-side, both + or neither. + """ + return type(self)._from_data( + {self.name: self._column.set_closed(closed)} + ) + + def to_tuples(self, na_tuple: bool = True) -> pd.Index: + """ + Return an Index of tuples of the form (left, right). + + Parameters + ---------- + na_tuple : bool, default True + If ``True``, return ``NA`` as a tuple ``(nan, nan)``. If ``False``, + just return ``NA`` as ``nan``. + """ + return self.to_pandas().to_tuples(na_tuple=na_tuple) + @_performance_tracking def as_index( diff --git a/python/cudf/cudf/tests/indexes/test_interval.py b/python/cudf/cudf/tests/indexes/test_interval.py index 87b76ab7609..3b3a9f96543 100644 --- a/python/cudf/cudf/tests/indexes/test_interval.py +++ b/python/cudf/cudf/tests/indexes/test_interval.py @@ -368,3 +368,36 @@ def test_intervalindex_conflicting_closed(): def test_intervalindex_invalid_data(): with pytest.raises(TypeError): cudf.IntervalIndex([1, 2]) + + +@pytest.mark.parametrize( + "attr", + [ + "is_empty", + "length", + "left", + "right", + "mid", + ], +) +def test_intervalindex_properties(attr): + pd_ii = pd.IntervalIndex.from_arrays([0, 1], [0, 2]) + cudf_ii = cudf.from_pandas(pd_ii) + + result = getattr(cudf_ii, attr) + expected = getattr(pd_ii, attr) + assert_eq(result, expected) + + +def test_set_closed(): + data = [pd.Interval(0, 1)] + result = cudf.IntervalIndex(data).set_closed("both") + expected = pd.IntervalIndex(data).set_closed("both") + assert_eq(result, expected) + + +def test_from_tuples(): + data = [(1, 2), (10, 20)] + result = cudf.IntervalIndex.from_tuples(data, closed="left", name="a") + expected = pd.IntervalIndex.from_tuples(data, closed="left", name="a") + assert_eq(result, expected) From be842259a835f4f7a5b9f7ff6fad1507d33c13cd Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:53:13 -0500 Subject: [PATCH 021/270] Remove cuDF dependency from pylibcudf column from_device tests (#16441) This removes the need to `import cudf` in `test_column_from_device` and removes a runtime dependency on numpy in the associated pylibcudf column method. Authors: - https://github.com/brandon-b-miller - Thomas Li (https://github.com/lithomas1) Approvers: - Thomas Li (https://github.com/lithomas1) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16441 --- python/cudf/cudf/_lib/pylibcudf/column.pyx | 9 ++--- .../cudf/_lib/pylibcudf/libcudf/types.pxd | 2 + python/cudf/cudf/_lib/pylibcudf/types.pxd | 2 + python/cudf/cudf/_lib/pylibcudf/types.pyx | 16 +++++++- .../test_column_from_device.py | 39 +++++++++++++++---- 5 files changed, 54 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/_lib/pylibcudf/column.pyx b/python/cudf/cudf/_lib/pylibcudf/column.pyx index a61e0629292..1d9902b0374 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/column.pyx @@ -15,13 +15,11 @@ from cudf._lib.pylibcudf.libcudf.types cimport size_type from .gpumemoryview cimport gpumemoryview from .scalar cimport Scalar -from .types cimport DataType, type_id +from .types cimport DataType, size_of, type_id from .utils cimport int_to_bitmask_ptr, int_to_void_ptr import functools -import numpy as np - cdef class Column: """A container of nullable device data as a column of elements. @@ -303,14 +301,15 @@ cdef class Column: raise ValueError("mask not yet supported.") typestr = iface['typestr'][1:] + data_type = _datatype_from_dtype_desc(typestr) + if not is_c_contiguous( iface['shape'], iface['strides'], - np.dtype(typestr).itemsize + size_of(data_type) ): raise ValueError("Data must be C-contiguous") - data_type = _datatype_from_dtype_desc(typestr) size = iface['shape'][0] return Column( data_type, diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/types.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/types.pxd index 8e94ec296cf..eabae68bc90 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/types.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/types.pxd @@ -98,3 +98,5 @@ cdef extern from "cudf/types.hpp" namespace "cudf" nogil: HIGHER MIDPOINT NEAREST + + cdef size_type size_of(data_type t) except + diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pxd b/python/cudf/cudf/_lib/pylibcudf/types.pxd index 7d3ddca14a1..1f3e1aa2fbb 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/types.pxd @@ -27,3 +27,5 @@ cdef class DataType: @staticmethod cdef DataType from_libcudf(data_type dt) + +cpdef size_type size_of(DataType t) diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pyx b/python/cudf/cudf/_lib/pylibcudf/types.pyx index c45c6071bb3..311f9ce4046 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/types.pyx @@ -2,7 +2,12 @@ from libc.stdint cimport int32_t -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type, type_id +from cudf._lib.pylibcudf.libcudf.types cimport ( + data_type, + size_of as cpp_size_of, + size_type, + type_id, +) from cudf._lib.pylibcudf.libcudf.utilities.type_dispatcher cimport type_to_id from cudf._lib.pylibcudf.libcudf.types import type_id as TypeId # no-cython-lint, isort:skip @@ -69,6 +74,15 @@ cdef class DataType: ret.c_obj = dt return ret +cpdef size_type size_of(DataType t): + """Returns the size in bytes of elements of the specified data_type. + + Only fixed-width types are supported. + + For details, see :cpp:func:`size_of`. + """ + with nogil: + return cpp_size_of(t.c_obj) SIZE_TYPE = DataType(type_to_id[size_type]()) SIZE_TYPE_ID = SIZE_TYPE.id() diff --git a/python/cudf/cudf/pylibcudf_tests/test_column_from_device.py b/python/cudf/cudf/pylibcudf_tests/test_column_from_device.py index c4ff7bb43a5..78ee2cb100e 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_column_from_device.py +++ b/python/cudf/cudf/pylibcudf_tests/test_column_from_device.py @@ -4,7 +4,8 @@ import pytest from utils import assert_column_eq -import cudf +import rmm + from cudf._lib import pylibcudf as plc VALID_TYPES = [ @@ -35,17 +36,39 @@ def valid_type(request): return request.param +class DataBuffer: + def __init__(self, obj, dtype): + self.obj = rmm.DeviceBuffer.to_device(obj) + self.dtype = dtype + self.shape = (int(len(self.obj) / self.dtype.itemsize),) + self.strides = (self.dtype.itemsize,) + self.typestr = self.dtype.str + + @property + def __cuda_array_interface__(self): + return { + "data": self.obj.__cuda_array_interface__["data"], + "shape": self.shape, + "strides": self.strides, + "typestr": self.typestr, + "version": 0, + } + + @pytest.fixture -def valid_column(valid_type): +def input_column(valid_type): if valid_type == pa.bool_(): return pa.array([True, False, True], type=valid_type) return pa.array([1, 2, 3], type=valid_type) -def test_from_cuda_array_interface(valid_column): - col = plc.column.Column.from_cuda_array_interface_obj( - cudf.Series(valid_column) - ) - expect = valid_column +@pytest.fixture +def iface_obj(input_column): + data = input_column.to_numpy(zero_copy_only=False) + return DataBuffer(data.view("uint8"), data.dtype) + + +def test_from_cuda_array_interface(input_column, iface_obj): + col = plc.column.Column.from_cuda_array_interface_obj(iface_obj) - assert_column_eq(expect, col) + assert_column_eq(input_column, col) From 9d0c57a64d63d52182bd1c1e930180bf62404f1a Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:59:27 -0700 Subject: [PATCH 022/270] Add skiprows and nrows to parquet reader (#16214) closes #15144 Authors: - Thomas Li (https://github.com/lithomas1) - Muhammad Haseeb (https://github.com/mhaseeb123) Approvers: - Muhammad Haseeb (https://github.com/mhaseeb123) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16214 --- python/cudf/cudf/_lib/parquet.pyx | 35 ++++++++++++----- .../cudf/cudf/_lib/pylibcudf/io/parquet.pxd | 2 +- .../cudf/cudf/_lib/pylibcudf/io/parquet.pyx | 18 ++++----- python/cudf/cudf/io/parquet.py | 23 +++++++++++ .../cudf/pylibcudf_tests/io/test_parquet.py | 2 +- python/cudf/cudf/tests/test_parquet.py | 39 +++++++++++++++++++ python/cudf/cudf/utils/ioutils.py | 10 +++++ python/cudf_polars/cudf_polars/dsl/ir.py | 2 +- 8 files changed, 110 insertions(+), 21 deletions(-) diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index a2eed94bb3c..4a4b13b0b31 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -22,7 +22,7 @@ from cudf._lib.utils cimport _data_from_columns, data_from_pylibcudf_io from cudf._lib.utils import _index_level_name, generate_pandas_metadata -from libc.stdint cimport uint8_t +from libc.stdint cimport int64_t, uint8_t from libcpp cimport bool from libcpp.map cimport map from libcpp.memory cimport make_unique, unique_ptr @@ -132,7 +132,10 @@ cdef object _process_metadata(object df, object filepaths_or_buffers, list pa_buffers, bool allow_range_index, - bool use_pandas_metadata): + bool use_pandas_metadata, + size_type nrows=-1, + int64_t skip_rows=0, + ): add_df_col_struct_names(df, child_names) index_col = None @@ -221,9 +224,13 @@ cdef object _process_metadata(object df, else: idx = cudf.Index(cudf.core.column.column_empty(0)) else: + start = range_index_meta["start"] + skip_rows + stop = range_index_meta["stop"] + if nrows != -1: + stop = start + nrows idx = cudf.RangeIndex( - start=range_index_meta['start'], - stop=range_index_meta['stop'], + start=start, + stop=stop, step=range_index_meta['step'], name=range_index_meta['name'] ) @@ -260,7 +267,9 @@ def read_parquet_chunked( row_groups=None, use_pandas_metadata=True, size_t chunk_read_limit=0, - size_t pass_read_limit=1024000000 + size_t pass_read_limit=1024000000, + size_type nrows=-1, + int64_t skip_rows=0 ): # Convert NativeFile buffers to NativeFileDatasource, # but save original buffers in case we need to use @@ -287,7 +296,9 @@ def read_parquet_chunked( row_groups, use_pandas_metadata, chunk_read_limit=chunk_read_limit, - pass_read_limit=pass_read_limit + pass_read_limit=pass_read_limit, + skip_rows=skip_rows, + nrows=nrows, ) tbl_w_meta = reader.read_chunk() @@ -320,13 +331,16 @@ def read_parquet_chunked( df = _process_metadata(df, column_names, child_names, per_file_user_data, row_groups, filepaths_or_buffers, pa_buffers, - allow_range_index, use_pandas_metadata) + allow_range_index, use_pandas_metadata, + nrows=nrows, skip_rows=skip_rows) return df cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, use_pandas_metadata=True, - Expression filters=None): + Expression filters=None, + size_type nrows=-1, + int64_t skip_rows=0): """ Cython function to call into libcudf API, see `read_parquet`. @@ -362,6 +376,8 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, filters, convert_strings_to_categories = False, use_pandas_metadata = use_pandas_metadata, + skip_rows = skip_rows, + nrows = nrows, ) df = cudf.DataFrame._from_data( @@ -371,7 +387,8 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, df = _process_metadata(df, tbl_w_meta.column_names(include_children=False), tbl_w_meta.child_names, tbl_w_meta.per_file_user_data, row_groups, filepaths_or_buffers, pa_buffers, - allow_range_index, use_pandas_metadata) + allow_range_index, use_pandas_metadata, + nrows=nrows, skip_rows=skip_rows) return df cpdef read_parquet_metadata(filepaths_or_buffers): diff --git a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd index 027f215fb91..93ef849b813 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd @@ -28,7 +28,7 @@ cpdef read_parquet( bool convert_strings_to_categories = *, bool use_pandas_metadata = *, int64_t skip_rows = *, - size_type num_rows = *, + size_type nrows = *, # disabled see comment in parquet.pyx for more # ReaderColumnSchema reader_column_schema = *, # DataType timestamp_type = * diff --git a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx index 96119e1b714..84a79f9565f 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx @@ -26,7 +26,7 @@ cdef parquet_reader_options _setup_parquet_reader_options( bool convert_strings_to_categories = False, bool use_pandas_metadata = True, int64_t skip_rows = 0, - size_type num_rows = -1, + size_type nrows = -1, # ReaderColumnSchema reader_column_schema = None, # DataType timestamp_type = DataType(type_id.EMPTY) ): @@ -40,8 +40,8 @@ cdef parquet_reader_options _setup_parquet_reader_options( ) if row_groups is not None: opts.set_row_groups(row_groups) - if num_rows != -1: - opts.set_num_rows(num_rows) + if nrows != -1: + opts.set_num_rows(nrows) if skip_rows != 0: opts.set_skip_rows(skip_rows) if columns is not None: @@ -73,7 +73,7 @@ cdef class ChunkedParquetReader: Whether to convert string columns to the category type skip_rows : int64_t, default 0 The number of rows to skip from the start of the file. - num_rows : size_type, default -1 + nrows : size_type, default -1 The number of rows to read. By default, read the entire file. chunk_read_limit : size_t, default 0 Limit on total number of bytes to be returned per read, @@ -90,7 +90,7 @@ cdef class ChunkedParquetReader: bool use_pandas_metadata=True, bool convert_strings_to_categories=False, int64_t skip_rows = 0, - size_type num_rows = -1, + size_type nrows = -1, size_t chunk_read_limit=0, size_t pass_read_limit=1024000000 ): @@ -103,7 +103,7 @@ cdef class ChunkedParquetReader: convert_strings_to_categories=convert_strings_to_categories, use_pandas_metadata=use_pandas_metadata, skip_rows=skip_rows, - num_rows=num_rows, + nrows=nrows, ) with nogil: @@ -152,7 +152,7 @@ cpdef read_parquet( bool convert_strings_to_categories = False, bool use_pandas_metadata = True, int64_t skip_rows = 0, - size_type num_rows = -1, + size_type nrows = -1, # Disabled, these aren't used by cudf-python # we should only add them back in if there's user demand # ReaderColumnSchema reader_column_schema = None, @@ -178,7 +178,7 @@ cpdef read_parquet( the per-file user metadata of the ``TableWithMetadata`` skip_rows : int64_t, default 0 The number of rows to skip from the start of the file. - num_rows : size_type, default -1 + nrows : size_type, default -1 The number of rows to read. By default, read the entire file. Returns @@ -195,7 +195,7 @@ cpdef read_parquet( convert_strings_to_categories, use_pandas_metadata, skip_rows, - num_rows, + nrows, ) with nogil: diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 7dab2f20100..4a419a2fbb6 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -539,6 +539,8 @@ def read_parquet( open_file_options=None, bytes_per_thread=None, dataset_kwargs=None, + nrows=None, + skip_rows=None, *args, **kwargs, ): @@ -685,6 +687,8 @@ def read_parquet( partition_keys=partition_keys, partition_categories=partition_categories, dataset_kwargs=dataset_kwargs, + nrows=nrows, + skip_rows=skip_rows, **kwargs, ) # Apply filters row-wise (if any are defined), and return @@ -813,6 +817,8 @@ def _parquet_to_frame( partition_keys=None, partition_categories=None, dataset_kwargs=None, + nrows=None, + skip_rows=None, **kwargs, ): # If this is not a partitioned read, only need @@ -820,11 +826,18 @@ def _parquet_to_frame( if not partition_keys: return _read_parquet( paths_or_buffers, + nrows=nrows, + skip_rows=skip_rows, *args, row_groups=row_groups, **kwargs, ) + if nrows is not None or skip_rows is not None: + raise NotImplementedError( + "nrows/skip_rows is not supported when reading a partitioned parquet dataset" + ) + partition_meta = None partitioning = (dataset_kwargs or {}).get("partitioning", None) if hasattr(partitioning, "schema"): @@ -912,6 +925,8 @@ def _read_parquet( columns=None, row_groups=None, use_pandas_metadata=None, + nrows=None, + skip_rows=None, *args, **kwargs, ): @@ -934,13 +949,21 @@ def _read_parquet( columns=columns, row_groups=row_groups, use_pandas_metadata=use_pandas_metadata, + nrows=nrows if nrows is not None else -1, + skip_rows=skip_rows if skip_rows is not None else 0, ) else: + if nrows is None: + nrows = -1 + if skip_rows is None: + skip_rows = 0 return libparquet.read_parquet( filepaths_or_buffers, columns=columns, row_groups=row_groups, use_pandas_metadata=use_pandas_metadata, + nrows=nrows, + skip_rows=skip_rows, ) else: if ( diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py b/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py index 07d2ab3d69a..dbd20cd473e 100644 --- a/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py +++ b/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py @@ -31,7 +31,7 @@ def test_read_parquet_basic( res = plc.io.parquet.read_parquet( plc.io.SourceInfo([source]), - num_rows=nrows, + nrows=nrows, skip_rows=skiprows, columns=columns, ) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 3806b901b10..879a2c50db7 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -1978,6 +1978,25 @@ def test_parquet_partitioned(tmpdir_factory, cols, filename): assert fn == filename +@pytest.mark.parametrize("kwargs", [{"nrows": 1}, {"skip_rows": 1}]) +def test_parquet_partitioned_notimplemented(tmpdir_factory, kwargs): + # Checks that write_to_dataset is wrapping to_parquet + # as expected + pdf_dir = str(tmpdir_factory.mktemp("pdf_dir")) + size = 100 + pdf = pd.DataFrame( + { + "a": np.arange(0, stop=size, dtype="int64"), + "b": np.random.choice(list("abcd"), size=size), + "c": np.random.choice(np.arange(4), size=size), + } + ) + pdf.to_parquet(pdf_dir, index=False, partition_cols=["b"]) + + with pytest.raises(NotImplementedError): + cudf.read_parquet(pdf_dir, **kwargs) + + @pytest.mark.parametrize("return_meta", [True, False]) def test_parquet_writer_chunked_partitioned(tmpdir_factory, return_meta): pdf_dir = str(tmpdir_factory.mktemp("pdf_dir")) @@ -3768,6 +3787,26 @@ def test_parquet_chunked_reader( assert_eq(expected, actual) +@pytest.mark.parametrize( + "nrows,skip_rows", + [ + (0, 0), + (1000, 0), + (0, 1000), + (1000, 10000), + ], +) +def test_parquet_reader_nrows_skiprows(nrows, skip_rows): + df = pd.DataFrame( + {"a": [1, 2, 3, 4] * 100000, "b": ["av", "qw", "hi", "xyz"] * 100000} + ) + expected = df[skip_rows : skip_rows + nrows] + buffer = BytesIO() + df.to_parquet(buffer) + got = cudf.read_parquet(buffer, nrows=nrows, skip_rows=skip_rows) + assert_eq(expected, got) + + def test_parquet_reader_pandas_compatibility(): df = pd.DataFrame( {"a": [1, 2, 3, 4] * 10000, "b": ["av", "qw", "hi", "xyz"] * 10000} diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 80555750b3a..448a815fe1b 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -199,6 +199,16 @@ in parallel (using a python thread pool). Default allocation is {bytes_per_thread} bytes. This parameter is functional only when `use_python_file_object=False`. +skiprows : int, default None + If not None, the number of rows to skip from the start of the file. + + .. note:: + This option is not supported when the low-memory mode is on. +nrows : int, default None + If not None, the total number of rows to read. + + .. note: + This option is not supported when the low-memory mode is on. Returns ------- diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 7f62dff4389..3754addeb11 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -321,7 +321,7 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: tbl_w_meta = plc.io.parquet.read_parquet( plc.io.SourceInfo(self.paths), columns=with_columns, - num_rows=nrows, + nrows=nrows, ) df = DataFrame.from_table( tbl_w_meta.tbl, From 05745d04e08ea494a50d12bad977af7e71aaf27b Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:00:19 -0400 Subject: [PATCH 023/270] Improve performance of hash_character_ngrams using warp-per-string kernel (#16212) Improves the performance of `nvtext::hash_character_ngrams` using a warp-per-string kernel instead of a string per thread. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Yunsong Wang (https://github.com/PointKernel) - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/16212 --- cpp/src/text/generate_ngrams.cu | 161 ++++++++++++++++++++++---------- 1 file changed, 113 insertions(+), 48 deletions(-) diff --git a/cpp/src/text/generate_ngrams.cu b/cpp/src/text/generate_ngrams.cu index 724f3603f29..6f700f84ec4 100644 --- a/cpp/src/text/generate_ngrams.cu +++ b/cpp/src/text/generate_ngrams.cu @@ -36,10 +36,12 @@ #include #include +#include +#include #include +#include #include #include -#include #include @@ -165,6 +167,47 @@ std::unique_ptr generate_ngrams(cudf::strings_column_view const& s namespace detail { namespace { +constexpr cudf::thread_index_type block_size = 256; +constexpr cudf::thread_index_type bytes_per_thread = 4; + +/** + * @brief Counts the number of ngrams in each row of the given strings column + * + * Each warp processes a single string. + * Formula is `count = max(0,str.length() - ngrams + 1)` + * If a string has less than ngrams characters, its count is 0. + */ +CUDF_KERNEL void count_char_ngrams_kernel(cudf::column_device_view const d_strings, + cudf::size_type ngrams, + cudf::size_type* d_counts) +{ + auto const idx = cudf::detail::grid_1d::global_thread_id(); + + auto const str_idx = idx / cudf::detail::warp_size; + if (str_idx >= d_strings.size()) { return; } + if (d_strings.is_null(str_idx)) { + d_counts[str_idx] = 0; + return; + } + + namespace cg = cooperative_groups; + auto const warp = cg::tiled_partition(cg::this_thread_block()); + + auto const d_str = d_strings.element(str_idx); + auto const end = d_str.data() + d_str.size_bytes(); + + auto const lane_idx = warp.thread_rank(); + cudf::size_type count = 0; + for (auto itr = d_str.data() + (lane_idx * bytes_per_thread); itr < end; + itr += cudf::detail::warp_size * bytes_per_thread) { + for (auto s = itr; (s < (itr + bytes_per_thread)) && (s < end); ++s) { + count += static_cast(cudf::strings::detail::is_begin_utf8_char(*s)); + } + } + auto const char_count = cg::reduce(warp, count, cg::plus()); + if (lane_idx == 0) { d_counts[str_idx] = cuda::std::max(0, char_count - ngrams + 1); } +} + /** * @brief Generate character ngrams for each string * @@ -220,17 +263,16 @@ std::unique_ptr generate_character_ngrams(cudf::strings_column_vie auto const d_strings = cudf::column_device_view::create(input.parent(), stream); - auto sizes_itr = cudf::detail::make_counting_transform_iterator( - 0, - cuda::proclaim_return_type( - [d_strings = *d_strings, ngrams] __device__(auto idx) { - if (d_strings.is_null(idx)) { return 0; } - auto const length = d_strings.element(idx).length(); - return std::max(0, static_cast(length + 1 - ngrams)); - })); - auto [offsets, total_ngrams] = - cudf::detail::make_offsets_child_column(sizes_itr, sizes_itr + input.size(), stream, mr); + auto [offsets, total_ngrams] = [&] { + auto counts = rmm::device_uvector(input.size(), stream); + auto const num_blocks = cudf::util::div_rounding_up_safe( + static_cast(input.size()) * cudf::detail::warp_size, block_size); + count_char_ngrams_kernel<<>>( + *d_strings, ngrams, counts.data()); + return cudf::detail::make_offsets_child_column(counts.begin(), counts.end(), stream, mr); + }(); auto d_offsets = offsets->view().data(); + CUDF_EXPECTS(total_ngrams > 0, "Insufficient number of characters in each string to generate ngrams"); @@ -246,36 +288,64 @@ std::unique_ptr generate_character_ngrams(cudf::strings_column_vie } namespace { + /** * @brief Computes the hash of each character ngram * - * Each thread processes a single string. Substrings are resolved for every character + * Each warp processes a single string. Substrings are resolved for every character * of the string and hashed. */ -struct character_ngram_hash_fn { - cudf::column_device_view const d_strings; - cudf::size_type ngrams; - cudf::size_type const* d_ngram_offsets; - cudf::hash_value_type* d_results; +CUDF_KERNEL void character_ngram_hash_kernel(cudf::column_device_view const d_strings, + cudf::size_type ngrams, + cudf::size_type const* d_ngram_offsets, + cudf::hash_value_type* d_results) +{ + auto const idx = cudf::detail::grid_1d::global_thread_id(); + if (idx >= (static_cast(d_strings.size()) * cudf::detail::warp_size)) { + return; + } - __device__ void operator()(cudf::size_type idx) const - { - if (d_strings.is_null(idx)) return; - auto const d_str = d_strings.element(idx); - if (d_str.empty()) return; - auto itr = d_str.begin(); - auto const ngram_offset = d_ngram_offsets[idx]; - auto const ngram_count = d_ngram_offsets[idx + 1] - ngram_offset; - auto const hasher = cudf::hashing::detail::MurmurHash3_x86_32{0}; - auto d_hashes = d_results + ngram_offset; - for (cudf::size_type n = 0; n < ngram_count; ++n, ++itr) { - auto const begin = itr.byte_offset(); - auto const end = (itr + ngrams).byte_offset(); - auto const ngram = cudf::string_view(d_str.data() + begin, end - begin); - *d_hashes++ = hasher(ngram); + auto const str_idx = idx / cudf::detail::warp_size; + + if (d_strings.is_null(str_idx)) { return; } + auto const d_str = d_strings.element(str_idx); + if (d_str.empty()) { return; } + + __shared__ cudf::hash_value_type hvs[block_size]; // temp store for hash values + + auto const ngram_offset = d_ngram_offsets[str_idx]; + auto const hasher = cudf::hashing::detail::MurmurHash3_x86_32{0}; + + auto const end = d_str.data() + d_str.size_bytes(); + auto const warp_count = (d_str.size_bytes() / cudf::detail::warp_size) + 1; + auto const lane_idx = idx % cudf::detail::warp_size; + + auto d_hashes = d_results + ngram_offset; + auto itr = d_str.data() + lane_idx; + for (auto i = 0; i < warp_count; ++i) { + cudf::hash_value_type hash = 0; + if (itr < end && cudf::strings::detail::is_begin_utf8_char(*itr)) { + // resolve ngram substring + auto const sub_str = + cudf::string_view(itr, static_cast(thrust::distance(itr, end))); + auto const [bytes, left] = + cudf::strings::detail::bytes_to_character_position(sub_str, ngrams); + if (left == 0) { hash = hasher(cudf::string_view(itr, bytes)); } + } + hvs[threadIdx.x] = hash; // store hash into shared memory + __syncwarp(); + if (lane_idx == 0) { + // copy valid hash values into d_hashes + auto const hashes = &hvs[threadIdx.x]; + d_hashes = thrust::copy_if( + thrust::seq, hashes, hashes + cudf::detail::warp_size, d_hashes, [](auto h) { + return h != 0; + }); } + __syncwarp(); + itr += cudf::detail::warp_size; } -}; +} } // namespace std::unique_ptr hash_character_ngrams(cudf::strings_column_view const& input, @@ -291,18 +361,16 @@ std::unique_ptr hash_character_ngrams(cudf::strings_column_view co if (input.is_empty()) { return cudf::make_empty_column(output_type); } auto const d_strings = cudf::column_device_view::create(input.parent(), stream); + auto const grid = cudf::detail::grid_1d( + static_cast(input.size()) * cudf::detail::warp_size, block_size); // build offsets column by computing the number of ngrams per string - auto sizes_itr = cudf::detail::make_counting_transform_iterator( - 0, - cuda::proclaim_return_type( - [d_strings = *d_strings, ngrams] __device__(auto idx) { - if (d_strings.is_null(idx)) { return 0; } - auto const length = d_strings.element(idx).length(); - return std::max(0, static_cast(length + 1 - ngrams)); - })); - auto [offsets, total_ngrams] = - cudf::detail::make_offsets_child_column(sizes_itr, sizes_itr + input.size(), stream, mr); + auto [offsets, total_ngrams] = [&] { + auto counts = rmm::device_uvector(input.size(), stream); + count_char_ngrams_kernel<<>>( + *d_strings, ngrams, counts.data()); + return cudf::detail::make_offsets_child_column(counts.begin(), counts.end(), stream, mr); + }(); auto d_offsets = offsets->view().data(); CUDF_EXPECTS(total_ngrams > 0, @@ -313,11 +381,8 @@ std::unique_ptr hash_character_ngrams(cudf::strings_column_view co cudf::make_numeric_column(output_type, total_ngrams, cudf::mask_state::UNALLOCATED, stream, mr); auto d_hashes = hashes->mutable_view().data(); - character_ngram_hash_fn generator{*d_strings, ngrams, d_offsets, d_hashes}; - thrust::for_each_n(rmm::exec_policy(stream), - thrust::counting_iterator(0), - input.size(), - generator); + character_ngram_hash_kernel<<>>( + *d_strings, ngrams, d_offsets, d_hashes); return make_lists_column( input.size(), std::move(offsets), std::move(hashes), 0, rmm::device_buffer{}, stream, mr); From a8a367009ff64478d78eb916fc9dc65b77b89aac Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:45:01 -0700 Subject: [PATCH 024/270] Move exception handler into pylibcudf from cudf (#16468) PR to help prepare for the splitting out of pylibcudf. Authors: - Thomas Li (https://github.com/lithomas1) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16468 --- docs/cudf/source/developer_guide/pylibcudf.md | 2 +- .../{ => pylibcudf}/exception_handler.pxd | 6 +-- .../cudf/_lib/pylibcudf/libcudf/binaryop.pxd | 12 ++--- .../cudf/_lib/pylibcudf/libcudf/copying.pxd | 44 +++++++++---------- .../_lib/pylibcudf/libcudf/lists/contains.pxd | 12 ++--- 5 files changed, 38 insertions(+), 38 deletions(-) rename python/cudf/cudf/_lib/{ => pylibcudf}/exception_handler.pxd (95%) diff --git a/docs/cudf/source/developer_guide/pylibcudf.md b/docs/cudf/source/developer_guide/pylibcudf.md index 0b881b2b057..2ae545a4955 100644 --- a/docs/cudf/source/developer_guide/pylibcudf.md +++ b/docs/cudf/source/developer_guide/pylibcudf.md @@ -149,7 +149,7 @@ Some guidelines on what should be tested: - Exception: In special cases where constructing suitable large tests is difficult in C++ (such as creating suitable input data for I/O testing), tests may be added to pylibcudf instead. - Nullable data should always be tested. - Expected exceptions should be tested. Tests should be written from the user's perspective in mind, and if the API is not currently throwing the appropriate exception it should be updated. - - Important note: If the exception should be produced by libcudf, the underlying libcudf API should be updated to throw the desired exception in C++. Such changes may require consultation with libcudf devs in nontrivial cases. [This issue](https://github.com/rapidsai/cudf/issues/12885) provides an overview and an indication of acceptable exception types that should cover most use cases. In rare cases a new C++ exception may need to be introduced in [`error.hpp`](https://github.com/rapidsai/cudf/blob/branch-24.04/cpp/include/cudf/utilities/error.hpp). If so, this exception will also need to be mapped to a suitable Python exception in [`exception_handler.pxd`](https://github.com/rapidsai/cudf/blob/branch-24.04/python/cudf/cudf/_lib/exception_handler.pxd). + - Important note: If the exception should be produced by libcudf, the underlying libcudf API should be updated to throw the desired exception in C++. Such changes may require consultation with libcudf devs in nontrivial cases. [This issue](https://github.com/rapidsai/cudf/issues/12885) provides an overview and an indication of acceptable exception types that should cover most use cases. In rare cases a new C++ exception may need to be introduced in [`error.hpp`](https://github.com/rapidsai/cudf/blob/branch-24.04/cpp/include/cudf/utilities/error.hpp). If so, this exception will also need to be mapped to a suitable Python exception in `exception_handler.pxd`. Some guidelines on how best to use pytests. - By default, fixtures producing device data containers should be of module scope and treated as immutable by tests. Allocating data on the GPU is expensive and slows tests. Almost all pylibcudf operations are out of place operations, so module-scoped fixtures should not typically be problematic to work with. Session-scoped fixtures would also work, but they are harder to reason about since they live in a different module, and if they need to change for any reason they could affect an arbitrarily large number of tests. Module scope is a good balance. diff --git a/python/cudf/cudf/_lib/exception_handler.pxd b/python/cudf/cudf/_lib/pylibcudf/exception_handler.pxd similarity index 95% rename from python/cudf/cudf/_lib/exception_handler.pxd rename to python/cudf/cudf/_lib/pylibcudf/exception_handler.pxd index 4337d8db285..6abcd0a1c0f 100644 --- a/python/cudf/cudf/_lib/exception_handler.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/exception_handler.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # See @@ -24,7 +24,7 @@ cdef extern from *: * Since this function interoperates with Python's exception state, it * does not throw any C++ exceptions. */ - void cudf_exception_handler() + void libcudf_exception_handler() { // Catch a handful of different errors here and turn them into the // equivalent Python errors. @@ -66,4 +66,4 @@ cdef extern from *: } // anonymous namespace """ - cdef void cudf_exception_handler() + cdef void libcudf_exception_handler() diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd index b34fea6a775..78da5980db4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd @@ -5,7 +5,7 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string -from cudf._lib.exception_handler cimport cudf_exception_handler +from cudf._lib.pylibcudf.exception_handler cimport libcudf_exception_handler from cudf._lib.pylibcudf.libcudf.column.column cimport column from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar @@ -55,28 +55,28 @@ cdef extern from "cudf/binaryop.hpp" namespace "cudf" nogil: const column_view& rhs, binary_operator op, data_type output_type - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] binary_operation ( const column_view& lhs, const scalar& rhs, binary_operator op, data_type output_type - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] binary_operation ( const column_view& lhs, const column_view& rhs, binary_operator op, data_type output_type - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] binary_operation ( const column_view& lhs, const column_view& rhs, const string& op, data_type output_type - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef extern from "cudf/binaryop.hpp" namespace "cudf::binops" nogil: cdef bool is_supported_operation( @@ -84,4 +84,4 @@ cdef extern from "cudf/binaryop.hpp" namespace "cudf::binops" nogil: data_type lhs_type, data_type rhs_type, binary_operator op - ) except +cudf_exception_handler + ) except +libcudf_exception_handler diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pxd index 001489d69bf..af3a16ad01b 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pxd @@ -8,7 +8,7 @@ from libcpp.vector cimport vector from rmm._lib.device_buffer cimport device_buffer -from cudf._lib.exception_handler cimport cudf_exception_handler +from cudf._lib.pylibcudf.exception_handler cimport libcudf_exception_handler from cudf._lib.pylibcudf.libcudf.column.column cimport column from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( column_view, @@ -30,25 +30,25 @@ cdef extern from "cudf/copying.hpp" namespace "cudf" nogil: const table_view& source_table, const column_view& gather_map, out_of_bounds_policy policy - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] shift( const column_view& input, size_type offset, const scalar& fill_values - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[table] scatter ( const table_view& source_table, const column_view& scatter_map, const table_view& target_table, - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[table] scatter ( const vector[reference_wrapper[constscalar]]& source_scalars, const column_view& indices, const table_view& target, - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cpdef enum class mask_allocation_policy(int32_t): NEVER @@ -57,22 +57,22 @@ cdef extern from "cudf/copying.hpp" namespace "cudf" nogil: cdef unique_ptr[column] empty_like ( const column_view& input_column - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] allocate_like ( const column_view& input_column, mask_allocation_policy policy - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] allocate_like ( const column_view& input_column, size_type size, mask_allocation_policy policy - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[table] empty_like ( const table_view& input_table - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef void copy_range_in_place ( const column_view& input_column, @@ -80,7 +80,7 @@ cdef extern from "cudf/copying.hpp" namespace "cudf" nogil: size_type input_begin, size_type input_end, size_type target_begin - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] copy_range ( const column_view& input_column, @@ -88,68 +88,68 @@ cdef extern from "cudf/copying.hpp" namespace "cudf" nogil: size_type input_begin, size_type input_end, size_type target_begin - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef vector[column_view] slice ( const column_view& input_column, vector[size_type] indices - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef vector[table_view] slice ( const table_view& input_table, vector[size_type] indices - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef vector[column_view] split ( const column_view& input_column, vector[size_type] splits - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef vector[table_view] split ( const table_view& input_table, vector[size_type] splits - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] copy_if_else ( const column_view& lhs, const column_view& rhs, const column_view& boolean_mask - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] copy_if_else ( const scalar& lhs, const column_view& rhs, const column_view& boolean_mask - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] copy_if_else ( const column_view& lhs, const scalar& rhs, const column_view boolean_mask - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] copy_if_else ( const scalar& lhs, const scalar& rhs, const column_view boolean_mask - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[table] boolean_mask_scatter ( const table_view& input, const table_view& target, const column_view& boolean_mask - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[table] boolean_mask_scatter ( const vector[reference_wrapper[constscalar]]& input, const table_view& target, const column_view& boolean_mask - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[scalar] get_element ( const column_view& input, size_type index - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cpdef enum class sample_with_replacement(bool): FALSE diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/contains.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/contains.pxd index 82aed7d70a0..40bb2e78970 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/contains.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/contains.pxd @@ -3,7 +3,7 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr -from cudf._lib.exception_handler cimport cudf_exception_handler +from cudf._lib.pylibcudf.exception_handler cimport libcudf_exception_handler from cudf._lib.pylibcudf.libcudf.column.column cimport column from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( @@ -21,25 +21,25 @@ cdef extern from "cudf/lists/contains.hpp" namespace "cudf::lists" nogil: cdef unique_ptr[column] contains( const lists_column_view& lists, const scalar& search_key, - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] contains( const lists_column_view& lists, const column_view& search_keys, - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] contains_nulls( const lists_column_view& lists, - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] index_of( const lists_column_view& lists, const scalar& search_key, duplicate_find_option find_option, - ) except +cudf_exception_handler + ) except +libcudf_exception_handler cdef unique_ptr[column] index_of( const lists_column_view& lists, const column_view& search_keys, duplicate_find_option find_option, - ) except +cudf_exception_handler + ) except +libcudf_exception_handler From cc19d8a7b424abbc87f7767e3bc60c54390dc9e3 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 2 Aug 2024 09:34:27 -1000 Subject: [PATCH 025/270] Use explicit construction of column subclass instead of `build_column` when type is known (#16470) When we need to construct a column with a specific type, we do not need to go through the indirection of `build_column`, which matches a column subclass to a passed type, and instead construct directly from the class instead Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Thomas Li (https://github.com/lithomas1) URL: https://github.com/rapidsai/cudf/pull/16470 --- python/cudf/cudf/core/_internals/where.py | 2 +- python/cudf/cudf/core/column/categorical.py | 46 +++++++++++++-------- python/cudf/cudf/core/column/column.py | 2 +- python/cudf/cudf/core/column/datetime.py | 10 ++--- python/cudf/cudf/core/column/numerical.py | 43 ++++++++----------- python/cudf/cudf/core/column/string.py | 6 +-- python/cudf/cudf/core/column/timedelta.py | 8 ++-- python/cudf/cudf/core/dataframe.py | 4 +- python/cudf/cudf/core/index.py | 8 ++-- 9 files changed, 64 insertions(+), 65 deletions(-) diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 18ab32d2c9e..9f36499586b 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -110,7 +110,7 @@ def _make_categorical_like(result, column): if isinstance(column, cudf.core.column.CategoricalColumn): result = cudf.core.column.build_categorical_column( categories=column.categories, - codes=cudf.core.column.build_column( + codes=cudf.core.column.NumericalColumn( result.base_data, dtype=result.dtype ), mask=result.base_mask, diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 9433a91b9c6..55bfae30470 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -572,13 +572,10 @@ def children(self) -> tuple[NumericalColumn]: codes_column = self.base_children[0] start = self.offset * codes_column.dtype.itemsize end = start + self.size * codes_column.dtype.itemsize - codes_column = cast( - cudf.core.column.NumericalColumn, - column.build_column( - data=codes_column.base_data[start:end], - dtype=codes_column.dtype, - size=self.size, - ), + codes_column = cudf.core.column.NumericalColumn( + data=codes_column.base_data[start:end], + dtype=codes_column.dtype, + size=self.size, ) self._children = (codes_column,) return self._children @@ -660,8 +657,9 @@ def slice(self, start: int, stop: int, stride: int | None = None) -> Self: Self, cudf.core.column.build_categorical_column( categories=self.categories, - codes=cudf.core.column.build_column( - codes.base_data, dtype=codes.dtype + codes=cudf.core.column.NumericalColumn( + codes.base_data, # type: ignore[arg-type] + dtype=codes.dtype, ), mask=codes.base_mask, ordered=self.ordered, @@ -734,7 +732,10 @@ def sort_values( codes = self.codes.sort_values(ascending, na_position) col = column.build_categorical_column( categories=self.dtype.categories._values, - codes=column.build_column(codes.base_data, dtype=codes.dtype), + codes=cudf.core.column.NumericalColumn( + codes.base_data, # type: ignore[arg-type] + dtype=codes.dtype, + ), mask=codes.base_mask, size=codes.size, ordered=self.dtype.ordered, @@ -842,7 +843,10 @@ def unique(self) -> CategoricalColumn: codes = self.codes.unique() return column.build_categorical_column( categories=self.categories, - codes=column.build_column(codes.base_data, dtype=codes.dtype), + codes=cudf.core.column.NumericalColumn( + codes.base_data, # type: ignore[arg-type] + dtype=codes.dtype, + ), mask=codes.base_mask, offset=codes.offset, size=codes.size, @@ -980,7 +984,9 @@ def find_and_replace( result = column.build_categorical_column( categories=new_cats["cats"], - codes=column.build_column(output.base_data, dtype=output.dtype), + codes=cudf.core.column.NumericalColumn( + output.base_data, dtype=output.dtype + ), mask=output.base_mask, offset=output.offset, size=output.size, @@ -1176,8 +1182,9 @@ def _concat( return column.build_categorical_column( categories=column.as_column(cats), - codes=column.build_column( - codes_col.base_data, dtype=codes_col.dtype + codes=cudf.core.column.NumericalColumn( + codes_col.base_data, # type: ignore[arg-type] + dtype=codes_col.dtype, ), mask=codes_col.base_mask, size=codes_col.size, @@ -1190,8 +1197,9 @@ def _with_type_metadata( if isinstance(dtype, CategoricalDtype): return column.build_categorical_column( categories=dtype.categories._values, - codes=column.build_column( - self.codes.base_data, dtype=self.codes.dtype + codes=cudf.core.column.NumericalColumn( + self.codes.base_data, # type: ignore[arg-type] + dtype=self.codes.dtype, ), mask=self.codes.base_mask, ordered=dtype.ordered, @@ -1339,7 +1347,7 @@ def _set_categories( Self, column.build_categorical_column( categories=new_cats, - codes=column.build_column( + codes=cudf.core.column.NumericalColumn( new_codes.base_data, dtype=new_codes.dtype ), mask=new_codes.base_mask, @@ -1472,7 +1480,9 @@ def pandas_categorical_as_column( return column.build_categorical_column( categories=categorical.categories, - codes=column.build_column(codes.base_data, codes.dtype), + codes=cudf.core.column.NumericalColumn( + codes.base_data, dtype=codes.dtype + ), size=codes.size, mask=mask, ordered=categorical.ordered, diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 7e0d8ced595..a7d2cb441dd 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1506,7 +1506,7 @@ def column_empty( elif isinstance(dtype, CategoricalDtype): data = None children = ( - build_column( + cudf.core.column.NumericalColumn( data=as_buffer( rmm.DeviceBuffer( size=row_count diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 81fbb914842..ce67ce81e6b 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -473,15 +473,15 @@ def as_timedelta_column(self, dtype: Dtype) -> None: # type: ignore[override] def as_numerical_column( self, dtype: Dtype - ) -> "cudf.core.column.NumericalColumn": - col = column.build_column( - data=self.base_data, - dtype=np.int64, + ) -> cudf.core.column.NumericalColumn: + col = cudf.core.column.NumericalColumn( + data=self.base_data, # type: ignore[arg-type] + dtype=np.dtype(np.int64), mask=self.base_mask, offset=self.offset, size=self.size, ) - return cast("cudf.core.column.NumericalColumn", col.astype(dtype)) + return cast(cudf.core.column.NumericalColumn, col.astype(dtype)) def strftime(self, format: str) -> cudf.core.column.StringColumn: if len(self) == 0: diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index f9404eb3b40..c326a10c844 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -13,13 +13,7 @@ from cudf import _lib as libcudf from cudf._lib import pylibcudf from cudf.api.types import is_integer, is_scalar -from cudf.core.column import ( - ColumnBase, - as_column, - build_column, - column, - string, -) +from cudf.core.column import ColumnBase, as_column, column, string from cudf.core.dtypes import CategoricalDtype from cudf.core.mixins import BinaryOperand from cudf.errors import MixedTypeError @@ -338,29 +332,23 @@ def as_string_column(self) -> cudf.core.column.StringColumn: def as_datetime_column( self, dtype: Dtype ) -> cudf.core.column.DatetimeColumn: - return cast( - "cudf.core.column.DatetimeColumn", - build_column( - data=self.astype("int64").base_data, - dtype=dtype, - mask=self.base_mask, - offset=self.offset, - size=self.size, - ), + return cudf.core.column.DatetimeColumn( + data=self.astype("int64").base_data, # type: ignore[arg-type] + dtype=dtype, + mask=self.base_mask, + offset=self.offset, + size=self.size, ) def as_timedelta_column( self, dtype: Dtype ) -> cudf.core.column.TimeDeltaColumn: - return cast( - "cudf.core.column.TimeDeltaColumn", - build_column( - data=self.astype("int64").base_data, - dtype=dtype, - mask=self.base_mask, - offset=self.offset, - size=self.size, - ), + return cudf.core.column.TimeDeltaColumn( + data=self.astype("int64").base_data, # type: ignore[arg-type] + dtype=dtype, + mask=self.base_mask, + offset=self.offset, + size=self.size, ) def as_decimal_column( @@ -637,7 +625,10 @@ def _with_type_metadata(self: ColumnBase, dtype: Dtype) -> ColumnBase: if isinstance(dtype, CategoricalDtype): return column.build_categorical_column( categories=dtype.categories._values, - codes=build_column(self.base_data, dtype=self.dtype), + codes=cudf.core.column.NumericalColumn( + self.base_data, # type: ignore[arg-type] + dtype=self.dtype, + ), mask=self.base_mask, ordered=dtype.ordered, size=self.size, diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index ec95c50f455..b422ff86b17 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5934,9 +5934,9 @@ def view(self, dtype) -> "cudf.core.column.ColumnBase": n_bytes_to_view = str_end_byte_offset - str_byte_offset - to_view = column.build_column( - self.base_data, - dtype=cudf.api.types.dtype("int8"), + to_view = cudf.core.column.NumericalColumn( + self.base_data, # type: ignore[arg-type] + dtype=np.dtype(np.int8), offset=str_byte_offset, size=n_bytes_to_view, ) diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index 47c8ed6fd95..ba0dc4779bb 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -265,10 +265,10 @@ def round(self, freq: str) -> ColumnBase: def as_numerical_column( self, dtype: Dtype - ) -> "cudf.core.column.NumericalColumn": - col = column.build_column( - data=self.base_data, - dtype=np.int64, + ) -> cudf.core.column.NumericalColumn: + col = cudf.core.column.NumericalColumn( + data=self.base_data, # type: ignore[arg-type] + dtype=np.dtype(np.int64), mask=self.base_mask, offset=self.offset, size=self.size, diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 52dc29974bf..865d2706ca3 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -46,10 +46,10 @@ from cudf.core.column import ( CategoricalColumn, ColumnBase, + NumericalColumn, StructColumn, as_column, build_categorical_column, - build_column, column_empty, concat_columns, ) @@ -8543,7 +8543,7 @@ def _reassign_categories(categories, cols, col_idxs): if idx in categories: cols[name] = build_categorical_column( categories=categories[idx], - codes=build_column( + codes=NumericalColumn( cols[name].base_data, dtype=cols[name].dtype ), mask=cols[name].base_mask, diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index cd879d559cd..0d29ef07e7d 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2434,12 +2434,10 @@ def to_pandas( return result @_performance_tracking - def _get_dt_field(self, field): + def _get_dt_field(self, field: str) -> Index: + """Return an Index of a numerical component of the DatetimeIndex.""" out_column = self._values.get_dt_field(field) - # column.column_empty_like always returns a Column object - # but we need a NumericalColumn for Index.. - # how should this be handled? - out_column = column.build_column( + out_column = NumericalColumn( data=out_column.base_data, dtype=out_column.dtype, mask=out_column.base_mask, From e0d1ac1efa9153f0a084bd72b7d4c300f9640196 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 2 Aug 2024 17:44:45 -0500 Subject: [PATCH 026/270] Fix typo in dispatch_row_equal. (#16473) This PR fixes a small typo in the C++ code. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Yunsong Wang (https://github.com/PointKernel) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/16473 --- cpp/src/stream_compaction/distinct.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/stream_compaction/distinct.cu b/cpp/src/stream_compaction/distinct.cu index e5cf29f3ebf..e2c5aba6802 100644 --- a/cpp/src/stream_compaction/distinct.cu +++ b/cpp/src/stream_compaction/distinct.cu @@ -51,7 +51,7 @@ namespace { * @param func The input functor to invoke */ template -rmm::device_uvector dipatch_row_equal( +rmm::device_uvector dispatch_row_equal( null_equality compare_nulls, nan_equality compare_nans, bool has_nulls, @@ -110,9 +110,9 @@ rmm::device_uvector distinct_indices(table_view const& input, }; if (cudf::detail::has_nested_columns(input)) { - return dipatch_row_equal(nulls_equal, nans_equal, has_nulls, row_equal, helper_func); + return dispatch_row_equal(nulls_equal, nans_equal, has_nulls, row_equal, helper_func); } else { - return dipatch_row_equal(nulls_equal, nans_equal, has_nulls, row_equal, helper_func); + return dispatch_row_equal(nulls_equal, nans_equal, has_nulls, row_equal, helper_func); } } From af57286536fc21b47b80e45be222773b751600c9 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Mon, 5 Aug 2024 07:16:34 -0500 Subject: [PATCH 027/270] Add missing pylibcudf strings docs (#16471) Noticed a few missing pylibcudf string docs that were missed, added them here. Authors: - https://github.com/brandon-b-miller - Thomas Li (https://github.com/lithomas1) Approvers: - Thomas Li (https://github.com/lithomas1) URL: https://github.com/rapidsai/cudf/pull/16471 --- .../api_docs/pylibcudf/strings/capitalize.rst | 6 +++ .../api_docs/pylibcudf/strings/char_types.rst | 6 +++ .../api_docs/pylibcudf/strings/find.rst | 6 +++ .../api_docs/pylibcudf/strings/index.rst | 5 ++ .../pylibcudf/strings/regex_flags.rst | 6 +++ .../pylibcudf/strings/regex_program.rst | 6 +++ .../_lib/pylibcudf/strings/capitalize.pyx | 48 ++++++++++++++++++- .../_lib/pylibcudf/strings/regex_program.pyx | 19 ++++++++ 8 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst new file mode 100644 index 00000000000..578b2b75e37 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst @@ -0,0 +1,6 @@ +========== +capitalize +========== + +.. automodule:: cudf._lib.pylibcudf.strings.capitalize + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst new file mode 100644 index 00000000000..577ec34915b --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst @@ -0,0 +1,6 @@ +========== +char_types +========== + +.. automodule:: cudf._lib.pylibcudf.strings.char_types + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst new file mode 100644 index 00000000000..61d4079e9a3 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst @@ -0,0 +1,6 @@ +==== +find +==== + +.. automodule:: cudf._lib.pylibcudf.strings.find + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/index.rst index cecf1ccc9bb..462a756a092 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/index.rst @@ -4,6 +4,11 @@ strings .. toctree:: :maxdepth: 1 + capitalize + char_types contains + find + regex_flags + regex_program replace slice diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst new file mode 100644 index 00000000000..0126b6a3706 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst @@ -0,0 +1,6 @@ +=========== +regex_flags +=========== + +.. automodule:: cudf._lib.pylibcudf.strings.regex_flags + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst new file mode 100644 index 00000000000..2f398186d51 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst @@ -0,0 +1,6 @@ +============= +regex_program +============= + +.. automodule:: cudf._lib.pylibcudf.strings.regex_program + :members: diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pyx b/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pyx index d3f79088018..ccf84d25572 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pyx @@ -22,7 +22,22 @@ cpdef Column capitalize( # TODO: default scalar values # https://github.com/rapidsai/cudf/issues/15505 ): - + """Returns a column of capitalized strings. + + For details, see :cpp:func:`cudf::strings::capitalize`. + + Parameters + ---------- + input : Column + String column + delimiters : Scalar, default None + Characters for identifying words to capitalize + + Returns + ------- + pylibcudf.Column + Column of strings capitalized from the input column + """ cdef unique_ptr[column] c_result if delimiters is None: @@ -47,6 +62,23 @@ cpdef Column title( Column input, string_character_types sequence_type=string_character_types.ALPHA ): + """Modifies first character of each word to upper-case and lower-cases + the rest. + + For details, see :cpp:func:`cudf::strings::title`. + + Parameters + ---------- + input : Column + String column + sequence_type : string_character_types, default string_character_types.ALPHA + The character type that is used when identifying words + + Returns + ------- + pylibcudf.Column + Column of titled strings + """ cdef unique_ptr[column] c_result with nogil: c_result = cpp_capitalize.title(input.view(), sequence_type) @@ -55,6 +87,20 @@ cpdef Column title( cpdef Column is_title(Column input): + """Checks if the strings in the input column are title formatted. + + For details, see :cpp:func:`cudf::strings::is_title`. + + Parameters + ---------- + input : Column + String column + + Returns + ------- + pylibcudf.Column + Column of type BOOL8 + """ cdef unique_ptr[column] c_result with nogil: c_result = cpp_capitalize.is_title(input.view()) diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pyx b/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pyx index d605b0aba02..5f0b8868452 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pyx @@ -13,12 +13,31 @@ from cudf._lib.pylibcudf.strings.regex_flags cimport regex_flags cdef class RegexProgram: + """Regex program class. + This is the Cython representation of + :cpp:class:`cudf::strings::regex_program`. + + Do not instantiate this class directly, use the `create` method. + + """ def __init__(self, *args, **kwargs): raise ValueError("Do not instantiate RegexProgram directly, use create") @staticmethod def create(str pattern, int flags): + """Create a program from a pattern. + + For detils, see :cpp:func:`cudf::strings::regex_program::create`. + + Parameters + ---------- + pattern : str + Regex pattern + flags : Uniont[int, RegexFlags] + Regex flags for interpreting special characters in the pattern + + """ cdef unique_ptr[regex_program] c_prog cdef regex_flags c_flags cdef string c_pattern = pattern.encode() From 837dfe51a2f4d0268d6976464eed637645f524ff Mon Sep 17 00:00:00 2001 From: Rahul Prabhu <100436830+sdrp713@users.noreply.github.com> Date: Mon, 5 Aug 2024 14:14:41 -0700 Subject: [PATCH 028/270] Added batch memset to memset data and validity buffers in parquet reader (#16281) Under some situations in the Parquet reader (particularly the case with tables containing many columns or deeply nested column) we burn a decent amount of time doing cudaMemset() operations on output buffers. A good amount of this overhead seems to stem from the fact that we're simply launching many tiny kernels. This PR adds a batched memset kernel that takes a list of device spans as a single input and does all the work under a single kernel launch. This PR addresses issue #15773 ## Improvements Using out performance cluster, improvements of 2.39% were shown on running the overall NDS queries Additionally, benchmarks were added showing big improvements(around 20%) especially on fixed width data types which can be shown below data_type | num_cols | cardinality | run_length | bytes_per_second_before_this_pr | bytes_per_second_after_this_pr | speedup --- | --- | --- | --- | --- | --- | --- INTEGRAL | 1000 | 0 | 1 | 36514934834 | 42756531566 | 1.170932709 INTEGRAL | 1000 | 1000 | 1 | 35364061247 | 39112512476 | 1.105996062 INTEGRAL | 1000 | 0 | 32 | 37349112510 | 39641370858 | 1.061373837 INTEGRAL | 1000 | 1000 | 32 | 39167079622 | 43740824957 | 1.116775245 FLOAT | 1000 | 0 | 1 | 51877322003 | 64083898838 | 1.235296973 FLOAT | 1000 | 1000 | 1 | 48983612272 | 58705522023 | 1.198472699 FLOAT | 1000 | 0 | 32 | 46544977658 | 53715018581 | 1.154045426 FLOAT | 1000 | 1000 | 32 | 54493432148 | 66617609904 | 1.22248879 DECIMAL | 1000 | 0 | 1 | 47616412888 | 57952310685 | 1.217065864 DECIMAL | 1000 | 1000 | 1 | 47166138095 | 54283772484 | 1.1509056 DECIMAL | 1000 | 0 | 32 | 45266163387 | 53770390830 | 1.18787162 DECIMAL | 1000 | 1000 | 32 | 52292176603 | 58847723569 | 1.125363819 TIMESTAMP | 1000 | 0 | 1 | 50245415328 | 60797982330 | 1.210020495 TIMESTAMP | 1000 | 1000 | 1 | 50300238706 | 60810368331 | 1.208947908 TIMESTAMP | 1000 | 0 | 32 | 55338354243 | 66786275739 | 1.206871376 TIMESTAMP | 1000 | 1000 | 32 | 55680028082 | 69029227374 | 1.23974843 DURATION | 1000 | 0 | 1 | 54680007758 | 66855201896 | 1.222662626 DURATION | 1000 | 1000 | 1 | 54305832171 | 66602436269 | 1.226432477 DURATION | 1000 | 0 | 32 | 60040760815 | 72663056969 | 1.210228784 DURATION | 1000 | 1000 | 32 | 60212221703 | 75646396131 | 1.256329595 STRING | 1000 | 0 | 1 | 29691707753 | 33388700976 | 1.12451265 STRING | 1000 | 1000 | 1 | 31411129876 | 35407241037 | 1.127219593 STRING | 1000 | 0 | 32 | 29680479388 | 33382478907 | 1.124728427 STRING | 1000 | 1000 | 32 | 35476213777 | 40478389269 | 1.141000827 LIST | 1000 | 0 | 1 | 6874253484 | 7370835717 | 1.072237987 LIST | 1000 | 1000 | 1 | 6763426009 | 7253762966 | 1.07249831 LIST | 1000 | 0 | 32 | 6981508808 | 7502741115 | 1.074658977 LIST | 1000 | 1000 | 32 | 6989374761 | 7506418252 | 1.073975643 STRUCT | 1000 | 0 | 1 | 2137525922 | 2189495762 | 1.024313081 STRUCT | 1000 | 1000 | 1 | 1057923939 | 1078475980 | 1.019426766 STRUCT | 1000 | 0 | 32 | 1637342446 | 1698913790 | 1.037604439 STRUCT | 1000 | 1000 | 32 | 1057587701 | 1082539399 | 1.02359303 Authors: - Rahul Prabhu (https://github.com/sdrp713) - Muhammad Haseeb (https://github.com/mhaseeb123) Approvers: - https://github.com/nvdbaranec - Muhammad Haseeb (https://github.com/mhaseeb123) - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16281 --- cpp/benchmarks/CMakeLists.txt | 5 + .../io/utilities/batched_memset_bench.cpp | 101 ++++++++++++++++++ cpp/include/cudf/io/detail/batched_memset.hpp | 82 ++++++++++++++ cpp/src/io/parquet/reader_impl_preprocess.cu | 29 ++++- cpp/src/io/utilities/column_buffer.cpp | 29 +++-- cpp/src/io/utilities/column_buffer.hpp | 23 +++- cpp/tests/CMakeLists.txt | 1 + .../utilities_tests/batched_memset_tests.cu | 97 +++++++++++++++++ 8 files changed, 353 insertions(+), 14 deletions(-) create mode 100644 cpp/benchmarks/io/utilities/batched_memset_bench.cpp create mode 100644 cpp/include/cudf/io/detail/batched_memset.hpp create mode 100644 cpp/tests/utilities_tests/batched_memset_tests.cu diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index ff431c7f260..7be456ddfba 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -353,6 +353,11 @@ ConfigureNVBench(JSON_READER_NVBENCH io/json/nested_json.cpp io/json/json_reader ConfigureNVBench(JSON_READER_OPTION_NVBENCH io/json/json_reader_option.cpp) ConfigureNVBench(JSON_WRITER_NVBENCH io/json/json_writer.cpp) +# ################################################################################################## +# * multi buffer memset benchmark +# ---------------------------------------------------------------------- +ConfigureNVBench(BATCHED_MEMSET_BENCH io/utilities/batched_memset_bench.cpp) + # ################################################################################################## # * io benchmark --------------------------------------------------------------------- ConfigureNVBench(MULTIBYTE_SPLIT_NVBENCH io/text/multibyte_split.cpp) diff --git a/cpp/benchmarks/io/utilities/batched_memset_bench.cpp b/cpp/benchmarks/io/utilities/batched_memset_bench.cpp new file mode 100644 index 00000000000..2905895a63b --- /dev/null +++ b/cpp/benchmarks/io/utilities/batched_memset_bench.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include + +// Size of the data in the benchmark dataframe; chosen to be low enough to allow benchmarks to +// run on most GPUs, but large enough to allow highest throughput +constexpr size_t data_size = 512 << 20; + +void parquet_read_common(cudf::size_type num_rows_to_read, + cudf::size_type num_cols_to_read, + cuio_source_sink_pair& source_sink, + nvbench::state& state) +{ + cudf::io::parquet_reader_options read_opts = + cudf::io::parquet_reader_options::builder(source_sink.make_source_info()); + + auto mem_stats_logger = cudf::memory_stats_logger(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); + state.exec( + nvbench::exec_tag::sync | nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) { + try_drop_l3_cache(); + + timer.start(); + auto const result = cudf::io::read_parquet(read_opts); + timer.stop(); + + CUDF_EXPECTS(result.tbl->num_columns() == num_cols_to_read, "Unexpected number of columns"); + CUDF_EXPECTS(result.tbl->num_rows() == num_rows_to_read, "Unexpected number of rows"); + }); + + auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); + state.add_element_count(static_cast(data_size) / time, "bytes_per_second"); + state.add_buffer_size( + mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); + state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); +} + +template +void bench_batched_memset(nvbench::state& state, nvbench::type_list>) +{ + auto const d_type = get_type_or_group(static_cast(DataType)); + auto const num_cols = static_cast(state.get_int64("num_cols")); + auto const cardinality = static_cast(state.get_int64("cardinality")); + auto const run_length = static_cast(state.get_int64("run_length")); + auto const source_type = retrieve_io_type_enum(state.get_string("io_type")); + auto const compression = cudf::io::compression_type::NONE; + cuio_source_sink_pair source_sink(source_type); + auto const tbl = + create_random_table(cycle_dtypes(d_type, num_cols), + table_size_bytes{data_size}, + data_profile_builder().cardinality(cardinality).avg_run_length(run_length)); + auto const view = tbl->view(); + + cudf::io::parquet_writer_options write_opts = + cudf::io::parquet_writer_options::builder(source_sink.make_sink_info(), view) + .compression(compression); + cudf::io::write_parquet(write_opts); + auto const num_rows = view.num_rows(); + + parquet_read_common(num_rows, num_cols, source_sink, state); +} + +using d_type_list = nvbench::enum_type_list; + +NVBENCH_BENCH_TYPES(bench_batched_memset, NVBENCH_TYPE_AXES(d_type_list)) + .set_name("batched_memset") + .set_type_axes_names({"data_type"}) + .add_int64_axis("num_cols", {1000}) + .add_string_axis("io_type", {"DEVICE_BUFFER"}) + .set_min_samples(4) + .add_int64_axis("cardinality", {0, 1000}) + .add_int64_axis("run_length", {1, 32}); diff --git a/cpp/include/cudf/io/detail/batched_memset.hpp b/cpp/include/cudf/io/detail/batched_memset.hpp new file mode 100644 index 00000000000..d0922cc64ee --- /dev/null +++ b/cpp/include/cudf/io/detail/batched_memset.hpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace CUDF_EXPORT cudf { +namespace io::detail { + +/** + * @brief A helper function that takes in a vector of device spans and memsets them to the + * value provided using batches sent to the GPU. + * + * @param bufs Vector with device spans of data + * @param value Value to memset all device spans to + * @param _stream Stream used for device memory operations and kernel launches + * + * @return The data in device spans all set to value + */ +template +void batched_memset(std::vector> const& bufs, + T const value, + rmm::cuda_stream_view stream) +{ + // define task and bytes parameters + auto const num_bufs = bufs.size(); + + // copy bufs into device memory and then get sizes + auto gpu_bufs = + cudf::detail::make_device_uvector_async(bufs, stream, rmm::mr::get_current_device_resource()); + + // get a vector with the sizes of all buffers + auto sizes = cudf::detail::make_counting_transform_iterator( + static_cast(0), + cuda::proclaim_return_type( + [gpu_bufs = gpu_bufs.data()] __device__(std::size_t i) { return gpu_bufs[i].size(); })); + + // get an iterator with a constant value to memset + auto iter_in = thrust::make_constant_iterator(thrust::make_constant_iterator(value)); + + // get an iterator pointing to each device span + auto iter_out = thrust::make_transform_iterator( + thrust::counting_iterator(0), + cuda::proclaim_return_type( + [gpu_bufs = gpu_bufs.data()] __device__(std::size_t i) { return gpu_bufs[i].data(); })); + + size_t temp_storage_bytes = 0; + + cub::DeviceCopy::Batched(nullptr, temp_storage_bytes, iter_in, iter_out, sizes, num_bufs, stream); + + rmm::device_buffer d_temp_storage( + temp_storage_bytes, stream, rmm::mr::get_current_device_resource()); + + cub::DeviceCopy::Batched( + d_temp_storage.data(), temp_storage_bytes, iter_in, iter_out, sizes, num_bufs, stream); +} + +} // namespace io::detail +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index e006cc7d714..557b1a45c1f 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -1494,6 +1495,11 @@ void reader::impl::allocate_columns(read_mode mode, size_t skip_rows, size_t num // buffers if they are not part of a list hierarchy. mark down // if we have any list columns that need further processing. bool has_lists = false; + // Casting to std::byte since data buffer pointer is void * + std::vector> memset_bufs; + // Validity Buffer is a uint32_t pointer + std::vector> nullmask_bufs; + for (size_t idx = 0; idx < _input_columns.size(); idx++) { auto const& input_col = _input_columns[idx]; size_t const max_depth = input_col.nesting_depth(); @@ -1514,13 +1520,19 @@ void reader::impl::allocate_columns(read_mode mode, size_t skip_rows, size_t num // we're going to start null mask as all valid and then turn bits off if necessary out_buf.create_with_mask( out_buf.type.id() == type_id::LIST && l_idx < max_depth ? num_rows + 1 : num_rows, - cudf::mask_state::ALL_VALID, + cudf::mask_state::UNINITIALIZED, + false, _stream, _mr); + memset_bufs.push_back(cudf::device_span(static_cast(out_buf.data()), + out_buf.data_size())); + nullmask_bufs.push_back(cudf::device_span( + out_buf.null_mask(), + cudf::util::round_up_safe(out_buf.null_mask_size(), sizeof(cudf::bitmask_type)) / + sizeof(cudf::bitmask_type))); } } } - // compute output column sizes by examining the pages of the -input- columns if (has_lists) { auto h_cols_info = @@ -1593,11 +1605,22 @@ void reader::impl::allocate_columns(read_mode mode, size_t skip_rows, size_t num // allocate // we're going to start null mask as all valid and then turn bits off if necessary - out_buf.create_with_mask(size, cudf::mask_state::ALL_VALID, _stream, _mr); + out_buf.create_with_mask(size, cudf::mask_state::UNINITIALIZED, false, _stream, _mr); + memset_bufs.push_back(cudf::device_span( + static_cast(out_buf.data()), out_buf.data_size())); + nullmask_bufs.push_back(cudf::device_span( + out_buf.null_mask(), + cudf::util::round_up_safe(out_buf.null_mask_size(), sizeof(cudf::bitmask_type)) / + sizeof(cudf::bitmask_type))); } } } } + + cudf::io::detail::batched_memset(memset_bufs, static_cast(0), _stream); + // Need to set null mask bufs to all high bits + cudf::io::detail::batched_memset( + nullmask_bufs, std::numeric_limits::max(), _stream); } std::vector reader::impl::calculate_page_string_offsets() diff --git a/cpp/src/io/utilities/column_buffer.cpp b/cpp/src/io/utilities/column_buffer.cpp index 2f4272b0367..8abfb000b94 100644 --- a/cpp/src/io/utilities/column_buffer.cpp +++ b/cpp/src/io/utilities/column_buffer.cpp @@ -33,7 +33,7 @@ namespace cudf::io::detail { -void gather_column_buffer::allocate_strings_data(rmm::cuda_stream_view stream) +void gather_column_buffer::allocate_strings_data(bool memset_data, rmm::cuda_stream_view stream) { CUDF_EXPECTS(type.id() == type_id::STRING, "allocate_strings_data called for non-string column"); // The contents of _strings will never be directly returned to the user. @@ -56,11 +56,12 @@ std::unique_ptr gather_column_buffer::make_string_column_impl(rmm::cuda_ return make_strings_column(*_strings, stream, _mr); } -void cudf::io::detail::inline_column_buffer::allocate_strings_data(rmm::cuda_stream_view stream) +void cudf::io::detail::inline_column_buffer::allocate_strings_data(bool memset_data, + rmm::cuda_stream_view stream) { CUDF_EXPECTS(type.id() == type_id::STRING, "allocate_strings_data called for non-string column"); // size + 1 for final offset. _string_data will be initialized later. - _data = create_data(data_type{type_id::INT32}, size + 1, stream, _mr); + _data = create_data(data_type{type_to_id()}, size + 1, memset_data, stream, _mr); } void cudf::io::detail::inline_column_buffer::create_string_data(size_t num_bytes, @@ -93,6 +94,7 @@ void copy_buffer_data(string_policy const& buff, string_policy& new_buff) template void column_buffer_base::create_with_mask(size_type _size, cudf::mask_state null_mask_state, + bool memset_data, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -100,16 +102,20 @@ void column_buffer_base::create_with_mask(size_type _size, _mr = mr; switch (type.id()) { - case type_id::STRING: static_cast(this)->allocate_strings_data(stream); break; + case type_id::STRING: + static_cast(this)->allocate_strings_data(memset_data, stream); + break; // list columns store a buffer of int32's as offsets to represent // their individual rows - case type_id::LIST: _data = create_data(data_type{type_id::INT32}, size, stream, _mr); break; + case type_id::LIST: + _data = create_data(data_type{type_to_id()}, size, memset_data, stream, _mr); + break; // struct columns store no data themselves. just validity and children. case type_id::STRUCT: break; - default: _data = create_data(type, size, stream, _mr); break; + default: _data = create_data(type, size, memset_data, stream, _mr); break; } if (is_nullable) { _null_mask = @@ -117,12 +123,21 @@ void column_buffer_base::create_with_mask(size_type _size, } } +template +void column_buffer_base::create(size_type _size, + bool memset_data, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + create_with_mask(_size, mask_state::ALL_NULL, memset_data, stream, mr); +} + template void column_buffer_base::create(size_type _size, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - create_with_mask(_size, mask_state::ALL_NULL, stream, mr); + create_with_mask(_size, mask_state::ALL_NULL, true, stream, mr); } template diff --git a/cpp/src/io/utilities/column_buffer.hpp b/cpp/src/io/utilities/column_buffer.hpp index ed6bb8bbdca..b2290965bb9 100644 --- a/cpp/src/io/utilities/column_buffer.hpp +++ b/cpp/src/io/utilities/column_buffer.hpp @@ -44,6 +44,7 @@ namespace detail { * * @param type The intended data type to populate * @param size The number of elements to be represented by the mask + * @param memset_data Defines whether data should be memset to 0 * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned device_buffer * @@ -51,17 +52,25 @@ namespace detail { */ inline rmm::device_buffer create_data(data_type type, size_type size, + bool memset_data, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { std::size_t data_size = size_of(type) * size; rmm::device_buffer data(data_size, stream, mr); - CUDF_CUDA_TRY(cudaMemsetAsync(data.data(), 0, data_size, stream.value())); - + if (memset_data) { CUDF_CUDA_TRY(cudaMemsetAsync(data.data(), 0, data_size, stream.value())); } return data; } +inline rmm::device_buffer create_data(data_type type, + size_type size, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + return create_data(type, size, true, stream, mr); +} + using string_index_pair = thrust::pair; // forward declare friend functions @@ -113,12 +122,18 @@ class column_buffer_base { // instantiate a column of known type with a specified size. Allows deferred creation for // preprocessing steps such as in the Parquet reader + void create(size_type _size, + bool memset_data, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + void create(size_type _size, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); // like create(), but also takes a `cudf::mask_state` to allow initializing the null mask as // something other than `ALL_NULL` void create_with_mask(size_type _size, cudf::mask_state null_mask_state, + bool memset_data, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); @@ -192,7 +207,7 @@ class gather_column_buffer : public column_buffer_base { create(_size, stream, mr); } - void allocate_strings_data(rmm::cuda_stream_view stream); + void allocate_strings_data(bool memset_data, rmm::cuda_stream_view stream); [[nodiscard]] void* data_impl() { return _strings ? _strings->data() : _data.data(); } [[nodiscard]] void const* data_impl() const { return _strings ? _strings->data() : _data.data(); } @@ -226,7 +241,7 @@ class inline_column_buffer : public column_buffer_base { create(_size, stream, mr); } - void allocate_strings_data(rmm::cuda_stream_view stream); + void allocate_strings_data(bool memset_data, rmm::cuda_stream_view stream); void* data_impl() { return _data.data(); } [[nodiscard]] void const* data_impl() const { return _data.data(); } diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 4dffcb41ba2..5e85b3e8adf 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -393,6 +393,7 @@ ConfigureTest( utilities_tests/pinned_memory_tests.cpp utilities_tests/type_check_tests.cpp utilities_tests/type_list_tests.cpp + utilities_tests/batched_memset_tests.cu ) # ################################################################################################## diff --git a/cpp/tests/utilities_tests/batched_memset_tests.cu b/cpp/tests/utilities_tests/batched_memset_tests.cu new file mode 100644 index 00000000000..9fc5baeec97 --- /dev/null +++ b/cpp/tests/utilities_tests/batched_memset_tests.cu @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include + +template +struct MultiBufferTestIntegral : public cudf::test::BaseFixture {}; + +TEST(MultiBufferTestIntegral, BasicTest1) +{ + std::vector const BUF_SIZES{ + 50000, 4, 1000, 0, 250000, 1, 100, 8000, 0, 1, 100, 1000, 10000, 100000, 0, 1, 100000}; + + // Device init + auto stream = cudf::get_default_stream(); + auto mr = rmm::mr::get_current_device_resource(); + + // Creating base vector for data and setting it to all 0xFF + std::vector> expected; + std::transform(BUF_SIZES.begin(), BUF_SIZES.end(), std::back_inserter(expected), [](auto size) { + return std::vector(size + 2000, std::numeric_limits::max()); + }); + + // set buffer region to other value + std::for_each(thrust::make_zip_iterator(thrust::make_tuple(expected.begin(), BUF_SIZES.begin())), + thrust::make_zip_iterator(thrust::make_tuple(expected.end(), BUF_SIZES.end())), + [](auto elem) { + std::fill_n( + thrust::get<0>(elem).begin() + 1000, thrust::get<1>(elem), 0xEEEEEEEEEEEEEEEE); + }); + + // Copy host vector data to device + std::vector> device_bufs; + std::transform(expected.begin(), + expected.end(), + std::back_inserter(device_bufs), + [stream, mr](auto const& vec) { + return cudf::detail::make_device_uvector_async(vec, stream, mr); + }); + + // Initialize device buffers for memset + std::vector> memset_bufs; + std::transform( + thrust::make_zip_iterator(thrust::make_tuple(device_bufs.begin(), BUF_SIZES.begin())), + thrust::make_zip_iterator(thrust::make_tuple(device_bufs.end(), BUF_SIZES.end())), + std::back_inserter(memset_bufs), + [](auto const& elem) { + return cudf::device_span(thrust::get<0>(elem).data() + 1000, thrust::get<1>(elem)); + }); + + // Function Call + cudf::io::detail::batched_memset(memset_bufs, uint64_t{0}, stream); + + // Set all buffer regions to 0 for expected comparison + std::for_each( + thrust::make_zip_iterator(thrust::make_tuple(expected.begin(), BUF_SIZES.begin())), + thrust::make_zip_iterator(thrust::make_tuple(expected.end(), BUF_SIZES.end())), + [](auto elem) { std::fill_n(thrust::get<0>(elem).begin() + 1000, thrust::get<1>(elem), 0UL); }); + + // Compare to see that only given buffers are zeroed out + std::for_each( + thrust::make_zip_iterator(thrust::make_tuple(device_bufs.begin(), expected.begin())), + thrust::make_zip_iterator(thrust::make_tuple(device_bufs.end(), expected.end())), + [stream](auto const& elem) { + auto after_memset = cudf::detail::make_std_vector_async(thrust::get<0>(elem), stream); + EXPECT_TRUE( + std::equal(thrust::get<1>(elem).begin(), thrust::get<1>(elem).end(), after_memset.begin())); + }); +} From 8068a2d616b6647bcd80720a2c24af858cbffd2d Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Mon, 5 Aug 2024 14:48:33 -0700 Subject: [PATCH 029/270] Fix build failures with GCC 13 (#16488) Closes #16395 This PR resolves two types of compilation errors, allowing for successful builds with GCC 13: - replacing the `cuco_allocator` strong type with an alias to fix a new build time check with GCC 13 - removing `std::move` when returning a temporary Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - David Wendt (https://github.com/davidwendt) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/16488 --- cpp/include/cudf/detail/cuco_helpers.hpp | 17 ++---- .../cudf/detail/distinct_hash_join.cuh | 2 +- .../cudf/detail/hash_reduce_by_row.cuh | 2 +- cpp/include/cudf/detail/join.hpp | 2 +- cpp/include/cudf_test/column_wrapper.hpp | 14 ++--- cpp/src/groupby/hash/groupby.cu | 19 +++---- cpp/src/io/json/json_tree.cu | 35 +++++++------ cpp/src/io/json/write_json.cu | 2 +- cpp/src/join/conditional_join.cu | 52 +++++++++---------- cpp/src/join/distinct_hash_join.cu | 2 +- cpp/src/join/hash_join.cu | 2 +- cpp/src/join/join_common_utils.hpp | 8 +-- cpp/src/join/mixed_join.cu | 22 ++++---- cpp/src/join/mixed_join_semi.cu | 11 ++-- cpp/src/reductions/histogram.cu | 12 +++-- cpp/src/search/contains_table.cu | 17 +++--- cpp/src/stream_compaction/distinct.cu | 19 +++---- cpp/src/stream_compaction/distinct_count.cu | 17 +++--- .../stream_compaction/distinct_helpers.hpp | 2 +- cpp/src/text/bpe/byte_pair_encoding.cuh | 4 +- cpp/src/text/bpe/load_merge_pairs.cu | 39 +++++++------- cpp/src/text/vocabulary_tokenize.cu | 4 +- cpp/tests/copying/gather_tests.cpp | 14 ++--- cpp/tests/reshape/byte_cast_tests.cpp | 22 ++++---- cpp/tests/structs/structs_column_tests.cpp | 48 ++++++++--------- 25 files changed, 195 insertions(+), 193 deletions(-) diff --git a/cpp/include/cudf/detail/cuco_helpers.hpp b/cpp/include/cudf/detail/cuco_helpers.hpp index dca5a39bece..926df921715 100644 --- a/cpp/include/cudf/detail/cuco_helpers.hpp +++ b/cpp/include/cudf/detail/cuco_helpers.hpp @@ -36,19 +36,10 @@ static double constexpr CUCO_DESIRED_LOAD_FACTOR = 0.5; * later expects a standard C++ `Allocator` interface. This allocator helper provides a simple way * to handle cuco memory allocation/deallocation with the given `stream` and the rmm default memory * resource. + * + * @tparam T The allocator's value type. */ -class cuco_allocator - : public rmm::mr::stream_allocator_adaptor> { - /// Default stream-ordered allocator type - using default_allocator = rmm::mr::polymorphic_allocator; - /// The base allocator adaptor type - using base_type = rmm::mr::stream_allocator_adaptor; - - public: - /** - * @brief Constructs the allocator adaptor with the given `stream` - */ - cuco_allocator(rmm::cuda_stream_view stream) : base_type{default_allocator{}, stream} {} -}; +template +using cuco_allocator = rmm::mr::stream_allocator_adaptor>; } // namespace cudf::detail diff --git a/cpp/include/cudf/detail/distinct_hash_join.cuh b/cpp/include/cudf/detail/distinct_hash_join.cuh index c3bc3ad89fa..0b3d7ac58bf 100644 --- a/cpp/include/cudf/detail/distinct_hash_join.cuh +++ b/cpp/include/cudf/detail/distinct_hash_join.cuh @@ -99,7 +99,7 @@ struct distinct_hash_join { cuda::thread_scope_device, comparator_adapter, probing_scheme_type, - cudf::detail::cuco_allocator, + cudf::detail::cuco_allocator, cuco_storage_type>; bool _has_nulls; ///< true if nulls are present in either build table or probe table diff --git a/cpp/include/cudf/detail/hash_reduce_by_row.cuh b/cpp/include/cudf/detail/hash_reduce_by_row.cuh index dfe79646167..7a1e38eefe0 100644 --- a/cpp/include/cudf/detail/hash_reduce_by_row.cuh +++ b/cpp/include/cudf/detail/hash_reduce_by_row.cuh @@ -32,7 +32,7 @@ namespace cudf::detail { using hash_map_type = cuco::legacy:: - static_map; + static_map>; /** * @brief The base struct for customized reduction functor to perform reduce-by-key with keys are diff --git a/cpp/include/cudf/detail/join.hpp b/cpp/include/cudf/detail/join.hpp index ff7da4462a2..af46dd79cdb 100644 --- a/cpp/include/cudf/detail/join.hpp +++ b/cpp/include/cudf/detail/join.hpp @@ -59,7 +59,7 @@ struct hash_join { cuco::static_multimap, cuco::legacy::double_hashing>; hash_join() = delete; diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp index 4e504ec1d30..d00db222b62 100644 --- a/cpp/include/cudf_test/column_wrapper.hpp +++ b/cpp/include/cudf_test/column_wrapper.hpp @@ -1337,7 +1337,7 @@ class lists_column_wrapper : public detail::column_wrapper { lists_column_wrapper(std::initializer_list elements) : column_wrapper{} { build_from_non_nested( - std::move(cudf::test::fixed_width_column_wrapper(elements).release())); + cudf::test::fixed_width_column_wrapper(elements).release()); } /** @@ -1361,7 +1361,7 @@ class lists_column_wrapper : public detail::column_wrapper { lists_column_wrapper(InputIterator begin, InputIterator end) : column_wrapper{} { build_from_non_nested( - std::move(cudf::test::fixed_width_column_wrapper(begin, end).release())); + cudf::test::fixed_width_column_wrapper(begin, end).release()); } /** @@ -1386,7 +1386,7 @@ class lists_column_wrapper : public detail::column_wrapper { : column_wrapper{} { build_from_non_nested( - std::move(cudf::test::fixed_width_column_wrapper(elements, v).release())); + cudf::test::fixed_width_column_wrapper(elements, v).release()); } /** @@ -1413,8 +1413,8 @@ class lists_column_wrapper : public detail::column_wrapper { lists_column_wrapper(InputIterator begin, InputIterator end, ValidityIterator v) : column_wrapper{} { - build_from_non_nested(std::move( - cudf::test::fixed_width_column_wrapper(begin, end, v).release())); + build_from_non_nested( + cudf::test::fixed_width_column_wrapper(begin, end, v).release()); } /** @@ -1435,7 +1435,7 @@ class lists_column_wrapper : public detail::column_wrapper { lists_column_wrapper(std::initializer_list elements) : column_wrapper{} { build_from_non_nested( - std::move(cudf::test::strings_column_wrapper(elements.begin(), elements.end()).release())); + cudf::test::strings_column_wrapper(elements.begin(), elements.end()).release()); } /** @@ -1460,7 +1460,7 @@ class lists_column_wrapper : public detail::column_wrapper { : column_wrapper{} { build_from_non_nested( - std::move(cudf::test::strings_column_wrapper(elements.begin(), elements.end(), v).release())); + cudf::test::strings_column_wrapper(elements.begin(), elements.end(), v).release()); } /** diff --git a/cpp/src/groupby/hash/groupby.cu b/cpp/src/groupby/hash/groupby.cu index 5fe4a5eb30f..35161eada28 100644 --- a/cpp/src/groupby/hash/groupby.cu +++ b/cpp/src/groupby/hash/groupby.cu @@ -568,15 +568,16 @@ std::unique_ptr
groupby(table_view const& keys, cudf::detail::result_cache sparse_results(requests.size()); auto const comparator_helper = [&](auto const d_key_equal) { - auto const set = cuco::static_set{num_keys, - 0.5, // desired load factor - cuco::empty_key{cudf::detail::CUDF_SIZE_TYPE_SENTINEL}, - d_key_equal, - probing_scheme_type{d_row_hash}, - cuco::thread_scope_device, - cuco::storage<1>{}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto const set = cuco::static_set{ + num_keys, + 0.5, // desired load factor + cuco::empty_key{cudf::detail::CUDF_SIZE_TYPE_SENTINEL}, + d_key_equal, + probing_scheme_type{d_row_hash}, + cuco::thread_scope_device, + cuco::storage<1>{}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; // Compute all single pass aggs first compute_single_pass_aggs(keys, diff --git a/cpp/src/io/json/json_tree.cu b/cpp/src/io/json/json_tree.cu index ad807b57766..ee6bc0b9f4b 100644 --- a/cpp/src/io/json/json_tree.cu +++ b/cpp/src/io/json/json_tree.cu @@ -545,15 +545,15 @@ rmm::device_uvector hash_node_type_with_field_name(device_span{d_hasher}, - {}, - {}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto key_set = cuco::static_set{ + cuco::extent{compute_hash_table_size(num_fields, 40)}, // 40% occupancy + cuco::empty_key{empty_node_index_sentinel}, + d_equal, + cuco::linear_probing<1, hasher_type>{d_hasher}, + {}, + {}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; key_set.insert_if_async(iter, iter + num_nodes, thrust::counting_iterator(0), // stencil @@ -734,14 +734,15 @@ std::pair, rmm::device_uvector> hash_n constexpr size_type empty_node_index_sentinel = -1; using hasher_type = decltype(d_hashed_cache); - auto key_set = cuco::static_set{cuco::extent{compute_hash_table_size(num_nodes)}, - cuco::empty_key{empty_node_index_sentinel}, - d_equal, - cuco::linear_probing<1, hasher_type>{d_hashed_cache}, - {}, - {}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto key_set = cuco::static_set{ + cuco::extent{compute_hash_table_size(num_nodes)}, + cuco::empty_key{empty_node_index_sentinel}, + d_equal, + cuco::linear_probing<1, hasher_type>{d_hashed_cache}, + {}, + {}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; // insert and convert node ids to unique set ids auto nodes_itr = thrust::make_counting_iterator(0); diff --git a/cpp/src/io/json/write_json.cu b/cpp/src/io/json/write_json.cu index c688c809e04..60bb2366e87 100644 --- a/cpp/src/io/json/write_json.cu +++ b/cpp/src/io/json/write_json.cu @@ -649,7 +649,7 @@ struct column_to_strings_fn { auto const list_child_string = make_lists_column( column.size(), std::move(new_offsets), - std::move(child_string_with_null()), + child_string_with_null(), column.null_count(), cudf::detail::copy_bitmask(column, stream_, rmm::mr::get_current_device_resource()), stream_); diff --git a/cpp/src/join/conditional_join.cu b/cpp/src/join/conditional_join.cu index d4ef2747c9d..789702ce538 100644 --- a/cpp/src/join/conditional_join.cu +++ b/cpp/src/join/conditional_join.cu @@ -432,13 +432,13 @@ std::unique_ptr> conditional_left_semi_join( rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return std::move(detail::conditional_join_anti_semi(left, - right, - binary_predicate, - detail::join_kind::LEFT_SEMI_JOIN, - output_size, - cudf::get_default_stream(), - mr)); + return detail::conditional_join_anti_semi(left, + right, + binary_predicate, + detail::join_kind::LEFT_SEMI_JOIN, + output_size, + cudf::get_default_stream(), + mr); } std::unique_ptr> conditional_left_anti_join( @@ -449,13 +449,13 @@ std::unique_ptr> conditional_left_anti_join( rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return std::move(detail::conditional_join_anti_semi(left, - right, - binary_predicate, - detail::join_kind::LEFT_ANTI_JOIN, - output_size, - cudf::get_default_stream(), - mr)); + return detail::conditional_join_anti_semi(left, + right, + binary_predicate, + detail::join_kind::LEFT_ANTI_JOIN, + output_size, + cudf::get_default_stream(), + mr); } std::size_t conditional_inner_join_size(table_view const& left, @@ -484,12 +484,12 @@ std::size_t conditional_left_semi_join_size(table_view const& left, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return std::move(detail::compute_conditional_join_output_size(left, - right, - binary_predicate, - detail::join_kind::LEFT_SEMI_JOIN, - cudf::get_default_stream(), - mr)); + return detail::compute_conditional_join_output_size(left, + right, + binary_predicate, + detail::join_kind::LEFT_SEMI_JOIN, + cudf::get_default_stream(), + mr); } std::size_t conditional_left_anti_join_size(table_view const& left, @@ -498,12 +498,12 @@ std::size_t conditional_left_anti_join_size(table_view const& left, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return std::move(detail::compute_conditional_join_output_size(left, - right, - binary_predicate, - detail::join_kind::LEFT_ANTI_JOIN, - cudf::get_default_stream(), - mr)); + return detail::compute_conditional_join_output_size(left, + right, + binary_predicate, + detail::join_kind::LEFT_ANTI_JOIN, + cudf::get_default_stream(), + mr); } } // namespace cudf diff --git a/cpp/src/join/distinct_hash_join.cu b/cpp/src/join/distinct_hash_join.cu index daa1bf17c0d..3d95b0c5a5c 100644 --- a/cpp/src/join/distinct_hash_join.cu +++ b/cpp/src/join/distinct_hash_join.cu @@ -119,7 +119,7 @@ distinct_hash_join::distinct_hash_join(cudf::table_view const& build, {}, cuco::thread_scope_device, cuco_storage_type{}, - cudf::detail::cuco_allocator{stream}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, stream.value()} { CUDF_FUNC_RANGE(); diff --git a/cpp/src/join/hash_join.cu b/cpp/src/join/hash_join.cu index eb9b687630b..5d01482f44a 100644 --- a/cpp/src/join/hash_join.cu +++ b/cpp/src/join/hash_join.cu @@ -374,7 +374,7 @@ hash_join::hash_join(cudf::table_view const& build, cuco::empty_key{std::numeric_limits::max()}, cuco::empty_value{cudf::detail::JoinNoneValue}, stream.value(), - cudf::detail::cuco_allocator{stream}}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}}, _build{build}, _preprocessed_build{ cudf::experimental::row::equality::preprocessed_table::create(_build, stream)} diff --git a/cpp/src/join/join_common_utils.hpp b/cpp/src/join/join_common_utils.hpp index 4157100b67e..86402a0e7de 100644 --- a/cpp/src/join/join_common_utils.hpp +++ b/cpp/src/join/join_common_utils.hpp @@ -48,11 +48,13 @@ using mixed_multimap_type = cuco::static_multimap, cuco::legacy::double_hashing<1, hash_type, hash_type>>; -using semi_map_type = cuco::legacy:: - static_map; +using semi_map_type = cuco::legacy::static_map>; using row_hash_legacy = cudf::row_hasher; diff --git a/cpp/src/join/mixed_join.cu b/cpp/src/join/mixed_join.cu index 90748e6f322..48b94c777de 100644 --- a/cpp/src/join/mixed_join.cu +++ b/cpp/src/join/mixed_join.cu @@ -126,11 +126,12 @@ mixed_join( auto build_view = table_device_view::create(build, stream); // Don't use multimap_type because we want a CG size of 1. - mixed_multimap_type hash_table{compute_hash_table_size(build.num_rows()), - cuco::empty_key{std::numeric_limits::max()}, - cuco::empty_value{cudf::detail::JoinNoneValue}, - stream.value(), - cudf::detail::cuco_allocator{stream}}; + mixed_multimap_type hash_table{ + compute_hash_table_size(build.num_rows()), + cuco::empty_key{std::numeric_limits::max()}, + cuco::empty_value{cudf::detail::JoinNoneValue}, + stream.value(), + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}}; // TODO: To add support for nested columns we will need to flatten in many // places. However, this probably isn't worth adding any time soon since we @@ -391,11 +392,12 @@ compute_mixed_join_output_size(table_view const& left_equality, auto build_view = table_device_view::create(build, stream); // Don't use multimap_type because we want a CG size of 1. - mixed_multimap_type hash_table{compute_hash_table_size(build.num_rows()), - cuco::empty_key{std::numeric_limits::max()}, - cuco::empty_value{cudf::detail::JoinNoneValue}, - stream.value(), - cudf::detail::cuco_allocator{stream}}; + mixed_multimap_type hash_table{ + compute_hash_table_size(build.num_rows()), + cuco::empty_key{std::numeric_limits::max()}, + cuco::empty_value{cudf::detail::JoinNoneValue}, + stream.value(), + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}}; // TODO: To add support for nested columns we will need to flatten in many // places. However, this probably isn't worth adding any time soon since we diff --git a/cpp/src/join/mixed_join_semi.cu b/cpp/src/join/mixed_join_semi.cu index c147ea3c253..3e4188a0fbd 100644 --- a/cpp/src/join/mixed_join_semi.cu +++ b/cpp/src/join/mixed_join_semi.cu @@ -163,11 +163,12 @@ std::unique_ptr> mixed_join_semi( cudf::experimental::row::equality::two_table_comparator{preprocessed_probe, preprocessed_build}; auto const equality_probe = row_comparator.equal_to(has_nulls, compare_nulls); - semi_map_type hash_table{compute_hash_table_size(build.num_rows()), - cuco::empty_key{std::numeric_limits::max()}, - cuco::empty_value{cudf::detail::JoinNoneValue}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + semi_map_type hash_table{ + compute_hash_table_size(build.num_rows()), + cuco::empty_key{std::numeric_limits::max()}, + cuco::empty_value{cudf::detail::JoinNoneValue}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; // Create hash table containing all keys found in right table // TODO: To add support for nested columns we will need to flatten in many diff --git a/cpp/src/reductions/histogram.cu b/cpp/src/reductions/histogram.cu index bebb9d14923..d49c0c6f0d2 100644 --- a/cpp/src/reductions/histogram.cu +++ b/cpp/src/reductions/histogram.cu @@ -164,11 +164,13 @@ compute_row_frequencies(table_view const& input, "Nested types are not yet supported in histogram aggregation.", std::invalid_argument); - auto map = cudf::detail::hash_map_type{compute_hash_table_size(input.num_rows()), - cuco::empty_key{-1}, - cuco::empty_value{std::numeric_limits::min()}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto map = cudf::detail::hash_map_type{ + compute_hash_table_size(input.num_rows()), + cuco::empty_key{-1}, + cuco::empty_value{std::numeric_limits::min()}, + + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; auto const preprocessed_input = cudf::experimental::row::hash::preprocessed_table::create(input, stream); diff --git a/cpp/src/search/contains_table.cu b/cpp/src/search/contains_table.cu index 81227cb9a2d..66cefd0aa2f 100644 --- a/cpp/src/search/contains_table.cu +++ b/cpp/src/search/contains_table.cu @@ -229,14 +229,15 @@ rmm::device_uvector contains(table_view const& haystack, [&](auto const& d_self_equal, auto const& d_two_table_equal, auto const& probing_scheme) { auto const d_equal = comparator_adapter{d_self_equal, d_two_table_equal}; - auto set = cuco::static_set{cuco::extent{compute_hash_table_size(haystack.num_rows())}, - cuco::empty_key{rhs_index_type{-1}}, - d_equal, - probing_scheme, - {}, - {}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto set = cuco::static_set{ + cuco::extent{compute_hash_table_size(haystack.num_rows())}, + cuco::empty_key{rhs_index_type{-1}}, + d_equal, + probing_scheme, + {}, + {}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; if (haystack_has_nulls && compare_nulls == null_equality::UNEQUAL) { auto const bitmask_buffer_and_ptr = build_row_bitmask(haystack, stream); diff --git a/cpp/src/stream_compaction/distinct.cu b/cpp/src/stream_compaction/distinct.cu index e2c5aba6802..6afd6e34c50 100644 --- a/cpp/src/stream_compaction/distinct.cu +++ b/cpp/src/stream_compaction/distinct.cu @@ -97,15 +97,16 @@ rmm::device_uvector distinct_indices(table_view const& input, auto const helper_func = [&](auto const& d_equal) { using RowHasher = std::decay_t; - auto set = hash_set_type{num_rows, - 0.5, // desired load factor - cuco::empty_key{cudf::detail::CUDF_SIZE_TYPE_SENTINEL}, - d_equal, - {row_hash.device_hasher(has_nulls)}, - {}, - {}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto set = hash_set_type{ + num_rows, + 0.5, // desired load factor + cuco::empty_key{cudf::detail::CUDF_SIZE_TYPE_SENTINEL}, + d_equal, + {row_hash.device_hasher(has_nulls)}, + {}, + {}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; return detail::reduce_by_row(set, num_rows, keep, stream, mr); }; diff --git a/cpp/src/stream_compaction/distinct_count.cu b/cpp/src/stream_compaction/distinct_count.cu index 9843bb889f4..cdf9faddf31 100644 --- a/cpp/src/stream_compaction/distinct_count.cu +++ b/cpp/src/stream_compaction/distinct_count.cu @@ -141,14 +141,15 @@ cudf::size_type distinct_count(table_view const& keys, auto const comparator_helper = [&](auto const row_equal) { using hasher_type = decltype(hash_key); - auto key_set = cuco::static_set{cuco::extent{compute_hash_table_size(num_rows)}, - cuco::empty_key{-1}, - row_equal, - cuco::linear_probing<1, hasher_type>{hash_key}, - {}, - {}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto key_set = cuco::static_set{ + cuco::extent{compute_hash_table_size(num_rows)}, + cuco::empty_key{-1}, + row_equal, + cuco::linear_probing<1, hasher_type>{hash_key}, + {}, + {}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()}; auto const iter = thrust::counting_iterator(0); // when nulls are equal, we skip hashing any row that has a null diff --git a/cpp/src/stream_compaction/distinct_helpers.hpp b/cpp/src/stream_compaction/distinct_helpers.hpp index fca67c98873..bea02e3dbe8 100644 --- a/cpp/src/stream_compaction/distinct_helpers.hpp +++ b/cpp/src/stream_compaction/distinct_helpers.hpp @@ -57,7 +57,7 @@ using hash_set_type = cudf::experimental::row::hash::device_row_hasher< cudf::hashing::detail::default_hash, cudf::nullate::DYNAMIC>>, - cudf::detail::cuco_allocator, + cudf::detail::cuco_allocator, cuco::storage<1>>; /** diff --git a/cpp/src/text/bpe/byte_pair_encoding.cuh b/cpp/src/text/bpe/byte_pair_encoding.cuh index a2e441c3284..69c77224eb7 100644 --- a/cpp/src/text/bpe/byte_pair_encoding.cuh +++ b/cpp/src/text/bpe/byte_pair_encoding.cuh @@ -106,7 +106,7 @@ using merge_pairs_map_type = cuco::static_map, cuco_storage>; /** @@ -164,7 +164,7 @@ using mp_table_map_type = cuco::static_map, cuco_storage>; } // namespace detail diff --git a/cpp/src/text/bpe/load_merge_pairs.cu b/cpp/src/text/bpe/load_merge_pairs.cu index f34c5c4f7f6..9fb86aecce3 100644 --- a/cpp/src/text/bpe/load_merge_pairs.cu +++ b/cpp/src/text/bpe/load_merge_pairs.cu @@ -43,16 +43,16 @@ namespace { std::unique_ptr initialize_merge_pairs_map( cudf::column_device_view const& input, rmm::cuda_stream_view stream) { - auto merge_pairs_map = - std::make_unique(static_cast(input.size()), - cuco::empty_key{-1}, - cuco::empty_value{-1}, - bpe_equal{input}, - bpe_probe_scheme{bpe_hasher{input}}, - cuco::thread_scope_device, - cuco_storage{}, - cudf::detail::cuco_allocator{stream}, - stream.value()); + auto merge_pairs_map = std::make_unique( + static_cast(input.size()), + cuco::empty_key{-1}, + cuco::empty_value{-1}, + bpe_equal{input}, + bpe_probe_scheme{bpe_hasher{input}}, + cuco::thread_scope_device, + cuco_storage{}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()); auto iter = cudf::detail::make_counting_transform_iterator( 0, @@ -67,15 +67,16 @@ std::unique_ptr initialize_merge_pairs_map( std::unique_ptr initialize_mp_table_map( cudf::column_device_view const& input, rmm::cuda_stream_view stream) { - auto mp_table_map = std::make_unique(static_cast(input.size()), - cuco::empty_key{-1}, - cuco::empty_value{-1}, - mp_equal{input}, - mp_probe_scheme{mp_hasher{input}}, - cuco::thread_scope_device, - cuco_storage{}, - cudf::detail::cuco_allocator{stream}, - stream.value()); + auto mp_table_map = std::make_unique( + static_cast(input.size()), + cuco::empty_key{-1}, + cuco::empty_value{-1}, + mp_equal{input}, + mp_probe_scheme{mp_hasher{input}}, + cuco::thread_scope_device, + cuco_storage{}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, + stream.value()); auto iter = cudf::detail::make_counting_transform_iterator( 0, diff --git a/cpp/src/text/vocabulary_tokenize.cu b/cpp/src/text/vocabulary_tokenize.cu index 97abb1487d8..5945921ed9d 100644 --- a/cpp/src/text/vocabulary_tokenize.cu +++ b/cpp/src/text/vocabulary_tokenize.cu @@ -100,7 +100,7 @@ using vocabulary_map_type = cuco::static_map, cuco_storage>; } // namespace } // namespace detail @@ -152,7 +152,7 @@ tokenize_vocabulary::tokenize_vocabulary(cudf::strings_column_view const& input, detail::probe_scheme{detail::vocab_hasher{*d_vocabulary}}, cuco::thread_scope_device, detail::cuco_storage{}, - cudf::detail::cuco_allocator{stream}, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}, stream.value()); // the row index is the token id (value for each key in the map) diff --git a/cpp/tests/copying/gather_tests.cpp b/cpp/tests/copying/gather_tests.cpp index 284b6c4c50c..07ce672b14d 100644 --- a/cpp/tests/copying/gather_tests.cpp +++ b/cpp/tests/copying/gather_tests.cpp @@ -43,7 +43,7 @@ TYPED_TEST(GatherTest, IdentityTest) cudf::table_view source_table({source_column}); - std::unique_ptr result = std::move(cudf::gather(source_table, gather_map)); + std::unique_ptr result = cudf::gather(source_table, gather_map); for (auto i = 0; i < source_table.num_columns(); ++i) { CUDF_TEST_EXPECT_COLUMNS_EQUAL(source_table.column(i), result->view().column(i)); @@ -66,7 +66,7 @@ TYPED_TEST(GatherTest, ReverseIdentityTest) cudf::table_view source_table({source_column}); - std::unique_ptr result = std::move(cudf::gather(source_table, gather_map)); + std::unique_ptr result = cudf::gather(source_table, gather_map); cudf::test::fixed_width_column_wrapper expect_column(reversed_data, reversed_data + source_size); @@ -94,7 +94,7 @@ TYPED_TEST(GatherTest, EveryOtherNullOdds) cudf::table_view source_table({source_column}); - std::unique_ptr result = std::move(cudf::gather(source_table, gather_map)); + std::unique_ptr result = cudf::gather(source_table, gather_map); auto expect_data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return 0; }); auto expect_valid = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return 0; }); @@ -126,7 +126,7 @@ TYPED_TEST(GatherTest, EveryOtherNullEvens) cudf::table_view source_table({source_column}); - std::unique_ptr result = std::move(cudf::gather(source_table, gather_map)); + std::unique_ptr result = cudf::gather(source_table, gather_map); auto expect_data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i * 2 + 1; }); @@ -160,7 +160,7 @@ TYPED_TEST(GatherTest, AllNull) cudf::table_view source_table({source_column}); - std::unique_ptr result = std::move(cudf::gather(source_table, gather_map)); + std::unique_ptr result = cudf::gather(source_table, gather_map); // Check that the result is also all invalid CUDF_TEST_EXPECT_TABLES_EQUAL(source_table, result->view()); @@ -190,7 +190,7 @@ TYPED_TEST(GatherTest, MultiColReverseIdentityTest) cudf::table_view source_table{source_columns}; - std::unique_ptr result = std::move(cudf::gather(source_table, gather_map)); + std::unique_ptr result = cudf::gather(source_table, gather_map); cudf::test::fixed_width_column_wrapper expect_column(reversed_data, reversed_data + source_size); @@ -228,7 +228,7 @@ TYPED_TEST(GatherTest, MultiColNulls) cudf::table_view source_table{source_columns}; - std::unique_ptr result = std::move(cudf::gather(source_table, gather_map)); + std::unique_ptr result = cudf::gather(source_table, gather_map); // Expected data auto expect_data = diff --git a/cpp/tests/reshape/byte_cast_tests.cpp b/cpp/tests/reshape/byte_cast_tests.cpp index cd280302677..b3d9b2e2f5f 100644 --- a/cpp/tests/reshape/byte_cast_tests.cpp +++ b/cpp/tests/reshape/byte_cast_tests.cpp @@ -61,8 +61,8 @@ TEST_F(ByteCastTest, int16ValuesWithNulls) auto [null_mask, null_count] = cudf::test::detail::make_null_mask(odd_validity, odd_validity + 5); auto int16_expected = cudf::make_lists_column( 5, - std::move(cudf::test::fixed_width_column_wrapper{0, 0, 2, 2, 4, 4}.release()), - std::move(int16_data.release()), + cudf::test::fixed_width_column_wrapper{0, 0, 2, 2, 4, 4}.release(), + int16_data.release(), null_count, std::move(null_mask)); @@ -109,8 +109,8 @@ TEST_F(ByteCastTest, int32ValuesWithNulls) auto int32_expected = cudf::make_lists_column( 5, - std::move(cudf::test::fixed_width_column_wrapper{0, 4, 4, 8, 8, 12}.release()), - std::move(int32_data.release()), + cudf::test::fixed_width_column_wrapper{0, 4, 4, 8, 8, 12}.release(), + int32_data.release(), null_count, std::move(null_mask)); @@ -163,9 +163,8 @@ TEST_F(ByteCastTest, int64ValuesWithNulls) auto [null_mask, null_count] = cudf::test::detail::make_null_mask(odd_validity, odd_validity + 5); auto int64_expected = cudf::make_lists_column( 5, - std::move( - cudf::test::fixed_width_column_wrapper{0, 0, 8, 8, 16, 16}.release()), - std::move(int64_data.release()), + cudf::test::fixed_width_column_wrapper{0, 0, 8, 8, 16, 16}.release(), + int64_data.release(), null_count, std::move(null_mask)); @@ -226,8 +225,8 @@ TEST_F(ByteCastTest, fp32ValuesWithNulls) cudf::test::detail::make_null_mask(even_validity, even_validity + 5); auto fp32_expected = cudf::make_lists_column( 5, - std::move(cudf::test::fixed_width_column_wrapper{0, 4, 4, 8, 8, 12}.release()), - std::move(fp32_data.release()), + cudf::test::fixed_width_column_wrapper{0, 4, 4, 8, 8, 12}.release(), + fp32_data.release(), null_count, std::move(null_mask)); @@ -297,9 +296,8 @@ TEST_F(ByteCastTest, fp64ValuesWithNulls) auto [null_mask, null_count] = cudf::test::detail::make_null_mask(odd_validity, odd_validity + 5); auto fp64_expected = cudf::make_lists_column( 5, - std::move( - cudf::test::fixed_width_column_wrapper{0, 0, 8, 8, 16, 16}.release()), - std::move(fp64_data.release()), + cudf::test::fixed_width_column_wrapper{0, 0, 8, 8, 16, 16}.release(), + fp64_data.release(), null_count, std::move(null_mask)); diff --git a/cpp/tests/structs/structs_column_tests.cpp b/cpp/tests/structs/structs_column_tests.cpp index df005dfa1dc..f0010fc1ed9 100644 --- a/cpp/tests/structs/structs_column_tests.cpp +++ b/cpp/tests/structs/structs_column_tests.cpp @@ -448,12 +448,12 @@ TYPED_TEST(TypedStructColumnWrapperTest, ListOfStructOfList) cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 3; }); auto [null_mask, null_count] = detail::make_null_mask(list_of_struct_of_list_validity, list_of_struct_of_list_validity + 5); - auto list_of_struct_of_list = cudf::make_lists_column( - 5, - std::move(fixed_width_column_wrapper{0, 2, 4, 6, 8, 10}.release()), - std::move(struct_of_lists_col), - null_count, - std::move(null_mask)); + auto list_of_struct_of_list = + cudf::make_lists_column(5, + fixed_width_column_wrapper{0, 2, 4, 6, 8, 10}.release(), + std::move(struct_of_lists_col), + null_count, + std::move(null_mask)); // Compare with expected values. @@ -468,12 +468,12 @@ TYPED_TEST(TypedStructColumnWrapperTest, ListOfStructOfList) std::tie(null_mask, null_count) = detail::make_null_mask(list_of_struct_of_list_validity, list_of_struct_of_list_validity + 5); - auto expected_level3_list = cudf::make_lists_column( - 5, - std::move(fixed_width_column_wrapper{0, 0, 2, 4, 4, 6}.release()), - std::move(expected_level2_struct), - null_count, - std::move(null_mask)); + auto expected_level3_list = + cudf::make_lists_column(5, + fixed_width_column_wrapper{0, 0, 2, 4, 4, 6}.release(), + std::move(expected_level2_struct), + null_count, + std::move(null_mask)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*list_of_struct_of_list, *expected_level3_list); } @@ -498,12 +498,12 @@ TYPED_TEST(TypedStructColumnWrapperTest, StructOfListOfStruct) cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 3; }); auto [null_mask, null_count] = detail::make_null_mask(list_validity, list_validity + 5); - auto lists_col = cudf::make_lists_column( - 5, - std::move(fixed_width_column_wrapper{0, 2, 4, 6, 8, 10}.release()), - std::move(structs_col), - null_count, - std::move(null_mask)); + auto lists_col = + cudf::make_lists_column(5, + fixed_width_column_wrapper{0, 2, 4, 6, 8, 10}.release(), + std::move(structs_col), + null_count, + std::move(null_mask)); std::vector> cols; cols.push_back(std::move(lists_col)); @@ -519,12 +519,12 @@ TYPED_TEST(TypedStructColumnWrapperTest, StructOfListOfStruct) std::tie(null_mask, null_count) = detail::make_null_mask(list_validity, list_validity + 5); - auto expected_lists_col = cudf::make_lists_column( - 5, - std::move(fixed_width_column_wrapper{0, 2, 4, 6, 8, 10}.release()), - std::move(expected_structs_col), - null_count, - std::move(null_mask)); + auto expected_lists_col = + cudf::make_lists_column(5, + fixed_width_column_wrapper{0, 2, 4, 6, 8, 10}.release(), + std::move(expected_structs_col), + null_count, + std::move(null_mask)); // Test that the lists child column is as expected. CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*expected_lists_col, struct_of_list_of_struct->child(0)); From e8156d42163fb02aa90baba9be20ab89bc9ebef1 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 6 Aug 2024 17:03:10 -0400 Subject: [PATCH 030/270] Fix segmented-sort overlapped input/output indices (#16463) Fixes call to CUB `DeviceSegmentedSort::SortPairs` where the input and output indices pointed to the same temp memory. The documentation from https://nvidia.github.io/cccl/cub/api/structcub_1_1DeviceSegmentedSort.html#id8 indicates the `d_values_in` and `d_values_out` memory must not overlap so using the same pointer for both created invalid output in certain conditions. The internal function was implemented to expect the input values to be updated in-place. The fix uses separate device memory for the input and output indices. Closes #16455 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/16463 --- cpp/src/sort/segmented_sort_impl.cuh | 4 +++- cpp/tests/sort/segmented_sort_tests.cpp | 26 ++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cpp/src/sort/segmented_sort_impl.cuh b/cpp/src/sort/segmented_sort_impl.cuh index 6d472925b30..281fdfa6b8f 100644 --- a/cpp/src/sort/segmented_sort_impl.cuh +++ b/cpp/src/sort/segmented_sort_impl.cuh @@ -79,6 +79,8 @@ struct column_fast_sort_fn { stream, rmm::mr::get_current_device_resource()); mutable_column_view output_view = temp_col->mutable_view(); + auto temp_indices = cudf::column( + cudf::column_view(indices.type(), indices.size(), indices.head(), nullptr, 0), stream); // DeviceSegmentedSort is faster than DeviceSegmentedRadixSort at this time auto fast_sort_impl = [stream](bool ascending, [[maybe_unused]] auto&&... args) { @@ -118,7 +120,7 @@ struct column_fast_sort_fn { fast_sort_impl(ascending, input.begin(), output_view.begin(), - indices.begin(), + temp_indices.view().begin(), indices.begin(), input.size(), segment_offsets.size() - 1, diff --git a/cpp/tests/sort/segmented_sort_tests.cpp b/cpp/tests/sort/segmented_sort_tests.cpp index da9666cbc74..f4fe2c5956a 100644 --- a/cpp/tests/sort/segmented_sort_tests.cpp +++ b/cpp/tests/sort/segmented_sort_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include #include @@ -338,3 +340,25 @@ TEST_F(SegmentedSortInt, Bool) result = cudf::stable_segmented_sorted_order(cudf::table_view({test_col}), segments); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view(), expected); } + +// Specific test for fix in https://github.com/rapidsai/cudf/pull/16463 +TEST_F(SegmentedSortInt, UnbalancedOffsets) +{ + auto h_input = std::vector(3535); + std::iota(h_input.begin(), h_input.end(), 1); + std::sort(h_input.begin(), h_input.end(), std::greater{}); + std::fill_n(h_input.begin(), 4, 0); + std::fill(h_input.begin() + 3533, h_input.end(), 10000); + auto d_input = cudf::detail::make_device_uvector_sync( + h_input, cudf::get_default_stream(), rmm::mr::get_current_device_resource()); + auto input = cudf::column_view(cudf::device_span(d_input)); + auto segments = cudf::test::fixed_width_column_wrapper({0, 4, 3533, 3535}); + // full sort should match handcrafted input data here + auto expected = cudf::sort(cudf::table_view({input})); + + auto input_view = cudf::table_view({input}); + auto result = cudf::segmented_sort_by_key(input_view, input_view, segments); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view().column(0), expected->view().column(0)); + result = cudf::stable_segmented_sort_by_key(input_view, input_view, segments); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view().column(0), expected->view().column(0)); +} From 6b0bff4b096ea87cd3436dba86146ed75af0f81e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:48:16 -1000 Subject: [PATCH 031/270] Disallow cudf.Series to accept column in favor of `._from_column` (#16454) `cudf.Series` is a public constructor that happens to accept a private `ColumnBase` object. Many ops return Columns and is natural to want to reconstruct a `Series`. This PR adds a `SingleColumnFrame._from_column` classmethod for instances where we need to wrap a new column in an `Index` or `Series`. This constructor also passes some unneeded validation in `ColumnAccessor` and `Series` Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16454 --- python/cudf/cudf/core/byte_pair_encoding.py | 6 +- python/cudf/cudf/core/column/categorical.py | 16 +-- python/cudf/cudf/core/column/methods.py | 15 ++- python/cudf/cudf/core/column/numerical.py | 12 +- python/cudf/cudf/core/column/string.py | 23 ++-- python/cudf/cudf/core/dataframe.py | 116 +++++++++---------- python/cudf/cudf/core/groupby/groupby.py | 13 +-- python/cudf/cudf/core/index.py | 44 ++++++- python/cudf/cudf/core/indexed_frame.py | 18 +-- python/cudf/cudf/core/multiindex.py | 19 ++- python/cudf/cudf/core/reshape.py | 8 +- python/cudf/cudf/core/series.py | 101 ++++++++++++---- python/cudf/cudf/core/single_column_frame.py | 41 +++---- python/cudf/cudf/core/tokenize_vocabulary.py | 8 +- python/cudf/cudf/core/tools/datetimes.py | 11 +- python/cudf/cudf/core/tools/numeric.py | 29 ++--- python/cudf/cudf/datasets.py | 5 +- python/cudf/cudf/io/dlpack.py | 2 +- python/cudf/cudf/tests/test_apply_rows.py | 8 +- python/cudf/cudf/tests/test_column.py | 44 ++++--- python/cudf/cudf/tests/test_dataframe.py | 26 +++-- python/cudf/cudf/tests/test_decimal.py | 10 +- python/cudf/cudf/tests/test_df_protocol.py | 6 +- python/cudf/cudf/tests/test_list.py | 2 +- python/cudf/cudf/tests/test_pickling.py | 4 +- python/cudf/cudf/tests/test_replace.py | 6 +- python/cudf/cudf/tests/test_series.py | 10 +- python/cudf/cudf/tests/test_setitem.py | 10 +- python/cudf/cudf/tests/test_string.py | 2 +- python/cudf/cudf/tests/test_string_udfs.py | 4 +- python/dask_cudf/dask_cudf/backends.py | 7 +- python/dask_cudf/dask_cudf/core.py | 2 +- 32 files changed, 360 insertions(+), 268 deletions(-) diff --git a/python/cudf/cudf/core/byte_pair_encoding.py b/python/cudf/cudf/core/byte_pair_encoding.py index 4c881022ecf..6ca64a0a2be 100644 --- a/python/cudf/cudf/core/byte_pair_encoding.py +++ b/python/cudf/cudf/core/byte_pair_encoding.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. from __future__ import annotations @@ -27,7 +27,7 @@ class BytePairEncoder: def __init__(self, merges_pair: "cudf.Series"): self.merge_pairs = cpp_merge_pairs(merges_pair._column) - def __call__(self, text, separator: str = " "): + def __call__(self, text, separator: str = " ") -> cudf.Series: """ Parameters @@ -56,4 +56,4 @@ def __call__(self, text, separator: str = " "): sep = cudf.Scalar(separator, dtype="str") result = cpp_byte_pair_encoding(text._column, self.merge_pairs, sep) - return cudf.Series(result) + return cudf.Series._from_column(result) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 55bfae30470..6fa69eb9cc1 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -123,7 +123,7 @@ def categories(self) -> "cudf.core.index.Index": return self._column.dtype.categories @property - def codes(self) -> "cudf.Series": + def codes(self) -> cudf.Series: """ Return Series of codes as well as the index. """ @@ -132,7 +132,7 @@ def codes(self) -> "cudf.Series": if isinstance(self._parent, cudf.Series) else None ) - return cudf.Series(self._column.codes, index=index) + return cudf.Series._from_column(self._column.codes, index=index) @property def ordered(self) -> bool: @@ -918,7 +918,7 @@ def find_and_replace( ) cur_categories = replaced.categories new_categories = cur_categories.apply_boolean_mask( - ~cudf.Series(cur_categories.isin(drop_values)) + cur_categories.isin(drop_values).unary_operator("not") ) replaced = replaced._set_categories(new_categories) df = df.dropna(subset=["new"]) @@ -943,7 +943,7 @@ def find_and_replace( # If a category is being replaced by an existing one, we # want to map it to None. If it's totally new, we want to # map it to the new label it is to be replaced by - dtype_replace = cudf.Series._from_data({None: replacement_col}) + dtype_replace = cudf.Series._from_column(replacement_col) dtype_replace[dtype_replace.isin(cats_col)] = None new_cats_col = cats_col.find_and_replace( to_replace_col, dtype_replace._column @@ -1273,12 +1273,8 @@ def _categories_equal( return False # if order doesn't matter, sort before the equals call below if not ordered: - cur_categories = cudf.Series(cur_categories).sort_values( - ignore_index=True - ) - new_categories = cudf.Series(new_categories).sort_values( - ignore_index=True - ) + cur_categories = cur_categories.sort_values() + new_categories = new_categories.sort_values() return cur_categories.equals(new_categories) def _set_categories( diff --git a/python/cudf/cudf/core/column/methods.py b/python/cudf/cudf/core/column/methods.py index 7c6f4e05577..8c46d238057 100644 --- a/python/cudf/cudf/core/column/methods.py +++ b/python/cudf/cudf/core/column/methods.py @@ -7,6 +7,8 @@ from typing_extensions import Literal import cudf +import cudf.core.column +import cudf.core.column_accessor from cudf.utils.utils import NotIterable ParentType = Union["cudf.Series", "cudf.core.index.Index"] @@ -84,14 +86,11 @@ def _return_or_inplace( data=table, index=self._parent.index ) elif isinstance(self._parent, cudf.Series): - if retain_index: - return cudf.Series( - new_col, - name=self._parent.name, - index=self._parent.index, - ) - else: - return cudf.Series(new_col, name=self._parent.name) + return cudf.Series._from_column( + new_col, + name=self._parent.name, + index=self._parent.index if retain_index else None, + ) elif isinstance(self._parent, cudf.BaseIndex): return cudf.Index(new_col, name=self._parent.name) else: diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index c326a10c844..df27134d458 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -555,11 +555,8 @@ def can_cast_safely(self, to_dtype: DtypeObj) -> bool: if self.dtype.kind == "f": # Exclude 'np.inf', '-np.inf' - s = cudf.Series(self) - # TODO: replace np.inf with cudf scalar when - # https://github.com/rapidsai/cudf/pull/6297 merges - non_infs = s[~((s == np.inf) | (s == -np.inf))] - col = non_infs._column + not_inf = (self != np.inf) & (self != -np.inf) + col = self.apply_boolean_mask(not_inf) else: col = self @@ -599,8 +596,7 @@ def can_cast_safely(self, to_dtype: DtypeObj) -> bool: else: filled = self.fillna(0) return ( - cudf.Series(filled).astype(to_dtype).astype(filled.dtype) - == cudf.Series(filled) + filled.astype(to_dtype).astype(filled.dtype) == filled ).all() # want to cast float to int: @@ -615,7 +611,7 @@ def can_cast_safely(self, to_dtype: DtypeObj) -> bool: # NOTE(seberg): it would make sense to limit to the mantissa range. if (float(self.min()) >= min_) and (float(self.max()) <= max_): filled = self.fillna(0) - return (cudf.Series(filled) % 1 == 0).all() + return (filled % 1 == 0).all() else: return False diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index b422ff86b17..1a4b558749d 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -358,7 +358,7 @@ def cat(self, others=None, sep=None, na_rep=None): ) if len(data) == 1 and data.null_count == 1: - data = [""] + data = cudf.core.column.as_column("", length=len(data)) # We only want to keep the index if we are adding something to each # row, not if we are joining all the rows into a single string. out = self._return_or_inplace(data, retain_index=others is not None) @@ -3623,7 +3623,7 @@ def findall(self, pat: str, flags: int = 0) -> SeriesOrIndex: data = libstrings.findall(self._column, pat, flags) return self._return_or_inplace(data) - def find_multiple(self, patterns: SeriesOrIndex) -> "cudf.Series": + def find_multiple(self, patterns: SeriesOrIndex) -> cudf.Series: """ Find all first occurrences of patterns in the Series/Index. @@ -3679,12 +3679,12 @@ def find_multiple(self, patterns: SeriesOrIndex) -> "cudf.Series": f"got: {patterns_column.dtype}" ) - return cudf.Series( + return cudf.Series._from_column( libstrings.find_multiple(self._column, patterns_column), + name=self._parent.name, index=self._parent.index if isinstance(self._parent, cudf.Series) else self._parent, - name=self._parent.name, ) def isempty(self) -> SeriesOrIndex: @@ -4376,14 +4376,9 @@ def code_points(self) -> SeriesOrIndex: 2 99 dtype: int32 """ - - new_col = libstrings.code_points(self._column) - if isinstance(self._parent, cudf.Series): - return cudf.Series(new_col, name=self._parent.name) - elif isinstance(self._parent, cudf.BaseIndex): - return cudf.Index(new_col, name=self._parent.name) - else: - return new_col + return self._return_or_inplace( + libstrings.code_points(self._column), retain_index=False + ) def translate(self, table: dict) -> SeriesOrIndex: """ @@ -4694,7 +4689,9 @@ def character_tokenize(self) -> SeriesOrIndex: if isinstance(self._parent, cudf.Series): lengths = self.len().fillna(0) index = self._parent.index.repeat(lengths) - return cudf.Series(result_col, name=self._parent.name, index=index) + return cudf.Series._from_column( + result_col, name=self._parent.name, index=index + ) elif isinstance(self._parent, cudf.BaseIndex): return cudf.Index(result_col, name=self._parent.name) else: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 865d2706ca3..a53c7bcc63c 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -382,7 +382,10 @@ def _setitem_tuple_arg(self, key, value): length = len(idx) if idx is not None else 1 value = as_column(value, length=length) - new_col = cudf.Series(value, index=idx) + if isinstance(value, ColumnBase): + new_col = cudf.Series._from_column(value, index=idx) + else: + new_col = cudf.Series(value, index=idx) if len(self._frame.index) != 0: new_col = new_col._align_to_index( self._frame.index, how="right" @@ -500,28 +503,33 @@ def __getitem__(self, arg): return frame._slice(row_spec.key) elif isinstance(row_spec, indexing_utils.ScalarIndexer): result = frame._gather(row_spec.key, keep_index=True) + new_name = result.index[0] + new_index = ensure_index(result.keys()) # Attempt to turn into series. - try: - # Behaviour difference from pandas, which will merrily - # turn any heterogeneous set of columns into a series if - # you only ask for one row. - new_name = result.index[0] - result = Series._concat( - [result[name] for name in column_names], - index=result.keys(), - ) - result.name = new_name - return result - except TypeError: - # Couldn't find a common type, Hence: - # Raise in pandas compatibility mode, - # or just return a 1xN dataframe otherwise - if cudf.get_option("mode.pandas_compatible"): - raise TypeError( - "All columns need to be of same type, please " - "typecast to common dtype." + if len(column_names) == 0: + return Series([], index=new_index, name=new_name) + else: + try: + # Behaviour difference from pandas, which will merrily + # turn any heterogeneous set of columns into a series if + # you only ask for one row. + ser = Series._concat( + [result[name] for name in column_names], ) - return result + except TypeError as err: + # Couldn't find a common type, Hence: + # Raise in pandas compatibility mode, + # or just return a 1xN dataframe otherwise + if cudf.get_option("mode.pandas_compatible"): + raise TypeError( + "All columns need to be of same type, please " + "typecast to common dtype." + ) from err + return result + else: + ser.index = new_index + ser.name = new_name + return ser elif isinstance(row_spec, indexing_utils.EmptyIndexer): return frame._empty_like(keep_index=True) assert_never(row_spec) @@ -1488,14 +1496,14 @@ def __delitem__(self, name): self._drop_column(name) @_performance_tracking - def memory_usage(self, index=True, deep=False): + def memory_usage(self, index=True, deep=False) -> cudf.Series: mem_usage = [col.memory_usage for col in self._data.columns] names = [str(name) for name in self._data.names] if index: mem_usage.append(self.index.memory_usage()) names.append("Index") - return Series._from_data( - data={None: as_column(mem_usage)}, + return Series._from_column( + as_column(mem_usage), index=cudf.Index(names), ) @@ -1752,7 +1760,7 @@ def _concat( if 1 == first_data_column_position: table_index = cudf.Index(cols[0]) elif first_data_column_position > 1: - table_index = DataFrame._from_data( + table_index = cudf.MultiIndex._from_data( data=dict( zip( indices[:first_data_column_position], @@ -3803,7 +3811,9 @@ def agg(self, aggs, axis=None): col_empty = column_empty( len(idxs), dtype=col.dtype, masked=True ) - ans = cudf.Series(data=col_empty, index=idxs) + ans = cudf.Series._from_column( + col_empty, index=cudf.Index(idxs) + ) if isinstance(aggs.get(key), abc.Iterable): # TODO : Allow simultaneous pass for multi-aggregation # as a future optimization @@ -4801,7 +4811,7 @@ def _func(x): # pragma: no cover # this could be written as a single kernel result = {} for name, col in self._data.items(): - apply_sr = Series._from_data({None: col}) + apply_sr = Series._from_column(col) result[name] = apply_sr.apply(_func)._column return DataFrame._from_data(result, index=self.index) @@ -6083,8 +6093,8 @@ def quantile( if q_is_number: result = result.transpose() - return Series( - data=result._columns[0], index=result.index, name=q + return Series._from_column( + result._columns[0], name=q, index=result.index ) else: # Ensure that qs is non-scalar so that we always get a column back. @@ -6346,13 +6356,9 @@ def count(self, axis=0, numeric_only=False): if axis != 0: raise NotImplementedError("Only axis=0 is currently supported.") length = len(self) - return Series._from_data( - { - None: as_column( - [length - col.null_count for col in self._columns] - ) - }, - cudf.Index(self._data.names), + return Series._from_column( + as_column([length - col.null_count for col in self._columns]), + index=cudf.Index(self._data.names), ) _SUPPORT_AXIS_LOOKUP = { @@ -6480,7 +6486,7 @@ def _reduce( ) else: idx = cudf.Index(source._data.names) - return Series._from_data({None: as_column(result)}, idx) + return Series._from_column(as_column(result), index=idx) elif axis == 1: return source._apply_cupy_method_axis_1(op, **kwargs) else: @@ -6710,11 +6716,7 @@ def _apply_cupy_method_axis_1(self, method, *args, **kwargs): result = result.set_mask( cudf._lib.transform.bools_to_mask(mask._column) ) - return Series( - result, - index=self.index, - dtype=result_dtype, - ) + return Series._from_column(result, index=self.index) else: result_df = DataFrame(result).set_index(self.index) result_df._set_columns_like(prepared._data) @@ -7302,9 +7304,7 @@ def unnamed_group_generator(): # Construct the resulting dataframe / series if not has_unnamed_levels: - result = Series._from_data( - data={None: stacked[0]}, index=new_index - ) + result = Series._from_column(stacked[0], index=new_index) else: if unnamed_level_values.nlevels == 1: unnamed_level_values = unnamed_level_values.get_level_values(0) @@ -7445,10 +7445,8 @@ def to_struct(self, name=None): size=len(self), offset=0, ) - return cudf.Series._from_data( - cudf.core.column_accessor.ColumnAccessor( - {name: col}, verify=False - ), + return cudf.Series._from_column( + col, index=self.index, name=name, ) @@ -7935,12 +7933,10 @@ def eval(self, expr: str, inplace: bool = False, **kwargs): raise ValueError( "Cannot operate inplace if there is no assignment" ) - return Series._from_data( - { - None: libcudf.transform.compute_column( - [*self._columns], self._column_names, statements[0] - ) - } + return Series._from_column( + libcudf.transform.compute_column( + [*self._columns], self._column_names, statements[0] + ) ) targets = [] @@ -8484,7 +8480,9 @@ def _get_non_null_cols_and_dtypes(col_idxs, list_of_columns): return non_null_columns, dtypes -def _find_common_dtypes_and_categories(non_null_columns, dtypes): +def _find_common_dtypes_and_categories( + non_null_columns, dtypes +) -> dict[Any, ColumnBase]: # A mapping of {idx: categories}, where `categories` is a # column of all the unique categorical values from each # categorical column across all input frames @@ -8500,9 +8498,9 @@ def _find_common_dtypes_and_categories(non_null_columns, dtypes): isinstance(col, cudf.core.column.CategoricalColumn) for col in cols ): # Combine and de-dupe the categories - categories[idx] = cudf.Series( - concat_columns([col.categories for col in cols]) - )._column.unique() + categories[idx] = concat_columns( + [col.categories for col in cols] + ).unique() # Set the column dtype to the codes' dtype. The categories # will be re-assigned at the end dtypes[idx] = min_signed_type(len(categories[idx])) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 3cfbd1d736a..92c4b73ceaa 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -458,12 +458,11 @@ def size(self): """ Return the size of each group. """ + col = cudf.core.column.column_empty( + len(self.obj), "int8", masked=False + ) return ( - cudf.Series( - cudf.core.column.column_empty( - len(self.obj), "int8", masked=False - ) - ) + cudf.Series._from_column(col) .groupby(self.grouping, sort=self._sort, dropna=self._dropna) .agg("size") ) @@ -484,7 +483,7 @@ def cumcount(self, ascending: bool = True): "ascending is currently not implemented." ) return ( - cudf.Series( + cudf.Series._from_column( cudf.core.column.column_empty( len(self.obj), "int8", masked=False ), @@ -1069,7 +1068,7 @@ def ngroup(self, ascending=True): # Count descending from num_groups - 1 to 0 groups = range(num_groups - 1, -1, -1) - group_ids = cudf.Series._from_data({None: as_column(groups)}) + group_ids = cudf.Series._from_column(as_column(groups)) if has_null_group: group_ids.iloc[-1] = cudf.NA diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 0d29ef07e7d..094da09ab08 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -60,7 +60,7 @@ from cudf.utils.utils import _warn_no_dask_cudf, search_range if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Generator, Hashable, Iterable from datetime import tzinfo @@ -1071,6 +1071,16 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return NotImplemented + @classmethod + @_performance_tracking + def _from_column( + cls, column: ColumnBase, *, name: Hashable = None + ) -> Self: + ca = cudf.core.column_accessor.ColumnAccessor( + {name: column}, verify=False + ) + return _index_from_data(ca) + @classmethod @_performance_tracking def _from_data(cls, data: MutableMapping, name: Any = no_default) -> Self: @@ -1092,8 +1102,30 @@ def _from_data_like_self( @classmethod @_performance_tracking def from_arrow(cls, obj): + """Create from PyArrow Array/ChunkedArray. + + Parameters + ---------- + array : PyArrow Array/ChunkedArray + PyArrow Object which has to be converted. + + Raises + ------ + TypeError for invalid input type. + + Returns + ------- + SingleColumnFrame + + Examples + -------- + >>> import cudf + >>> import pyarrow as pa + >>> cudf.Index.from_arrow(pa.array(["a", "b", None])) + Index(['a', 'b', ], dtype='object') + """ try: - return cls(ColumnBase.from_arrow(obj)) + return cls._from_column(ColumnBase.from_arrow(obj)) except TypeError: # Try interpreting object as a MultiIndex before failing. return cudf.MultiIndex.from_arrow(obj) @@ -1297,22 +1329,22 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): return _return_get_indexer_result(result.values) scatter_map, indices = libcudf.join.join([lcol], [rcol], how="inner") - (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) - result_series = cudf.Series(result) + result = libcudf.copying.scatter([indices], scatter_map, [result])[0] + result_series = cudf.Series._from_column(result) if method in {"ffill", "bfill", "pad", "backfill"}: result_series = _get_indexer_basic( index=self, positions=result_series, method=method, - target_col=cudf.Series(needle), + target_col=cudf.Series._from_column(needle), tolerance=tolerance, ) elif method == "nearest": result_series = _get_nearest_indexer( index=self, positions=result_series, - target_col=cudf.Series(needle), + target_col=cudf.Series._from_column(needle), tolerance=tolerance, ) elif method is not None: diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 0678ebfdd81..24d947a574a 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -35,6 +35,7 @@ is_list_like, is_scalar, ) +from cudf.core._base_index import BaseIndex from cudf.core._compat import PANDAS_LT_300 from cudf.core.buffer import acquire_spill_lock from cudf.core.column import ColumnBase, as_column @@ -67,7 +68,6 @@ Dtype, NotImplementedType, ) - from cudf.core._base_index import BaseIndex doc_reset_index_template = """ @@ -304,6 +304,10 @@ def _from_data( index: BaseIndex | None = None, ): out = super()._from_data(data) + if not (index is None or isinstance(index, BaseIndex)): + raise ValueError( + f"index must be None or a cudf.Index not {type(index).__name__}" + ) out._index = RangeIndex(out._data.nrows) if index is None else index return out @@ -2934,8 +2938,8 @@ def hash_values(self, method="murmur3", seed=None): # Note that both Series and DataFrame return Series objects from this # calculation, necessitating the unfortunate circular reference to the # child class here. - return cudf.Series._from_data( - {None: libcudf.hash.hash([*self._columns], method, seed)}, + return cudf.Series._from_column( + libcudf.hash.hash([*self._columns], method, seed), index=self.index, ) @@ -3219,13 +3223,13 @@ def duplicated(self, subset=None, keep="first"): distinct = libcudf.stream_compaction.distinct_indices( columns, keep=keep ) - (result,) = libcudf.copying.scatter( + result = libcudf.copying.scatter( [cudf.Scalar(False, dtype=bool)], distinct, [as_column(True, length=len(self), dtype=bool)], bounds_check=False, - ) - return cudf.Series(result, index=self.index) + )[0] + return cudf.Series._from_column(result, index=self.index) @_performance_tracking def _empty_like(self, keep_index=True) -> Self: @@ -3506,7 +3510,7 @@ def _apply(self, func, kernel_getter, *args, **kwargs): col = _post_process_output_col(ans_col, retty) col.set_base_mask(libcudf.transform.bools_to_mask(ans_mask)) - result = cudf.Series._from_data({None: col}, self.index) + result = cudf.Series._from_column(col, index=self.index) return result diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 2788455aebf..9646b34830f 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -702,12 +702,8 @@ def _compute_validity_mask(self, index, row_tuple, max_length): data_table = cudf.concat( [ frame, - cudf.DataFrame( - { - "idx": cudf.Series( - column.as_column(range(len(frame))) - ) - } + cudf.DataFrame._from_data( + {"idx": column.as_column(range(len(frame)))} ), ], axis=1, @@ -786,7 +782,7 @@ def _index_and_downcast(self, result, index, index_key): out_index.insert( out_index._num_columns, k, - cudf.Series._from_data({None: index._data.columns[k]}), + cudf.Series._from_column(index._data.columns[k]), ) # determine if we should downcast from a DataFrame to a Series @@ -852,7 +848,10 @@ def _get_row_major( valid_indices = self._get_valid_indices_by_tuple( df.index, row_tuple, len(df.index) ) - indices = cudf.Series(valid_indices) + if isinstance(valid_indices, column.ColumnBase): + indices = cudf.Series._from_column(valid_indices) + else: + indices = cudf.Series(valid_indices) result = df.take(indices) final = self._index_and_downcast(result, result.index, row_tuple) return final @@ -1925,8 +1924,8 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): *join_keys, how="inner", ) - (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) - result_series = cudf.Series(result) + result = libcudf.copying.scatter([indices], scatter_map, [result])[0] + result_series = cudf.Series._from_column(result) if method in {"ffill", "bfill", "pad", "backfill"}: result_series = _get_indexer_basic( diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index e7248977b1d..52a55760d4a 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -484,9 +484,7 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): if len(new_objs) == 1 and not ignore_index: return new_objs[0] else: - return cudf.Series._concat( - objs, axis=axis, index=None if ignore_index else True - ) + return cudf.Series._concat(objs, axis=axis, index=not ignore_index) elif typ is cudf.MultiIndex: return cudf.MultiIndex._concat(objs) elif issubclass(typ, cudf.Index): @@ -632,7 +630,7 @@ def melt( def _tile(A, reps): series_list = [A] * reps if reps > 0: - return cudf.Series._concat(objs=series_list, index=None) + return cudf.Series._concat(objs=series_list, index=False) else: return cudf.Series([], dtype=A.dtype) @@ -661,7 +659,7 @@ def _tile(A, reps): # Step 3: add values mdata[value_name] = cudf.Series._concat( - objs=[frame[val] for val in value_vars], index=None + objs=[frame[val] for val in value_vars], index=False ) return cudf.DataFrame(mdata) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 929af5cd981..de57ac5f290 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -69,6 +69,8 @@ from cudf.utils.performance_tracking import _performance_tracking if TYPE_CHECKING: + import pyarrow as pa + from cudf._typing import ( ColumnLike, DataFrameOrSeries, @@ -294,8 +296,8 @@ def __getitem__(self, arg: Any) -> ScalarLike | DataFrameOrSeries: return result try: arg = self._loc_to_iloc(arg) - except (TypeError, KeyError, IndexError, ValueError): - raise KeyError(arg) + except (TypeError, KeyError, IndexError, ValueError) as err: + raise KeyError(arg) from err return self._frame.iloc[arg] @@ -394,8 +396,10 @@ def _loc_to_iloc(self, arg): return _indices_from_labels(self._frame, arg) else: - arg = cudf.core.series.Series(cudf.core.column.as_column(arg)) - if arg.dtype in (bool, np.bool_): + arg = cudf.core.series.Series._from_column( + cudf.core.column.as_column(arg) + ) + if arg.dtype.kind == "b": return arg else: indices = _indices_from_labels(self._frame, arg) @@ -510,7 +514,37 @@ def from_categorical(cls, categorical, codes=None): col = cudf.core.column.categorical.pandas_categorical_as_column( categorical, codes=codes ) - return Series(data=col) + return Series._from_column(col) + + @classmethod + @_performance_tracking + def from_arrow(cls, array: pa.Array): + """Create from PyArrow Array/ChunkedArray. + + Parameters + ---------- + array : PyArrow Array/ChunkedArray + PyArrow Object which has to be converted. + + Raises + ------ + TypeError for invalid input type. + + Returns + ------- + SingleColumnFrame + + Examples + -------- + >>> import cudf + >>> import pyarrow as pa + >>> cudf.Series.from_arrow(pa.array(["a", "b", None])) + 0 a + 1 b + 2 + dtype: object + """ + return cls._from_column(ColumnBase.from_arrow(array)) @classmethod @_performance_tracking @@ -560,7 +594,8 @@ def from_masked_array(cls, data, mask, null_count=None): dtype: int64 """ col = as_column(data).set_mask(mask) - return cls(data=col) + ca = ColumnAccessor({None: col}, verify=False) + return cls._from_data(ca) @_performance_tracking def __init__( @@ -586,10 +621,10 @@ def __init__( column = as_column(data, nan_as_null=nan_as_null, dtype=dtype) if isinstance(data, (pd.Series, Series)): index_from_data = ensure_index(data.index) - elif isinstance(data, ColumnAccessor): + elif isinstance(data, (ColumnAccessor, ColumnBase)): raise TypeError( "Use cudf.Series._from_data for constructing a Series from " - "ColumnAccessor" + "ColumnAccessor or a ColumnBase" ) elif isinstance(data, dict): if not data: @@ -656,6 +691,18 @@ def __init__( self._index = second_index self._check_data_index_length_match() + @classmethod + @_performance_tracking + def _from_column( + cls, + column: ColumnBase, + *, + name: abc.Hashable = None, + index: BaseIndex | None = None, + ) -> Self: + ca = ColumnAccessor({name: column}, verify=False) + return cls._from_data(ca, index=index) + @classmethod @_performance_tracking def _from_data( @@ -1535,17 +1582,21 @@ def dtype(self): @classmethod @_performance_tracking - def _concat(cls, objs, axis=0, index=True): + def _concat(cls, objs, axis=0, index: bool = True): # Concatenate index if not provided if index is True: if isinstance(objs[0].index, cudf.MultiIndex): - index = cudf.MultiIndex._concat([o.index for o in objs]) + result_index = cudf.MultiIndex._concat([o.index for o in objs]) else: with warnings.catch_warnings(): warnings.simplefilter("ignore", FutureWarning) - index = cudf.core.index.Index._concat( + result_index = cudf.core.index.Index._concat( [o.index for o in objs] ) + elif index is False: + result_index = None + else: + raise ValueError(f"{index=} must be a bool") names = {obj.name for obj in objs} if len(names) == 1: @@ -1597,7 +1648,9 @@ def _concat(cls, objs, axis=0, index=True): if len(objs): col = col._with_type_metadata(objs[0].dtype) - return cls(data=col, index=index, name=name) + return cls._from_data( + ColumnAccessor({name: col}, verify=False), index=result_index + ) @property # type: ignore @_performance_tracking @@ -2709,8 +2762,8 @@ def mode(self, dropna=True): if len(val_counts) > 0: val_counts = val_counts[val_counts == val_counts.iloc[0]] - return Series._from_data( - {self.name: val_counts.index.sort_values()._column}, name=self.name + return Series._from_column( + val_counts.index.sort_values()._column, name=self.name ) @_performance_tracking @@ -2999,8 +3052,8 @@ def isin(self, values): f"to isin(), you passed a [{type(values).__name__}]" ) - return Series._from_data( - {self.name: self._column.isin(values)}, index=self.index + return Series._from_column( + self._column.isin(values), name=self.name, index=self.index ) @_performance_tracking @@ -3036,7 +3089,7 @@ def unique(self): res = self._column.unique() if cudf.get_option("mode.pandas_compatible"): return res.values - return Series(res, name=self.name) + return Series._from_column(res, name=self.name) @_performance_tracking def value_counts( @@ -3268,8 +3321,9 @@ def quantile( if return_scalar: return result - return Series._from_data( - data={self.name: result}, + return Series._from_column( + result, + name=self.name, index=cudf.Index(np_array_q) if quant_index else None, ) @@ -3351,8 +3405,9 @@ def digitize(self, bins, right=False): 3 2 dtype: int32 """ - return Series( - cudf.core.column.numerical.digitize(self._column, bins, right) + return Series._from_column( + cudf.core.column.numerical.digitize(self._column, bins, right), + name=self.name, ) @_performance_tracking @@ -5293,10 +5348,10 @@ def isclose(a, b, rtol=1e-05, atol=1e-08, equal_nan=False): elif b_col.null_count: null_values = b_col.isnull() else: - return Series(result_col, index=index) + return Series._from_column(result_col, index=index) result_col[null_values] = False if equal_nan is True and a_col.null_count and b_col.null_count: result_col[equal_nulls] = True - return Series(result_col, index=index) + return Series._from_column(result_col, index=index) diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index a5ff1223791..eb6714029cf 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -15,11 +15,14 @@ is_numeric_dtype, ) from cudf.core.column import ColumnBase, as_column +from cudf.core.column_accessor import ColumnAccessor from cudf.core.frame import Frame from cudf.utils.performance_tracking import _performance_tracking from cudf.utils.utils import NotIterable if TYPE_CHECKING: + from collections.abc import Hashable + import cupy import numpy import pyarrow as pa @@ -112,35 +115,17 @@ def values_host(self) -> numpy.ndarray: # noqa: D102 @classmethod @_performance_tracking - def from_arrow(cls, array) -> Self: - """Create from PyArrow Array/ChunkedArray. - - Parameters - ---------- - array : PyArrow Array/ChunkedArray - PyArrow Object which has to be converted. - - Raises - ------ - TypeError for invalid input type. - - Returns - ------- - SingleColumnFrame + def _from_column( + cls, column: ColumnBase, *, name: Hashable = None + ) -> Self: + """Constructor for a single Column.""" + ca = ColumnAccessor({name: column}, verify=False) + return cls._from_data(ca) - Examples - -------- - >>> import cudf - >>> import pyarrow as pa - >>> cudf.Index.from_arrow(pa.array(["a", "b", None])) - Index(['a', 'b', None], dtype='object') - >>> cudf.Series.from_arrow(pa.array(["a", "b", None])) - 0 a - 1 b - 2 - dtype: object - """ - return cls(ColumnBase.from_arrow(array)) + @classmethod + @_performance_tracking + def from_arrow(cls, array) -> Self: + raise NotImplementedError @_performance_tracking def to_arrow(self) -> pa.Array: diff --git a/python/cudf/cudf/core/tokenize_vocabulary.py b/python/cudf/cudf/core/tokenize_vocabulary.py index afb3496311b..99d85c0c5c0 100644 --- a/python/cudf/cudf/core/tokenize_vocabulary.py +++ b/python/cudf/cudf/core/tokenize_vocabulary.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. from __future__ import annotations @@ -22,7 +22,9 @@ class TokenizeVocabulary: def __init__(self, vocabulary: "cudf.Series"): self.vocabulary = cpp_tokenize_vocabulary(vocabulary._column) - def tokenize(self, text, delimiter: str = "", default_id: int = -1): + def tokenize( + self, text, delimiter: str = "", default_id: int = -1 + ) -> cudf.Series: """ Parameters ---------- @@ -45,4 +47,4 @@ def tokenize(self, text, delimiter: str = "", default_id: int = -1): text._column, self.vocabulary, delim, default_id ) - return cudf.Series(result) + return cudf.Series._from_column(result) diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index c6e2b5d10e1..2f77778116f 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -18,6 +18,8 @@ ) from cudf.api.types import is_integer, is_scalar from cudf.core import column +from cudf.core.column_accessor import ColumnAccessor +from cudf.core.index import ensure_index # https://github.com/pandas-dev/pandas/blob/2.2.x/pandas/core/tools/datetimes.py#L1112 _unit_map = { @@ -275,7 +277,7 @@ def to_datetime( format=format, utc=utc, ) - return cudf.Series(col, index=arg.index) + return cudf.Series._from_column(col, index=arg.index) else: col = _process_col( col=column.as_column(arg), @@ -286,9 +288,12 @@ def to_datetime( utc=utc, ) if isinstance(arg, (cudf.BaseIndex, pd.Index)): - return cudf.Index(col, name=arg.name) + ca = ColumnAccessor({arg.name: col}, verify=False) + return cudf.DatetimeIndex._from_data(ca) elif isinstance(arg, (cudf.Series, pd.Series)): - return cudf.Series(col, index=arg.index, name=arg.name) + return cudf.Series._from_column( + col, name=arg.name, index=ensure_index(arg.index) + ) elif is_scalar(arg): return col.element_indexing(0) else: diff --git a/python/cudf/cudf/core/tools/numeric.py b/python/cudf/cudf/core/tools/numeric.py index 07158e4ee61..8b95f6f6a04 100644 --- a/python/cudf/cudf/core/tools/numeric.py +++ b/python/cudf/cudf/core/tools/numeric.py @@ -1,6 +1,8 @@ # Copyright (c) 2018-2024, NVIDIA CORPORATION. +from __future__ import annotations import warnings +from typing import TYPE_CHECKING import numpy as np import pandas as pd @@ -11,8 +13,12 @@ from cudf.api.types import _is_non_decimal_numeric_dtype, is_string_dtype from cudf.core.column import as_column from cudf.core.dtypes import CategoricalDtype +from cudf.core.index import ensure_index from cudf.utils.dtypes import can_convert_to_column +if TYPE_CHECKING: + from cudf.core.column import ColumnBase + def to_numeric(arg, errors="raise", downcast=None): """ @@ -164,7 +170,9 @@ def to_numeric(arg, errors="raise", downcast=None): break if isinstance(arg, (cudf.Series, pd.Series)): - return cudf.Series(col, index=arg.index, name=arg.name) + return cudf.Series._from_column( + col, name=arg.name, index=ensure_index(arg.index) + ) else: if col.has_nulls(): # To match pandas, always return a floating type filled with nan. @@ -226,25 +234,10 @@ def _convert_str_col(col, errors, _downcast=None): raise ValueError("Unable to convert some strings to numerics.") -def _proc_inf_empty_strings(col): +def _proc_inf_empty_strings(col: ColumnBase) -> ColumnBase: """Handles empty and infinity strings""" col = libstrings.to_lower(col) - col = _proc_empty_strings(col) - col = _proc_inf_strings(col) - return col - - -def _proc_empty_strings(col): - """Replaces empty strings with NaN""" - s = cudf.Series(col) - s = s.where(s != "", "NaN") - return s._column - - -def _proc_inf_strings(col): - """Convert "inf/infinity" strings into "Inf", the native string - representing infinity in libcudf - """ + col = col.find_and_replace(as_column([""]), as_column(["NaN"])) # TODO: This can be handled by libcudf in # future see StringColumn.as_numerical_column col = libstrings.replace_multi( diff --git a/python/cudf/cudf/datasets.py b/python/cudf/cudf/datasets.py index 7b183d5f1a3..dbabaacf6b5 100644 --- a/python/cudf/cudf/datasets.py +++ b/python/cudf/cudf/datasets.py @@ -5,7 +5,6 @@ import cudf from cudf._lib.transform import bools_to_mask -from cudf.core.column_accessor import ColumnAccessor __all__ = ["timeseries", "randomdata"] @@ -73,9 +72,7 @@ def timeseries( ) mask_buf = bools_to_mask(cudf.core.column.as_column(mask)) masked_col = gdf[col]._column.set_mask(mask_buf) - gdf[col] = cudf.Series._from_data( - ColumnAccessor({None: masked_col}), index=gdf.index - ) + gdf[col] = cudf.Series._from_column(masked_col, index=gdf.index) return gdf diff --git a/python/cudf/cudf/io/dlpack.py b/python/cudf/cudf/io/dlpack.py index d3d99aab0cd..1347b2cc38f 100644 --- a/python/cudf/cudf/io/dlpack.py +++ b/python/cudf/cudf/io/dlpack.py @@ -71,7 +71,7 @@ def to_dlpack(cudf_obj): if isinstance(cudf_obj, (cudf.DataFrame, cudf.Series, cudf.BaseIndex)): gdf = cudf_obj elif isinstance(cudf_obj, ColumnBase): - gdf = cudf.Series._from_data({None: cudf_obj}) + gdf = cudf.Series._from_column(cudf_obj) else: raise TypeError( f"Input of type {type(cudf_obj)} cannot be converted " diff --git a/python/cudf/cudf/tests/test_apply_rows.py b/python/cudf/cudf/tests/test_apply_rows.py index a11022c1a17..f9b0d9c1e78 100644 --- a/python/cudf/cudf/tests/test_apply_rows.py +++ b/python/cudf/cudf/tests/test_apply_rows.py @@ -27,8 +27,12 @@ def test_dataframe_apply_rows(dtype, has_nulls, pessimistic): gdf_series_expected = gdf_series_a * gdf_series_b else: # optimistically ignore the null masks - a = cudf.Series(column.build_column(gdf_series_a.data, dtype)) - b = cudf.Series(column.build_column(gdf_series_b.data, dtype)) + a = cudf.Series._from_column( + column.build_column(gdf_series_a.data, dtype) + ) + b = cudf.Series._from_column( + column.build_column(gdf_series_b.data, dtype) + ) gdf_series_expected = a * b df_expected = cudf.DataFrame( diff --git a/python/cudf/cudf/tests/test_column.py b/python/cudf/cudf/tests/test_column.py index c288155112c..4aa7fb27c9b 100644 --- a/python/cudf/cudf/tests/test_column.py +++ b/python/cudf/cudf/tests/test_column.py @@ -95,7 +95,7 @@ def test_column_offset_and_size(pandas_input, offset, size): else: assert col.size == (col.data.size / col.dtype.itemsize) - got = cudf.Series(col) + got = cudf.Series._from_column(col) if offset is None: offset = 0 @@ -112,8 +112,8 @@ def test_column_offset_and_size(pandas_input, offset, size): def column_slicing_test(col, offset, size, cast_to_float=False): col_slice = col.slice(offset, offset + size) - series = cudf.Series(col) - sliced_series = cudf.Series(col_slice) + series = cudf.Series._from_column(col) + sliced_series = cudf.Series._from_column(col_slice) if cast_to_float: pd_series = series.astype(float).to_pandas() @@ -208,7 +208,9 @@ def test_as_column_scalar_with_nan(nan_as_null, scalar, size): ) got = ( - cudf.Series(as_column(scalar, length=size, nan_as_null=nan_as_null)) + cudf.Series._from_column( + as_column(scalar, length=size, nan_as_null=nan_as_null) + ) .dropna() .to_numpy() ) @@ -250,12 +252,18 @@ def test_column_chunked_array_creation(): actual_column = cudf.core.column.as_column(chunked_array, dtype="float") expected_column = cudf.core.column.as_column(pyarrow_array, dtype="float") - assert_eq(cudf.Series(actual_column), cudf.Series(expected_column)) + assert_eq( + cudf.Series._from_column(actual_column), + cudf.Series._from_column(expected_column), + ) actual_column = cudf.core.column.as_column(chunked_array) expected_column = cudf.core.column.as_column(pyarrow_array) - assert_eq(cudf.Series(actual_column), cudf.Series(expected_column)) + assert_eq( + cudf.Series._from_column(actual_column), + cudf.Series._from_column(expected_column), + ) @pytest.mark.parametrize( @@ -287,7 +295,7 @@ def test_column_view_valid_numeric_to_numeric(data, from_dtype, to_dtype): gpu_data_view = gpu_data.view(to_dtype) expect = pd.Series(cpu_data_view, dtype=cpu_data_view.dtype) - got = cudf.Series(gpu_data_view, dtype=gpu_data_view.dtype) + got = cudf.Series._from_column(gpu_data_view).astype(gpu_data_view.dtype) gpu_ptr = gpu_data.data.get_ptr(mode="read") assert gpu_ptr == got._column.data.get_ptr(mode="read") @@ -327,7 +335,7 @@ def test_column_view_invalid_numeric_to_numeric(data, from_dtype, to_dtype): ], ) def test_column_view_valid_string_to_numeric(data, to_dtype): - expect = cudf.Series(cudf.Series(data)._column.view(to_dtype)) + expect = cudf.Series._from_column(cudf.Series(data)._column.view(to_dtype)) got = cudf.Series(str_host_view(data, to_dtype)) assert_eq(expect, got) @@ -342,7 +350,7 @@ def test_column_view_nulls_widths_even(): sr = cudf.Series(data, dtype="int32") expect = cudf.Series(expect_data, dtype="float32") - got = cudf.Series(sr._column.view("float32")) + got = cudf.Series._from_column(sr._column.view("float32")) assert_eq(expect, got) @@ -354,7 +362,7 @@ def test_column_view_nulls_widths_even(): sr = cudf.Series(data, dtype="float64") expect = cudf.Series(expect_data, dtype="int64") - got = cudf.Series(sr._column.view("int64")) + got = cudf.Series._from_column(sr._column.view("int64")) assert_eq(expect, got) @@ -365,7 +373,9 @@ def test_column_view_numeric_slice(slc): sr = cudf.Series(data) expect = cudf.Series(data[slc].view("int64")) - got = cudf.Series(sr._column.slice(slc.start, slc.stop).view("int64")) + got = cudf.Series._from_column( + sr._column.slice(slc.start, slc.stop).view("int64") + ) assert_eq(expect, got) @@ -376,7 +386,7 @@ def test_column_view_numeric_slice(slc): def test_column_view_string_slice(slc): data = ["a", "bcde", "cd", "efg", "h"] - expect = cudf.Series( + expect = cudf.Series._from_column( cudf.Series(data)._column.slice(slc.start, slc.stop).view("int8") ) got = cudf.Series(str_host_view(data[slc], "int8")) @@ -409,7 +419,10 @@ def test_as_column_buffer(data, expected): actual_column = cudf.core.column.as_column( cudf.core.buffer.as_buffer(data), dtype=data.dtype ) - assert_eq(cudf.Series(actual_column), cudf.Series(expected)) + assert_eq( + cudf.Series._from_column(actual_column), + cudf.Series._from_column(expected), + ) @pytest.mark.parametrize( @@ -436,7 +449,10 @@ def test_as_column_arrow_array(data, pyarrow_kwargs, cudf_kwargs): pyarrow_data = pa.array(data, **pyarrow_kwargs) cudf_from_pyarrow = as_column(pyarrow_data) expected = as_column(data, **cudf_kwargs) - assert_eq(cudf.Series(cudf_from_pyarrow), cudf.Series(expected)) + assert_eq( + cudf.Series._from_column(cudf_from_pyarrow), + cudf.Series._from_column(expected), + ) @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index e2ce5c03b70..2c59253d500 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -4264,34 +4264,36 @@ def test_empty_dataframe_describe(): def test_as_column_types(): col = column.as_column(cudf.Series([], dtype="float64")) assert_eq(col.dtype, np.dtype("float64")) - gds = cudf.Series(col) + gds = cudf.Series._from_column(col) pds = pd.Series(pd.Series([], dtype="float64")) assert_eq(pds, gds) col = column.as_column(cudf.Series([], dtype="float64"), dtype="float32") assert_eq(col.dtype, np.dtype("float32")) - gds = cudf.Series(col) + gds = cudf.Series._from_column(col) pds = pd.Series(pd.Series([], dtype="float32")) assert_eq(pds, gds) col = column.as_column(cudf.Series([], dtype="float64"), dtype="str") assert_eq(col.dtype, np.dtype("object")) - gds = cudf.Series(col) + gds = cudf.Series._from_column(col) pds = pd.Series(pd.Series([], dtype="str")) assert_eq(pds, gds) col = column.as_column(cudf.Series([], dtype="float64"), dtype="object") assert_eq(col.dtype, np.dtype("object")) - gds = cudf.Series(col) + gds = cudf.Series._from_column(col) pds = pd.Series(pd.Series([], dtype="object")) assert_eq(pds, gds) pds = pd.Series(np.array([1, 2, 3]), dtype="float32") - gds = cudf.Series(column.as_column(np.array([1, 2, 3]), dtype="float32")) + gds = cudf.Series._from_column( + column.as_column(np.array([1, 2, 3]), dtype="float32") + ) assert_eq(pds, gds) @@ -4301,23 +4303,25 @@ def test_as_column_types(): assert_eq(pds, gds) pds = pd.Series([], dtype="float64") - gds = cudf.Series(column.as_column(pds)) + gds = cudf.Series._from_column(column.as_column(pds)) assert_eq(pds, gds) pds = pd.Series([1, 2, 4], dtype="int64") - gds = cudf.Series(column.as_column(cudf.Series([1, 2, 4]), dtype="int64")) + gds = cudf.Series._from_column( + column.as_column(cudf.Series([1, 2, 4]), dtype="int64") + ) assert_eq(pds, gds) pds = pd.Series([1.2, 18.0, 9.0], dtype="float32") - gds = cudf.Series( + gds = cudf.Series._from_column( column.as_column(cudf.Series([1.2, 18.0, 9.0]), dtype="float32") ) assert_eq(pds, gds) pds = pd.Series([1.2, 18.0, 9.0], dtype="str") - gds = cudf.Series( + gds = cudf.Series._from_column( column.as_column(cudf.Series([1.2, 18.0, 9.0]), dtype="str") ) @@ -6521,7 +6525,9 @@ def test_from_pandas_for_series_nan_as_null(nan_as_null): data = [np.nan, 2.0, 3.0] psr = pd.Series(data) - expected = cudf.Series(column.as_column(data, nan_as_null=nan_as_null)) + expected = cudf.Series._from_column( + column.as_column(data, nan_as_null=nan_as_null) + ) got = cudf.from_pandas(psr, nan_as_null=nan_as_null) assert_eq(expected, got) diff --git a/python/cudf/cudf/tests/test_decimal.py b/python/cudf/cudf/tests/test_decimal.py index 65f739bc74a..b63788d20b7 100644 --- a/python/cudf/cudf/tests/test_decimal.py +++ b/python/cudf/cudf/tests/test_decimal.py @@ -106,7 +106,7 @@ def test_typecast_from_float_to_decimal(request, data, from_dtype, to_dtype): pa_arr = got.to_arrow().cast( pa.decimal128(to_dtype.precision, to_dtype.scale) ) - expected = cudf.Series(Decimal64Column.from_arrow(pa_arr)) + expected = cudf.Series._from_column(Decimal64Column.from_arrow(pa_arr)) got = got.astype(to_dtype) @@ -146,7 +146,7 @@ def test_typecast_from_int_to_decimal(data, from_dtype, to_dtype): .cast("float64") .cast(pa.decimal128(to_dtype.precision, to_dtype.scale)) ) - expected = cudf.Series(Decimal64Column.from_arrow(pa_arr)) + expected = cudf.Series._from_column(Decimal64Column.from_arrow(pa_arr)) got = got.astype(to_dtype) @@ -206,9 +206,9 @@ def test_typecast_to_from_decimal(data, from_dtype, to_dtype): pa.decimal128(to_dtype.precision, to_dtype.scale), safe=False ) if isinstance(to_dtype, Decimal32Dtype): - expected = cudf.Series(Decimal32Column.from_arrow(pa_arr)) + expected = cudf.Series._from_column(Decimal32Column.from_arrow(pa_arr)) elif isinstance(to_dtype, Decimal64Dtype): - expected = cudf.Series(Decimal64Column.from_arrow(pa_arr)) + expected = cudf.Series._from_column(Decimal64Column.from_arrow(pa_arr)) with expect_warning_if(to_dtype.scale < s.dtype.scale, UserWarning): got = s.astype(to_dtype) @@ -245,7 +245,7 @@ def test_typecast_from_decimal(data, from_dtype, to_dtype): pa_arr = got.to_arrow().cast(to_dtype, safe=False) got = got.astype(to_dtype) - expected = cudf.Series(NumericalColumn.from_arrow(pa_arr)) + expected = cudf.Series._from_column(NumericalColumn.from_arrow(pa_arr)) assert_eq(got, expected) assert_eq(got.dtype, expected.dtype) diff --git a/python/cudf/cudf/tests/test_df_protocol.py b/python/cudf/cudf/tests/test_df_protocol.py index 7f48e414180..44270d20d59 100644 --- a/python/cudf/cudf/tests/test_df_protocol.py +++ b/python/cudf/cudf/tests/test_df_protocol.py @@ -78,7 +78,7 @@ def assert_buffer_equal(buffer_and_dtype: tuple[_CuDFBuffer, Any], cudfcol): # FIXME: In gh-10202 some minimal fixes were added to unblock CI. But # currently only non-null values are compared, null positions are # unchecked. - non_null_idxs = ~cudf.Series(cudfcol).isna() + non_null_idxs = cudfcol.notnull() assert_eq( col_from_buf.apply_boolean_mask(non_null_idxs), cudfcol.apply_boolean_mask(non_null_idxs), @@ -86,8 +86,8 @@ def assert_buffer_equal(buffer_and_dtype: tuple[_CuDFBuffer, Any], cudfcol): array_from_dlpack = cp.from_dlpack(buf.__dlpack__()).get() col_array = cp.asarray(cudfcol.data_array_view(mode="read")).get() assert_eq( - array_from_dlpack[non_null_idxs.to_numpy()].flatten(), - col_array[non_null_idxs.to_numpy()].flatten(), + array_from_dlpack[non_null_idxs.values_host].flatten(), + col_array[non_null_idxs.values_host].flatten(), ) diff --git a/python/cudf/cudf/tests/test_list.py b/python/cudf/cudf/tests/test_list.py index 36bcaa66d7d..c4c883ca9f9 100644 --- a/python/cudf/cudf/tests/test_list.py +++ b/python/cudf/cudf/tests/test_list.py @@ -946,5 +946,5 @@ def test_empty_nested_list_uninitialized_offsets_memory_usage(): null_count=col.null_count, children=(column_empty(0, col.children[0].dtype), empty_inner), ) - ser = cudf.Series._from_data({None: col_empty_offset}) + ser = cudf.Series._from_column(col_empty_offset) assert ser.memory_usage() == 8 diff --git a/python/cudf/cudf/tests/test_pickling.py b/python/cudf/cudf/tests/test_pickling.py index 719e8a33285..0f13a9e173a 100644 --- a/python/cudf/cudf/tests/test_pickling.py +++ b/python/cudf/cudf/tests/test_pickling.py @@ -127,7 +127,7 @@ def test_pickle_categorical_column(slices): pickled = pickle.dumps(input_col) out = pickle.loads(pickled) - assert_eq(Series(out), Series(input_col)) + assert_eq(Series._from_column(out), Series._from_column(input_col)) @pytest.mark.parametrize( @@ -148,4 +148,4 @@ def test_pickle_string_column(slices): pickled = pickle.dumps(input_col) out = pickle.loads(pickled) - assert_eq(Series(out), Series(input_col)) + assert_eq(Series._from_column(out), Series._from_column(input_col)) diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index d4fe5ff3bb5..1973fe6fb41 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -817,12 +817,12 @@ def test_fillna_string(ps_data, fill_value, inplace): def test_series_fillna_invalid_dtype(data_dtype): gdf = cudf.Series([1, 2, None, 3], dtype=data_dtype) fill_value = 2.5 - with pytest.raises(TypeError) as raises: - gdf.fillna(fill_value) - raises.match( + msg = ( f"Cannot safely cast non-equivalent" f" {type(fill_value).__name__} to {gdf.dtype.type.__name__}" ) + with pytest.raises(TypeError, match=msg): + gdf.fillna(fill_value) @pytest.mark.parametrize("data_dtype", NUMERIC_TYPES) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 8ed78d804bf..6a1887afb1f 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2041,7 +2041,7 @@ def test_series_ordered_dedup(): sr = cudf.Series(np.random.randint(0, 100, 1000)) # pandas unique() preserves order expect = pd.Series(sr.to_pandas().unique()) - got = cudf.Series(sr._column.unique()) + got = cudf.Series._from_column(sr._column.unique()) assert_eq(expect.values, got.values) @@ -2697,7 +2697,9 @@ def test_series_duplicate_index_reindex(): def test_list_category_like_maintains_dtype(): dtype = cudf.CategoricalDtype(categories=[1, 2, 3, 4], ordered=True) data = [1, 2, 3] - result = cudf.Series(cudf.core.column.as_column(data, dtype=dtype)) + result = cudf.Series._from_column( + cudf.core.column.as_column(data, dtype=dtype) + ) expected = pd.Series(data, dtype=dtype.to_pandas()) assert_eq(result, expected) @@ -2705,7 +2707,9 @@ def test_list_category_like_maintains_dtype(): def test_list_interval_like_maintains_dtype(): dtype = cudf.IntervalDtype(subtype=np.int8) data = [pd.Interval(1, 2)] - result = cudf.Series(cudf.core.column.as_column(data, dtype=dtype)) + result = cudf.Series._from_column( + cudf.core.column.as_column(data, dtype=dtype) + ) expected = pd.Series(data, dtype=dtype.to_pandas()) assert_eq(result, expected) diff --git a/python/cudf/cudf/tests/test_setitem.py b/python/cudf/cudf/tests/test_setitem.py index 69122cdbafa..5406836ba61 100644 --- a/python/cudf/cudf/tests/test_setitem.py +++ b/python/cudf/cudf/tests/test_setitem.py @@ -178,13 +178,19 @@ def test_column_set_equal_length_object_by_mask(): bool_col = cudf.Series([True, True, True, True, True])._column data[bool_col] = replace_data - assert_eq(cudf.Series(data), cudf.Series(replace_data)) + assert_eq( + cudf.Series._from_column(data), + cudf.Series._from_column(replace_data), + ) data = cudf.Series([0, 0, 1, 1, 1])._column bool_col = cudf.Series([True, False, True, False, True])._column data[bool_col] = replace_data - assert_eq(cudf.Series(data), cudf.Series([100, 0, 300, 1, 500])) + assert_eq( + cudf.Series._from_column(data), + cudf.Series([100, 0, 300, 1, 500]), + ) def test_column_set_unequal_length_object_by_mask(): diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index f447759d010..4bd084a3938 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -2677,7 +2677,7 @@ def test_string_int_to_ipv4(): ["0.0.0.0", None, "0.0.0.0", "41.168.0.1", "127.0.0.1", "41.197.0.1"] ) - got = cudf.Series(gsr._column.int2ip()) + got = cudf.Series._from_column(gsr._column.int2ip()) assert_eq(expected, got) diff --git a/python/cudf/cudf/tests/test_string_udfs.py b/python/cudf/cudf/tests/test_string_udfs.py index 4432d2afc8e..69876d97aad 100644 --- a/python/cudf/cudf/tests/test_string_udfs.py +++ b/python/cudf/cudf/tests/test_string_udfs.py @@ -96,7 +96,7 @@ def run_udf_test(data, func, dtype): else: result = output - got = cudf.Series(result, dtype=dtype) + got = cudf.Series._from_column(result.astype(dtype)) assert_eq(expect, got, check_dtype=False) with _CUDFNumbaConfig(): udf_str_kernel.forall(len(data))(str_views, output) @@ -105,7 +105,7 @@ def run_udf_test(data, func, dtype): else: result = output - got = cudf.Series(result, dtype=dtype) + got = cudf.Series._from_column(result.astype(dtype)) assert_eq(expect, got, check_dtype=False) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 4bdb5d921ec..2b1f745fc04 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -102,6 +102,7 @@ def _nest_list_data(data, leaf_type): @_dask_cudf_performance_tracking def _get_non_empty_data(s): + """Return a non empty column as metadata.""" if isinstance(s, cudf.core.column.CategoricalColumn): categories = ( s.categories if len(s.categories) else [UNKNOWN_CATEGORIES] @@ -128,7 +129,7 @@ def _get_non_empty_data(s): data = [{key: None for key in struct_dtype.fields.keys()}] * 2 data = cudf.core.column.as_column(data, dtype=s.dtype) elif is_string_dtype(s.dtype): - data = pa.array(["cat", "dog"]) + data = cudf.core.column.as_column(pa.array(["cat", "dog"])) elif isinstance(s.dtype, pd.DatetimeTZDtype): from cudf.utils.dtypes import get_time_unit @@ -153,7 +154,7 @@ def _nonempty_series(s, idx=None): idx = _nonempty_index(s.index) data = _get_non_empty_data(s._column) - return cudf.Series(data, name=s.name, index=idx) + return cudf.Series._from_column(data, name=s.name, index=idx) @meta_nonempty.register(cudf.DataFrame) @@ -424,7 +425,7 @@ def hash_object_cudf_index(ind, index=None): return ind.to_frame(index=False).hash_values() col = cudf.core.column.as_column(ind) - return cudf.Series(col).hash_values() + return cudf.Series._from_column(col).hash_values() @group_split_dispatch.register((cudf.Series, cudf.DataFrame)) diff --git a/python/dask_cudf/dask_cudf/core.py b/python/dask_cudf/dask_cudf/core.py index aab56e3a1b0..3181c8d69ec 100644 --- a/python/dask_cudf/dask_cudf/core.py +++ b/python/dask_cudf/dask_cudf/core.py @@ -342,7 +342,7 @@ def groupby(self, by=None, **kwargs): def sum_of_squares(x): x = x.astype("f8")._column outcol = libcudf.reduce.reduce("sum_of_squares", x) - return cudf.Series(outcol) + return cudf.Series._from_column(outcol) @_dask_cudf_performance_tracking From 3fd8783e49246f4ae61351375201d616d5ab6b55 Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Wed, 7 Aug 2024 13:00:09 -0700 Subject: [PATCH 032/270] Add `stream` param to stream compaction APIs (#16295) Add `stream` param to a bunch of stream compaction APIs. Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Nghia Truong (https://github.com/ttnghia) - Mark Harris (https://github.com/harrism) - Karthikeyan (https://github.com/karthikeyann) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/16295 --- cpp/include/cudf/detail/stream_compaction.hpp | 30 +- .../cudf/lists/detail/stream_compaction.hpp | 9 +- cpp/include/cudf/stream_compaction.hpp | 30 +- .../stream_compaction/apply_boolean_mask.cu | 3 +- cpp/src/stream_compaction/distinct.cu | 4 +- cpp/src/stream_compaction/distinct_count.cu | 11 +- cpp/src/stream_compaction/drop_nans.cu | 6 +- cpp/src/stream_compaction/drop_nulls.cu | 6 +- cpp/src/stream_compaction/unique.cu | 3 +- cpp/src/stream_compaction/unique_count.cu | 8 +- .../stream_compaction/unique_count_column.cu | 7 +- cpp/tests/streams/stream_compaction_test.cpp | 365 ++++++++++++++---- java/src/main/native/src/TableJni.cpp | 1 + 13 files changed, 362 insertions(+), 121 deletions(-) diff --git a/cpp/include/cudf/detail/stream_compaction.hpp b/cpp/include/cudf/detail/stream_compaction.hpp index 05194148a70..85d2ee9790f 100644 --- a/cpp/include/cudf/detail/stream_compaction.hpp +++ b/cpp/include/cudf/detail/stream_compaction.hpp @@ -29,9 +29,7 @@ namespace CUDF_EXPORT cudf { namespace detail { /** * @copydoc cudf::drop_nulls(table_view const&, std::vector const&, - * cudf::size_type, rmm::device_async_resource_ref) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * cudf::size_type, rmm::cuda_stream_view, rmm::device_async_resource_ref) */ std::unique_ptr
drop_nulls(table_view const& input, std::vector const& keys, @@ -41,9 +39,7 @@ std::unique_ptr
drop_nulls(table_view const& input, /** * @copydoc cudf::drop_nans(table_view const&, std::vector const&, - * cudf::size_type, rmm::device_async_resource_ref) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * cudf::size_type, rmm::cuda_stream_view, rmm::device_async_resource_ref) */ std::unique_ptr
drop_nans(table_view const& input, std::vector const& keys, @@ -53,8 +49,6 @@ std::unique_ptr
drop_nans(table_view const& input, /** * @copydoc cudf::apply_boolean_mask - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. */ std::unique_ptr
apply_boolean_mask(table_view const& input, column_view const& boolean_mask, @@ -63,8 +57,6 @@ std::unique_ptr
apply_boolean_mask(table_view const& input, /** * @copydoc cudf::unique - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. */ std::unique_ptr
unique(table_view const& input, std::vector const& keys, @@ -75,8 +67,6 @@ std::unique_ptr
unique(table_view const& input, /** * @copydoc cudf::distinct - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. */ std::unique_ptr
distinct(table_view const& input, std::vector const& keys, @@ -110,9 +100,7 @@ rmm::device_uvector distinct_indices(table_view const& input, rmm::device_async_resource_ref mr); /** - * @copydoc cudf::unique_count(column_view const&, null_policy, nan_policy) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * @copydoc cudf::unique_count(column_view const&, null_policy, nan_policy, rmm::cuda_stream_view) */ cudf::size_type unique_count(column_view const& input, null_policy null_handling, @@ -120,18 +108,14 @@ cudf::size_type unique_count(column_view const& input, rmm::cuda_stream_view stream); /** - * @copydoc cudf::unique_count(table_view const&, null_equality) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * @copydoc cudf::unique_count(table_view const&, null_equality, rmm::cuda_stream_view) */ cudf::size_type unique_count(table_view const& input, null_equality nulls_equal, rmm::cuda_stream_view stream); /** - * @copydoc cudf::distinct_count(column_view const&, null_policy, nan_policy) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * @copydoc cudf::distinct_count(column_view const&, null_policy, nan_policy, rmm::cuda_stream_view) */ cudf::size_type distinct_count(column_view const& input, null_policy null_handling, @@ -139,9 +123,7 @@ cudf::size_type distinct_count(column_view const& input, rmm::cuda_stream_view stream); /** - * @copydoc cudf::distinct_count(table_view const&, null_equality) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * @copydoc cudf::distinct_count(table_view const&, null_equality, rmm::cuda_stream_view) */ cudf::size_type distinct_count(table_view const& input, null_equality nulls_equal, diff --git a/cpp/include/cudf/lists/detail/stream_compaction.hpp b/cpp/include/cudf/lists/detail/stream_compaction.hpp index c11e07cd190..be0bd27083c 100644 --- a/cpp/include/cudf/lists/detail/stream_compaction.hpp +++ b/cpp/include/cudf/lists/detail/stream_compaction.hpp @@ -26,10 +26,7 @@ namespace CUDF_EXPORT cudf { namespace lists::detail { /** - * @copydoc cudf::lists::apply_boolean_mask(lists_column_view const&, lists_column_view const&, - * rmm::device_async_resource_ref) - * - * @param stream CUDA stream used for device memory operations and kernel launches + * @copydoc cudf::lists::apply_boolean_mask */ std::unique_ptr apply_boolean_mask(lists_column_view const& input, lists_column_view const& boolean_mask, @@ -37,9 +34,7 @@ std::unique_ptr apply_boolean_mask(lists_column_view const& input, rmm::device_async_resource_ref mr); /** - * @copydoc cudf::list::distinct - * - * @param stream CUDA stream used for device memory operations and kernel launches. + * @copydoc cudf::lists::distinct */ std::unique_ptr distinct(lists_column_view const& input, null_equality nulls_equal, diff --git a/cpp/include/cudf/stream_compaction.hpp b/cpp/include/cudf/stream_compaction.hpp index cfe404ff6ab..ced8d5849d0 100644 --- a/cpp/include/cudf/stream_compaction.hpp +++ b/cpp/include/cudf/stream_compaction.hpp @@ -67,6 +67,7 @@ namespace CUDF_EXPORT cudf { * @param[in] keys vector of indices representing key columns from `input` * @param[in] keep_threshold The minimum number of non-null fields in a row * required to keep the row. + * @param[in] stream CUDA stream used for device memory operations and kernel launches * @param[in] mr Device memory resource used to allocate the returned table's device memory * @return Table containing all rows of the `input` with at least @p * keep_threshold non-null fields in @p keys. @@ -75,6 +76,7 @@ std::unique_ptr
drop_nulls( table_view const& input, std::vector const& keys, cudf::size_type keep_threshold, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -99,6 +101,7 @@ std::unique_ptr
drop_nulls( * * @param[in] input The input `table_view` to filter * @param[in] keys vector of indices representing key columns from `input` + * @param[in] stream CUDA stream used for device memory operations and kernel launches * @param[in] mr Device memory resource used to allocate the returned table's device memory * @return Table containing all rows of the `input` without nulls in the columns * of @p keys. @@ -106,6 +109,7 @@ std::unique_ptr
drop_nulls( std::unique_ptr
drop_nulls( table_view const& input, std::vector const& keys, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -141,6 +145,7 @@ std::unique_ptr
drop_nulls( * @param[in] keys vector of indices representing key columns from `input` * @param[in] keep_threshold The minimum number of non-NAN elements in a row * required to keep the row. + * @param[in] stream CUDA stream used for device memory operations and kernel launches * @param[in] mr Device memory resource used to allocate the returned table's device memory * @return Table containing all rows of the `input` with at least @p * keep_threshold non-NAN elements in @p keys. @@ -149,6 +154,7 @@ std::unique_ptr
drop_nans( table_view const& input, std::vector const& keys, cudf::size_type keep_threshold, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -174,6 +180,7 @@ std::unique_ptr
drop_nans( * * @param[in] input The input `table_view` to filter * @param[in] keys vector of indices representing key columns from `input` + * @param[in] stream CUDA stream used for device memory operations and kernel launches * @param[in] mr Device memory resource used to allocate the returned table's device memory * @return Table containing all rows of the `input` without NANs in the columns * of @p keys. @@ -181,6 +188,7 @@ std::unique_ptr
drop_nans( std::unique_ptr
drop_nans( table_view const& input, std::vector const& keys, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -200,6 +208,7 @@ std::unique_ptr
drop_nans( * @param[in] input The input table_view to filter * @param[in] boolean_mask A nullable column_view of type type_id::BOOL8 used * as a mask to filter the `input`. + * @param[in] stream CUDA stream used for device memory operations and kernel launches * @param[in] mr Device memory resource used to allocate the returned table's device memory * @return Table containing copy of all rows of @p input passing * the filter defined by @p boolean_mask. @@ -207,6 +216,7 @@ std::unique_ptr
drop_nans( std::unique_ptr
apply_boolean_mask( table_view const& input, column_view const& boolean_mask, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -241,6 +251,7 @@ enum class duplicate_keep_option { * @param[in] keep keep any, first, last, or none of the found duplicates * @param[in] nulls_equal flag to denote nulls are equal if null_equality::EQUAL, nulls are not * equal if null_equality::UNEQUAL + * @param[in] stream CUDA stream used for device memory operations and kernel launches * @param[in] mr Device memory resource used to allocate the returned table's device * memory * @@ -251,6 +262,7 @@ std::unique_ptr
unique( std::vector const& keys, duplicate_keep_option keep, null_equality nulls_equal = null_equality::EQUAL, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -269,6 +281,7 @@ std::unique_ptr
unique( * @param keep Copy any, first, last, or none of the found duplicates * @param nulls_equal Flag to specify whether null elements should be considered as equal * @param nans_equal Flag to specify whether NaN elements should be considered as equal + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned table * @return Table with distinct rows in an unspecified order */ @@ -278,6 +291,7 @@ std::unique_ptr
distinct( duplicate_keep_option keep = duplicate_keep_option::KEEP_ANY, null_equality nulls_equal = null_equality::EQUAL, nan_equality nans_equal = nan_equality::ALL_EQUAL, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -346,12 +360,14 @@ std::unique_ptr
stable_distinct( * @param[in] input The column_view whose consecutive groups of equivalent rows will be counted * @param[in] null_handling flag to include or ignore `null` while counting * @param[in] nan_handling flag to consider `NaN==null` or not + * @param[in] stream CUDA stream used for device memory operations and kernel launches * * @return number of consecutive groups of equivalent rows in the column */ cudf::size_type unique_count(column_view const& input, null_policy null_handling, - nan_policy nan_handling); + nan_policy nan_handling, + rmm::cuda_stream_view stream = cudf::get_default_stream()); /** * @brief Count the number of consecutive groups of equivalent rows in a table. @@ -359,11 +375,13 @@ cudf::size_type unique_count(column_view const& input, * @param[in] input Table whose consecutive groups of equivalent rows will be counted * @param[in] nulls_equal flag to denote if null elements should be considered equal * nulls are not equal if null_equality::UNEQUAL. + * @param[in] stream CUDA stream used for device memory operations and kernel launches * * @return number of consecutive groups of equivalent rows in the column */ cudf::size_type unique_count(table_view const& input, - null_equality nulls_equal = null_equality::EQUAL); + null_equality nulls_equal = null_equality::EQUAL, + rmm::cuda_stream_view stream = cudf::get_default_stream()); /** * @brief Count the distinct elements in the column_view. @@ -382,12 +400,14 @@ cudf::size_type unique_count(table_view const& input, * @param[in] input The column_view whose distinct elements will be counted * @param[in] null_handling flag to include or ignore `null` while counting * @param[in] nan_handling flag to consider `NaN==null` or not + * @param[in] stream CUDA stream used for device memory operations and kernel launches * * @return number of distinct rows in the table */ cudf::size_type distinct_count(column_view const& input, null_policy null_handling, - nan_policy nan_handling); + nan_policy nan_handling, + rmm::cuda_stream_view stream = cudf::get_default_stream()); /** * @brief Count the distinct rows in a table. @@ -395,11 +415,13 @@ cudf::size_type distinct_count(column_view const& input, * @param[in] input Table whose distinct rows will be counted * @param[in] nulls_equal flag to denote if null elements should be considered equal. * nulls are not equal if null_equality::UNEQUAL. + * @param[in] stream CUDA stream used for device memory operations and kernel launches * * @return number of distinct rows in the table */ cudf::size_type distinct_count(table_view const& input, - null_equality nulls_equal = null_equality::EQUAL); + null_equality nulls_equal = null_equality::EQUAL, + rmm::cuda_stream_view stream = cudf::get_default_stream()); /** @} */ } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/stream_compaction/apply_boolean_mask.cu b/cpp/src/stream_compaction/apply_boolean_mask.cu index cdca9517d94..9812f4ffbd7 100644 --- a/cpp/src/stream_compaction/apply_boolean_mask.cu +++ b/cpp/src/stream_compaction/apply_boolean_mask.cu @@ -91,9 +91,10 @@ std::unique_ptr
apply_boolean_mask(table_view const& input, */ std::unique_ptr
apply_boolean_mask(table_view const& input, column_view const& boolean_mask, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::apply_boolean_mask(input, boolean_mask, cudf::get_default_stream(), mr); + return detail::apply_boolean_mask(input, boolean_mask, stream, mr); } } // namespace cudf diff --git a/cpp/src/stream_compaction/distinct.cu b/cpp/src/stream_compaction/distinct.cu index 6afd6e34c50..24e2692cb6f 100644 --- a/cpp/src/stream_compaction/distinct.cu +++ b/cpp/src/stream_compaction/distinct.cu @@ -150,11 +150,11 @@ std::unique_ptr
distinct(table_view const& input, duplicate_keep_option keep, null_equality nulls_equal, nan_equality nans_equal, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::distinct( - input, keys, keep, nulls_equal, nans_equal, cudf::get_default_stream(), mr); + return detail::distinct(input, keys, keep, nulls_equal, nans_equal, stream, mr); } std::unique_ptr distinct_indices(table_view const& input, diff --git a/cpp/src/stream_compaction/distinct_count.cu b/cpp/src/stream_compaction/distinct_count.cu index cdf9faddf31..78eb0fa5212 100644 --- a/cpp/src/stream_compaction/distinct_count.cu +++ b/cpp/src/stream_compaction/distinct_count.cu @@ -218,15 +218,18 @@ cudf::size_type distinct_count(column_view const& input, cudf::size_type distinct_count(column_view const& input, null_policy null_handling, - nan_policy nan_handling) + nan_policy nan_handling, + rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - return detail::distinct_count(input, null_handling, nan_handling, cudf::get_default_stream()); + return detail::distinct_count(input, null_handling, nan_handling, stream); } -cudf::size_type distinct_count(table_view const& input, null_equality nulls_equal) +cudf::size_type distinct_count(table_view const& input, + null_equality nulls_equal, + rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - return detail::distinct_count(input, nulls_equal, cudf::get_default_stream()); + return detail::distinct_count(input, nulls_equal, stream); } } // namespace cudf diff --git a/cpp/src/stream_compaction/drop_nans.cu b/cpp/src/stream_compaction/drop_nans.cu index b46381c8ff6..b98ebbc2ecc 100644 --- a/cpp/src/stream_compaction/drop_nans.cu +++ b/cpp/src/stream_compaction/drop_nans.cu @@ -117,20 +117,22 @@ std::unique_ptr
drop_nans(table_view const& input, std::unique_ptr
drop_nans(table_view const& input, std::vector const& keys, cudf::size_type keep_threshold, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::drop_nans(input, keys, keep_threshold, cudf::get_default_stream(), mr); + return detail::drop_nans(input, keys, keep_threshold, stream, mr); } /* * Filters a table to remove nan elements. */ std::unique_ptr
drop_nans(table_view const& input, std::vector const& keys, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::drop_nans(input, keys, keys.size(), cudf::get_default_stream(), mr); + return detail::drop_nans(input, keys, keys.size(), stream, mr); } } // namespace cudf diff --git a/cpp/src/stream_compaction/drop_nulls.cu b/cpp/src/stream_compaction/drop_nulls.cu index cb7cd61bf02..2497e4e5065 100644 --- a/cpp/src/stream_compaction/drop_nulls.cu +++ b/cpp/src/stream_compaction/drop_nulls.cu @@ -90,20 +90,22 @@ std::unique_ptr
drop_nulls(table_view const& input, std::unique_ptr
drop_nulls(table_view const& input, std::vector const& keys, cudf::size_type keep_threshold, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::drop_nulls(input, keys, keep_threshold, cudf::get_default_stream(), mr); + return detail::drop_nulls(input, keys, keep_threshold, stream, mr); } /* * Filters a table to remove null elements. */ std::unique_ptr
drop_nulls(table_view const& input, std::vector const& keys, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::drop_nulls(input, keys, keys.size(), cudf::get_default_stream(), mr); + return detail::drop_nulls(input, keys, keys.size(), stream, mr); } } // namespace cudf diff --git a/cpp/src/stream_compaction/unique.cu b/cpp/src/stream_compaction/unique.cu index edb47984d13..93de0e60b6d 100644 --- a/cpp/src/stream_compaction/unique.cu +++ b/cpp/src/stream_compaction/unique.cu @@ -119,10 +119,11 @@ std::unique_ptr
unique(table_view const& input, std::vector const& keys, duplicate_keep_option const keep, null_equality nulls_equal, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::unique(input, keys, keep, nulls_equal, cudf::get_default_stream(), mr); + return detail::unique(input, keys, keep, nulls_equal, stream, mr); } } // namespace cudf diff --git a/cpp/src/stream_compaction/unique_count.cu b/cpp/src/stream_compaction/unique_count.cu index 19607fe8105..d842f63cd7b 100644 --- a/cpp/src/stream_compaction/unique_count.cu +++ b/cpp/src/stream_compaction/unique_count.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,10 +67,12 @@ cudf::size_type unique_count(table_view const& keys, } // namespace detail -cudf::size_type unique_count(table_view const& input, null_equality nulls_equal) +cudf::size_type unique_count(table_view const& input, + null_equality nulls_equal, + rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - return detail::unique_count(input, nulls_equal, cudf::get_default_stream()); + return detail::unique_count(input, nulls_equal, stream); } } // namespace cudf diff --git a/cpp/src/stream_compaction/unique_count_column.cu b/cpp/src/stream_compaction/unique_count_column.cu index 16758b6e3a7..89ce2391a7b 100644 --- a/cpp/src/stream_compaction/unique_count_column.cu +++ b/cpp/src/stream_compaction/unique_count_column.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -101,10 +101,11 @@ cudf::size_type unique_count(column_view const& input, cudf::size_type unique_count(column_view const& input, null_policy null_handling, - nan_policy nan_handling) + nan_policy nan_handling, + rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - return detail::unique_count(input, null_handling, nan_handling, cudf::get_default_stream()); + return detail::unique_count(input, null_handling, nan_handling, stream); } } // namespace cudf diff --git a/cpp/tests/streams/stream_compaction_test.cpp b/cpp/tests/streams/stream_compaction_test.cpp index 56443870602..443f4548b2c 100644 --- a/cpp/tests/streams/stream_compaction_test.cpp +++ b/cpp/tests/streams/stream_compaction_test.cpp @@ -41,6 +41,7 @@ auto constexpr NULL_UNEQUAL = cudf::null_equality::UNEQUAL; auto constexpr NAN_EQUAL = cudf::nan_equality::ALL_EQUAL; auto constexpr NAN_UNEQUAL = cudf::nan_equality::UNEQUAL; +using int16s_col = cudf::test::fixed_width_column_wrapper; using int32s_col = cudf::test::fixed_width_column_wrapper; using floats_col = cudf::test::fixed_width_column_wrapper; @@ -51,50 +52,9 @@ using cudf::test::iterators::no_nulls; using cudf::test::iterators::null_at; using cudf::test::iterators::nulls_at; -struct StableDistinctKeepAny : public cudf::test::BaseFixture {}; +struct StreamCompactionTest : public cudf::test::BaseFixture {}; -struct StableDistinctKeepFirstLastNone : public cudf::test::BaseFixture {}; - -TEST_F(StableDistinctKeepAny, NoNullsTableWithNaNs) -{ - // Column(s) used to test KEEP_ANY needs to have same rows in contiguous - // groups for equivalent keys because KEEP_ANY is nondeterministic. - auto const col1 = int32s_col{6, 6, 6, 1, 1, 1, 3, 5, 8, 5}; - auto const col2 = floats_col{6, 6, 6, 1, 1, 1, 3, 4, 9, 4}; - auto const keys1 = int32s_col{20, 20, 20, 15, 15, 15, 20, 19, 21, 9}; - auto const keys2 = floats_col{19., 19., 19., NaN, NaN, NaN, 20., 20., 9., 21.}; - - auto const input = cudf::table_view{{col1, col2, keys1, keys2}}; - auto const key_idx = std::vector{2, 3}; - - // NaNs are unequal. - { - auto const exp_col1 = int32s_col{6, 1, 1, 1, 3, 5, 8, 5}; - auto const exp_col2 = floats_col{6, 1, 1, 1, 3, 4, 9, 4}; - auto const exp_keys1 = int32s_col{20, 15, 15, 15, 20, 19, 21, 9}; - auto const exp_keys2 = floats_col{19., NaN, NaN, NaN, 20., 20., 9., 21.}; - auto const expected = cudf::table_view{{exp_col1, exp_col2, exp_keys1, exp_keys2}}; - - auto const result = cudf::stable_distinct( - input, key_idx, KEEP_ANY, NULL_EQUAL, NAN_UNEQUAL, cudf::test::get_default_stream()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, *result); - } - - // NaNs are equal. - { - auto const exp_col1 = int32s_col{6, 1, 3, 5, 8, 5}; - auto const exp_col2 = floats_col{6, 1, 3, 4, 9, 4}; - auto const exp_keys1 = int32s_col{20, 15, 20, 19, 21, 9}; - auto const exp_keys2 = floats_col{19., NaN, 20., 20., 9., 21.}; - auto const expected = cudf::table_view{{exp_col1, exp_col2, exp_keys1, exp_keys2}}; - - auto const result = cudf::stable_distinct( - input, key_idx, KEEP_ANY, NULL_EQUAL, NAN_EQUAL, cudf::test::get_default_stream()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, *result); - } -} - -TEST_F(StableDistinctKeepAny, InputWithNullsAndNaNs) +TEST_F(StreamCompactionTest, StableDistinctKeepAny) { auto constexpr null{0.0}; // shadow the global `null` variable of type int @@ -150,7 +110,7 @@ TEST_F(StableDistinctKeepAny, InputWithNullsAndNaNs) } } -TEST_F(StableDistinctKeepFirstLastNone, InputWithNaNsEqual) +TEST_F(StreamCompactionTest, StableDistinctKeepFirstLastNone) { // Column(s) used to test needs to have different rows for the same keys. auto const col = int32s_col{0, 1, 2, 3, 4, 5, 6}; @@ -192,44 +152,313 @@ TEST_F(StableDistinctKeepFirstLastNone, InputWithNaNsEqual) } } -TEST_F(StableDistinctKeepFirstLastNone, InputWithNaNsUnequal) +TEST_F(StreamCompactionTest, DropNaNs) { - // Column(s) used to test needs to have different rows for the same keys. - auto const col = int32s_col{0, 1, 2, 3, 4, 5, 6, 7}; - auto const keys = floats_col{20., NaN, NaN, 19., 21., 19., 22., 20.}; - auto const input = cudf::table_view{{col, keys}}; - auto const key_idx = std::vector{1}; + auto const col1 = floats_col{{1., 2., NaN, NaN, 5., 6.}, nulls_at({2, 5})}; + auto const col2 = int32s_col{{10, 40, 70, 5, 2, 10}, nulls_at({2, 5})}; + auto const col3 = floats_col{{NaN, 40., 70., NaN, 2., 10.}, nulls_at({2, 5})}; + cudf::table_view input{{col1, col2, col3}}; + + std::vector keys{0, 2}; - // KEEP_FIRST { - auto const exp_col = int32s_col{0, 1, 2, 3, 4, 6}; - auto const exp_keys = floats_col{20., NaN, NaN, 19., 21., 22.}; - auto const expected = cudf::table_view{{exp_col, exp_keys}}; + // With keep_threshold + auto const col1_expected = floats_col{{1., 2., 3., 5., 6.}, nulls_at({2, 4})}; + auto const col2_expected = int32s_col{{10, 40, 70, 2, 10}, nulls_at({2, 4})}; + auto const col3_expected = floats_col{{NaN, 40., 70., 2., 10.}, nulls_at({2, 4})}; + cudf::table_view expected{{col1_expected, col2_expected, col3_expected}}; + + auto result = cudf::drop_nans(input, keys, keys.size() - 1, cudf::test::get_default_stream()); - auto const result = cudf::stable_distinct( - input, key_idx, KEEP_FIRST, NULL_UNEQUAL, NAN_UNEQUAL, cudf::test::get_default_stream()); CUDF_TEST_EXPECT_TABLES_EQUAL(expected, *result); } - // KEEP_LAST { - auto const exp_col = int32s_col{1, 2, 4, 5, 6, 7}; - auto const exp_keys = floats_col{NaN, NaN, 21., 19., 22., 20.}; - auto const expected = cudf::table_view{{exp_col, exp_keys}}; + // Without keep_threshold + auto const col1_expected = floats_col{{2., 3., 5., 6.}, nulls_at({1, 3})}; + auto const col2_expected = int32s_col{{40, 70, 2, 10}, nulls_at({1, 3})}; + auto const col3_expected = floats_col{{40., 70., 2., 10.}, nulls_at({1, 3})}; + cudf::table_view expected{{col1_expected, col2_expected, col3_expected}}; + + auto result = cudf::drop_nans(input, keys, cudf::test::get_default_stream()); - auto const result = cudf::stable_distinct( - input, key_idx, KEEP_LAST, NULL_UNEQUAL, NAN_UNEQUAL, cudf::test::get_default_stream()); CUDF_TEST_EXPECT_TABLES_EQUAL(expected, *result); } +} + +TEST_F(StreamCompactionTest, DropNulls) +{ + auto const col1 = int16s_col{{1, 0, 1, 0, 1, 0}, nulls_at({2, 5})}; + auto const col2 = int32s_col{{10, 40, 70, 5, 2, 10}, nulls_at({2})}; + auto const col3 = floats_col{{10., 40., 70., 5., 2., 10.}, no_nulls()}; + cudf::table_view input{{col1, col2, col3}}; + std::vector keys{0, 1, 2}; - // KEEP_NONE { - auto const exp_col = int32s_col{1, 2, 4, 6}; - auto const exp_keys = floats_col{NaN, NaN, 21., 22.}; - auto const expected = cudf::table_view{{exp_col, exp_keys}}; + // With keep_threshold + auto const col1_expected = int16s_col{{1, 0, 0, 1, 0}, null_at(4)}; + auto const col2_expected = int32s_col{{10, 40, 5, 2, 10}, no_nulls()}; + auto const col3_expected = floats_col{{10., 40., 5., 2., 10.}, no_nulls()}; + cudf::table_view expected{{col1_expected, col2_expected, col3_expected}}; + + auto result = cudf::drop_nulls(input, keys, keys.size() - 1, cudf::test::get_default_stream()); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, *result); + } + + { + // Without keep_threshold + auto const col1_expected = int16s_col{{1, 0, 0, 1}, no_nulls()}; + auto const col2_expected = int32s_col{{10, 40, 5, 2}, no_nulls()}; + auto const col3_expected = floats_col{{10., 40., 5., 2.}, no_nulls()}; + cudf::table_view expected{{col1_expected, col2_expected, col3_expected}}; + + auto result = cudf::drop_nulls(input, keys, cudf::test::get_default_stream()); - auto const result = cudf::stable_distinct( - input, key_idx, KEEP_NONE, NULL_UNEQUAL, NAN_UNEQUAL, cudf::test::get_default_stream()); CUDF_TEST_EXPECT_TABLES_EQUAL(expected, *result); } } + +TEST_F(StreamCompactionTest, Unique) +{ + auto const col1 = int32s_col{5, 4, 3, 5, 8, 5}; + auto const col2 = floats_col{4., 5., 3., 4., 9., 4.}; + auto const col1_key = int32s_col{20, 20, 20, 19, 21, 9}; + auto const col2_key = int32s_col{19, 19, 20, 20, 9, 21}; + + cudf::table_view input{{col1, col2, col1_key, col2_key}}; + std::vector keys = {2, 3}; + + { + // KEEP_FIRST + auto const exp_col1_first = int32s_col{5, 3, 5, 8, 5}; + auto const exp_col2_first = floats_col{4., 3., 4., 9., 4.}; + auto const exp_col1_key_first = int32s_col{20, 20, 19, 21, 9}; + auto const exp_col2_key_first = int32s_col{19, 20, 20, 9, 21}; + cudf::table_view expected_first{ + {exp_col1_first, exp_col2_first, exp_col1_key_first, exp_col2_key_first}}; + + auto const result = cudf::unique(input, + keys, + cudf::duplicate_keep_option::KEEP_FIRST, + cudf::null_equality::EQUAL, + cudf::test::get_default_stream()); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_first, *result); + } + + { + // KEEP_LAST + auto const exp_col1_last = int32s_col{4, 3, 5, 8, 5}; + auto const exp_col2_last = floats_col{5., 3., 4., 9., 4.}; + auto const exp_col1_key_last = int32s_col{20, 20, 19, 21, 9}; + auto const exp_col2_key_last = int32s_col{19, 20, 20, 9, 21}; + cudf::table_view expected_last{ + {exp_col1_last, exp_col2_last, exp_col1_key_last, exp_col2_key_last}}; + + auto const result = cudf::unique(input, + keys, + cudf::duplicate_keep_option::KEEP_LAST, + cudf::null_equality::EQUAL, + cudf::test::get_default_stream()); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_last, *result); + } + + { + // KEEP_NONE + auto const exp_col1_unique = int32s_col{3, 5, 8, 5}; + auto const exp_col2_unique = floats_col{3., 4., 9., 4.}; + auto const exp_col1_key_unique = int32s_col{20, 19, 21, 9}; + auto const exp_col2_key_unique = int32s_col{20, 20, 9, 21}; + cudf::table_view expected_unique{ + {exp_col1_unique, exp_col2_unique, exp_col1_key_unique, exp_col2_key_unique}}; + + auto const result = cudf::unique(input, + keys, + cudf::duplicate_keep_option::KEEP_NONE, + cudf::null_equality::EQUAL, + cudf::test::get_default_stream()); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_unique, *result); + } +} + +TEST_F(StreamCompactionTest, Distinct) +{ + // Column(s) used to test needs to have different rows for the same keys. + auto const col1 = int32s_col{0, 1, 2, 3, 4, 5, 6}; + auto const col2 = floats_col{10, 11, 12, 13, 14, 15, 16}; + auto const keys1 = int32s_col{20, 20, 20, 20, 19, 21, 9}; + auto const keys2 = int32s_col{19, 19, 19, 20, 20, 9, 21}; + + auto const input = cudf::table_view{{col1, col2, keys1, keys2}}; + auto const key_idx = std::vector{2, 3}; + + // KEEP_FIRST + { + auto const exp_col1_sort = int32s_col{6, 4, 0, 3, 5}; + auto const exp_col2_sort = floats_col{16, 14, 10, 13, 15}; + auto const exp_keys1_sort = int32s_col{9, 19, 20, 20, 21}; + auto const exp_keys2_sort = int32s_col{21, 20, 19, 20, 9}; + auto const expected_sort = + cudf::table_view{{exp_col1_sort, exp_col2_sort, exp_keys1_sort, exp_keys2_sort}}; + + auto const result = cudf::distinct(input, + key_idx, + cudf::duplicate_keep_option::KEEP_FIRST, + cudf::null_equality::EQUAL, + cudf::nan_equality::ALL_EQUAL, + cudf::test::get_default_stream()); + auto const result_sort = + cudf::sort_by_key(*result, result->select(key_idx), {}, {}, cudf::test::get_default_stream()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_sort, *result_sort); + } + + // KEEP_LAST + { + auto const exp_col1_sort = int32s_col{6, 4, 2, 3, 5}; + auto const exp_col2_sort = floats_col{16, 14, 12, 13, 15}; + auto const exp_keys1_sort = int32s_col{9, 19, 20, 20, 21}; + auto const exp_keys2_sort = int32s_col{21, 20, 19, 20, 9}; + auto const expected_sort = + cudf::table_view{{exp_col1_sort, exp_col2_sort, exp_keys1_sort, exp_keys2_sort}}; + + auto const result = cudf::distinct(input, + key_idx, + cudf::duplicate_keep_option::KEEP_LAST, + cudf::null_equality::EQUAL, + cudf::nan_equality::ALL_EQUAL, + cudf::test::get_default_stream()); + auto const result_sort = + cudf::sort_by_key(*result, result->select(key_idx), {}, {}, cudf::test::get_default_stream()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_sort, *result_sort); + } + + // KEEP_NONE + { + auto const exp_col1_sort = int32s_col{6, 4, 3, 5}; + auto const exp_col2_sort = floats_col{16, 14, 13, 15}; + auto const exp_keys1_sort = int32s_col{9, 19, 20, 21}; + auto const exp_keys2_sort = int32s_col{21, 20, 20, 9}; + auto const expected_sort = + cudf::table_view{{exp_col1_sort, exp_col2_sort, exp_keys1_sort, exp_keys2_sort}}; + + auto const result = cudf::distinct(input, + key_idx, + cudf::duplicate_keep_option::KEEP_NONE, + cudf::null_equality::EQUAL, + cudf::nan_equality::ALL_EQUAL, + cudf::test::get_default_stream()); + auto const result_sort = + cudf::sort_by_key(*result, result->select(key_idx), {}, {}, cudf::test::get_default_stream()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_sort, *result_sort); + } +} + +TEST_F(StreamCompactionTest, ApplyBooleanMask) +{ + auto const col = int32s_col{ + 9668, 9590, 9526, 9205, 9434, 9347, 9160, 9569, 9143, 9807, 9606, 9446, 9279, 9822, 9691}; + cudf::test::fixed_width_column_wrapper mask({false, + false, + true, + false, + false, + true, + false, + true, + false, + true, + false, + false, + true, + false, + true}); + cudf::table_view input({col}); + auto const col_expected = int32s_col{9526, 9347, 9569, 9807, 9279, 9691}; + cudf::table_view expected({col_expected}); + auto const result = cudf::apply_boolean_mask(input, mask, cudf::test::get_default_stream()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, *result); +} + +TEST_F(StreamCompactionTest, UniqueCountColumn) +{ + std::vector const input = {1, 3, 3, 4, 31, 1, 8, 2, 0, 4, 1, + 4, 10, 40, 31, 42, 0, 42, 8, 5, 4}; + + cudf::test::fixed_width_column_wrapper input_col(input.begin(), input.end()); + std::vector input_data(input.begin(), input.end()); + + auto const new_end = std::unique(input_data.begin(), input_data.end()); + auto const expected = std::distance(input_data.begin(), new_end); + EXPECT_EQ( + expected, + cudf::unique_count( + input_col, null_policy::INCLUDE, nan_policy::NAN_IS_VALID, cudf::test::get_default_stream())); +} + +TEST_F(StreamCompactionTest, UniqueCountTable) +{ + std::vector const input1 = {1, 3, 3, 3, 4, 31, 1, 8, 2, 0, 4, + 1, 4, 10, 40, 31, 42, 0, 42, 8, 5, 4}; + std::vector const input2 = {3, 3, 3, 4, 31, 1, 8, 5, 0, 4, 1, + 4, 10, 40, 31, 42, 0, 42, 8, 5, 4, 1}; + + std::vector> pair_input; + std::transform(input1.begin(), + input1.end(), + input2.begin(), + std::back_inserter(pair_input), + [](int32_t a, int32_t b) { return std::pair(a, b); }); + + cudf::test::fixed_width_column_wrapper input_col1(input1.begin(), input1.end()); + cudf::test::fixed_width_column_wrapper input_col2(input2.begin(), input2.end()); + cudf::table_view input_table({input_col1, input_col2}); + + auto const new_end = std::unique(pair_input.begin(), pair_input.end()); + auto const result = std::distance(pair_input.begin(), new_end); + EXPECT_EQ( + result, + cudf::unique_count(input_table, null_equality::EQUAL, cudf::test::get_default_stream())); +} + +TEST_F(StreamCompactionTest, DistinctCountColumn) +{ + std::vector const input = {1, 3, 3, 4, 31, 1, 8, 2, 0, 4, 1, + 4, 10, 40, 31, 42, 0, 42, 8, 5, 4}; + + cudf::test::fixed_width_column_wrapper input_col(input.begin(), input.end()); + + auto const expected = + static_cast(std::set(input.begin(), input.end()).size()); + EXPECT_EQ( + expected, + cudf::distinct_count( + input_col, null_policy::INCLUDE, nan_policy::NAN_IS_VALID, cudf::test::get_default_stream())); +} + +TEST_F(StreamCompactionTest, DistinctCountTable) +{ + std::vector const input1 = {1, 3, 3, 3, 4, 31, 1, 8, 2, 0, 4, + 1, 4, 10, 40, 31, 42, 0, 42, 8, 5, 4}; + std::vector const input2 = {3, 3, 3, 4, 31, 1, 8, 5, 0, 4, 1, + 4, 10, 40, 31, 42, 0, 42, 8, 5, 4, 1}; + + std::vector> pair_input; + std::transform(input1.begin(), + input1.end(), + input2.begin(), + std::back_inserter(pair_input), + [](int32_t a, int32_t b) { return std::pair(a, b); }); + + cudf::test::fixed_width_column_wrapper input_col1(input1.begin(), input1.end()); + cudf::test::fixed_width_column_wrapper input_col2(input2.begin(), input2.end()); + cudf::table_view input_table({input_col1, input_col2}); + + auto const expected = static_cast( + std::set>(pair_input.begin(), pair_input.end()).size()); + EXPECT_EQ( + expected, + cudf::distinct_count(input_table, null_equality::EQUAL, cudf::test::get_default_stream())); +} diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index a9ace1398e4..76ca8c533ce 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -3919,6 +3919,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_dropDuplicates( keep_option, nulls_equal ? cudf::null_equality::EQUAL : cudf::null_equality::UNEQUAL, cudf::nan_equality::ALL_EQUAL, + cudf::get_default_stream(), rmm::mr::get_current_device_resource()); return convert_table_for_return(env, result); } From b933b54858a84082980f20522738fda4969a1318 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 7 Aug 2024 20:07:42 -0500 Subject: [PATCH 033/270] Use tool.scikit-build.cmake.version, set scikit-build-core minimum-version (#16503) Contributes to https://github.com/rapidsai/build-planning/issues/58. `scikit-build-core==0.10.0` was released today (https://github.com/scikit-build/scikit-build-core/releases/tag/v0.10.0), and wheel-building configurations across RAPIDS are incompatible with it. This proposes upgrading to that version and fixing configuration here in a way that: * is compatible with that new `scikit-build-core` version * takes advantage of the forward-compatibility mechanism (`minimum-version`) that `scikit-build-core` provides, to reduce the risk of needing to do this again in the future Authors: - James Lamb (https://github.com/jameslamb) Approvers: - https://github.com/jakirkham URL: https://github.com/rapidsai/cudf/pull/16503 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/cudf_kafka/meta.yaml | 2 +- dependencies.yaml | 4 ++-- python/cudf/pyproject.toml | 5 +++-- python/cudf_kafka/pyproject.toml | 5 +++-- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d04804cafaf..8d5fc2e31d9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -82,7 +82,7 @@ dependencies: - rich - rmm==24.10.*,>=0.0.0a0 - s3fs>=2022.3.0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - scipy - spdlog>=1.12.0,<1.13 - sphinx diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index e2c3558030d..7b0485d7f29 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -80,7 +80,7 @@ dependencies: - rich - rmm==24.10.*,>=0.0.0a0 - s3fs>=2022.3.0 -- scikit-build-core>=0.7.0 +- scikit-build-core>=0.10.0 - scipy - spdlog>=1.12.0,<1.13 - sphinx diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 9137f099ad1..8d7ef63715b 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -62,7 +62,7 @@ requirements: - python - cython >=3.0.3 - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - scikit-build-core >=0.7.0 + - scikit-build-core >=0.10.0 - dlpack >=0.8,<1.0 # TODO: Change to `2.0` for NumPy 2 - numpy 1.23 diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index 1b0e0e2c236..748a32e5518 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -61,7 +61,7 @@ requirements: - cudf ={{ version }} - libcudf_kafka ={{ version }} - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - scikit-build-core >=0.7.0 + - scikit-build-core >=0.10.0 {% if cuda_major != "11" %} - cuda-cudart-dev {% endif %} diff --git a/dependencies.yaml b/dependencies.yaml index abb55a5e011..b0d62a9fb0d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -301,10 +301,10 @@ dependencies: - &rapids_build_backend rapids-build-backend>=0.3.0,<0.4.0.dev0 - output_types: conda packages: - - scikit-build-core>=0.7.0 + - scikit-build-core>=0.10.0 - output_types: [requirements, pyproject] packages: - - scikit-build-core[pyproject]>=0.7.0 + - scikit-build-core[pyproject]>=0.10.0 rapids_build_setuptools: common: - output_types: [requirements, pyproject] diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index b2ddb06d8c9..60ac171f3d7 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "rapids_build_backend.build" requires = [ "rapids-build-backend>=0.3.0,<0.4.0.dev0", - "scikit-build-core[pyproject]>=0.7.0", + "scikit-build-core[pyproject]>=0.10.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] @@ -133,7 +133,8 @@ requires = [ [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" -cmake.minimum-version = "3.26.4" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index a9b60133f42..63c5b07c5f3 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "rapids_build_backend.build" requires = [ "rapids-build-backend>=0.3.0,<0.4.0.dev0", - "scikit-build-core[pyproject]>=0.7.0", + "scikit-build-core[pyproject]>=0.10.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] @@ -86,7 +86,8 @@ filterwarnings = [ [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" -cmake.minimum-version = "3.26.4" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true From c146eed6f36e7c82052a3288e1bf6ab8c2216637 Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Wed, 7 Aug 2024 22:19:46 -0700 Subject: [PATCH 034/270] Expose `stream` param in transform APIs (#16452) Exposes the `stream` param in transform APIs Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) Approvers: - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/16452 --- cpp/include/cudf/transform.hpp | 21 +++- cpp/src/interop/to_arrow.cu | 2 +- cpp/src/interop/to_arrow_device.cu | 4 +- cpp/src/interop/to_arrow_host.cu | 2 +- cpp/src/transform/bools_to_mask.cu | 4 +- cpp/src/transform/compute_column.cu | 3 +- cpp/src/transform/encode.cu | 4 +- cpp/src/transform/mask_to_bools.cu | 3 +- cpp/src/transform/nans_to_nulls.cu | 4 +- cpp/src/transform/one_hot_encode.cu | 3 +- cpp/src/transform/row_bit_count.cu | 11 +- cpp/src/transform/transform.cpp | 3 +- cpp/tests/CMakeLists.txt | 1 + cpp/tests/streams/transform_test.cpp | 164 +++++++++++++++++++++++++++ 14 files changed, 210 insertions(+), 19 deletions(-) create mode 100644 cpp/tests/streams/transform_test.cpp diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index adc5bdb2af8..f16214260f7 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -47,6 +47,7 @@ namespace CUDF_EXPORT cudf { * @param unary_udf The PTX/CUDA string of the unary function to apply * @param output_type The output type that is compatible with the output type in the UDF * @param is_ptx true: the UDF is treated as PTX code; false: the UDF is treated as CUDA code + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory * @return The column resulting from applying the unary function to * every element of the input @@ -56,6 +57,7 @@ std::unique_ptr transform( std::string const& unary_udf, data_type output_type, bool is_ptx, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -65,12 +67,14 @@ std::unique_ptr transform( * @throws cudf::logic_error if `input.type()` is a non-floating type * * @param input An immutable view of the input column of floating-point type + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned bitmask * @return A pair containing a `device_buffer` with the new bitmask and it's * null count obtained by replacing `NaN` in `input` with null. */ std::pair, size_type> nans_to_nulls( column_view const& input, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -83,12 +87,14 @@ std::pair, size_type> nans_to_nulls( * * @param table The table used for expression evaluation * @param expr The root of the expression tree + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource * @return Output column */ std::unique_ptr compute_column( table_view const& table, ast::expression const& expr, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -101,6 +107,7 @@ std::unique_ptr compute_column( * @throws cudf::logic_error if `input.type()` is a non-boolean type * * @param input Boolean elements to convert to a bitmask + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned bitmask * @return A pair containing a `device_buffer` with the new bitmask and it's * null count obtained from input considering `true` represent `valid`/`1` and @@ -108,6 +115,7 @@ std::unique_ptr compute_column( */ std::pair, cudf::size_type> bools_to_mask( column_view const& input, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -130,12 +138,14 @@ std::pair, cudf::size_type> bools_to_mask( * @endcode * * @param input Table containing values to be encoded + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned table's device memory * @return A pair containing the distinct row of the input table in sorter order, * and a column of integer indices representing the encoded rows. */ std::pair, std::unique_ptr> encode( cudf::table_view const& input, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -162,12 +172,14 @@ std::pair, std::unique_ptr> encode( * * @param input Column containing values to be encoded * @param categories Column containing categories + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned table's device memory * @return A pair containing the owner to all encoded data and a table view into the data */ std::pair, table_view> one_hot_encode( column_view const& input, column_view const& categories, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -188,6 +200,7 @@ std::pair, table_view> one_hot_encode( * @param bitmask A device pointer to the bitmask which needs to be converted * @param begin_bit position of the bit from which the conversion should start * @param end_bit position of the bit before which the conversion should stop + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned columns' device memory * @return A boolean column representing the given mask from [begin_bit, end_bit) */ @@ -195,6 +208,7 @@ std::unique_ptr mask_to_bools( bitmask_type const* bitmask, size_type begin_bit, size_type end_bit, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @@ -219,11 +233,14 @@ std::unique_ptr mask_to_bools( * row_bit_count(column(x)) >= row_bit_count(gather(column(x))) * * @param t The table view to perform the computation on + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned columns' device memory * @return A 32-bit integer column containing the per-row bit counts */ std::unique_ptr row_bit_count( - table_view const& t, rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + table_view const& t, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** * @brief Returns an approximate cumulative size in bits of all columns in the `table_view` for @@ -240,12 +257,14 @@ std::unique_ptr row_bit_count( * * @param t The table view to perform the computation on * @param segment_length The number of rows in each segment for which the total size is computed + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned columns' device memory * @return A 32-bit integer column containing the bit counts for each segment of rows */ std::unique_ptr segmented_row_bit_count( table_view const& t, size_type segment_length, + rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); /** @} */ // end of group diff --git a/cpp/src/interop/to_arrow.cu b/cpp/src/interop/to_arrow.cu index 3d41f856f4f..a867d4adfa1 100644 --- a/cpp/src/interop/to_arrow.cu +++ b/cpp/src/interop/to_arrow.cu @@ -247,7 +247,7 @@ std::shared_ptr dispatch_to_arrow::operator()(column_view in arrow::MemoryPool* ar_mr, rmm::cuda_stream_view stream) { - auto bitmask = bools_to_mask(input, stream, rmm::mr::get_current_device_resource()); + auto bitmask = detail::bools_to_mask(input, stream, rmm::mr::get_current_device_resource()); auto data_buffer = allocate_arrow_buffer(static_cast(bitmask.first->size()), ar_mr); diff --git a/cpp/src/interop/to_arrow_device.cu b/cpp/src/interop/to_arrow_device.cu index cea7cdebcba..a5f3f9d87f5 100644 --- a/cpp/src/interop/to_arrow_device.cu +++ b/cpp/src/interop/to_arrow_device.cu @@ -200,7 +200,7 @@ int dispatch_to_arrow_device::operator()(cudf::column&& column, nanoarrow::UniqueArray tmp; NANOARROW_RETURN_NOT_OK(initialize_array(tmp.get(), NANOARROW_TYPE_BOOL, column)); - auto bitmask = bools_to_mask(column.view(), stream, mr); + auto bitmask = detail::bools_to_mask(column.view(), stream, mr); auto contents = column.release(); NANOARROW_RETURN_NOT_OK(set_null_mask(contents, tmp.get())); NANOARROW_RETURN_NOT_OK( @@ -442,7 +442,7 @@ int dispatch_to_arrow_device_view::operator()(ArrowArray* out) const nanoarrow::UniqueArray tmp; NANOARROW_RETURN_NOT_OK(initialize_array(tmp.get(), NANOARROW_TYPE_BOOL, column)); - auto bitmask = bools_to_mask(column, stream, mr); + auto bitmask = detail::bools_to_mask(column, stream, mr); NANOARROW_RETURN_NOT_OK( set_buffer(std::move(bitmask.first), fixed_width_data_buffer_idx, tmp.get())); NANOARROW_RETURN_NOT_OK(set_null_mask(column, tmp.get())); diff --git a/cpp/src/interop/to_arrow_host.cu b/cpp/src/interop/to_arrow_host.cu index 193b3a3b5a2..26f7c7e6e53 100644 --- a/cpp/src/interop/to_arrow_host.cu +++ b/cpp/src/interop/to_arrow_host.cu @@ -147,7 +147,7 @@ int dispatch_to_arrow_host::operator()(ArrowArray* out) const NANOARROW_RETURN_NOT_OK(initialize_array(tmp.get(), NANOARROW_TYPE_BOOL, column)); NANOARROW_RETURN_NOT_OK(populate_validity_bitmap(ArrowArrayValidityBitmap(tmp.get()))); - auto bitmask = bools_to_mask(column, stream, mr); + auto bitmask = detail::bools_to_mask(column, stream, mr); NANOARROW_RETURN_NOT_OK(populate_data_buffer( device_span(reinterpret_cast(bitmask.first->data()), bitmask.first->size()), diff --git a/cpp/src/transform/bools_to_mask.cu b/cpp/src/transform/bools_to_mask.cu index c12f65deb46..452aebf4428 100644 --- a/cpp/src/transform/bools_to_mask.cu +++ b/cpp/src/transform/bools_to_mask.cu @@ -59,10 +59,10 @@ std::pair, cudf::size_type> bools_to_mask( } // namespace detail std::pair, cudf::size_type> bools_to_mask( - column_view const& input, rmm::device_async_resource_ref mr) + column_view const& input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::bools_to_mask(input, cudf::get_default_stream(), mr); + return detail::bools_to_mask(input, stream, mr); } } // namespace cudf diff --git a/cpp/src/transform/compute_column.cu b/cpp/src/transform/compute_column.cu index 7960731f3a1..c4fc8d58552 100644 --- a/cpp/src/transform/compute_column.cu +++ b/cpp/src/transform/compute_column.cu @@ -138,10 +138,11 @@ std::unique_ptr compute_column(table_view const& table, std::unique_ptr compute_column(table_view const& table, ast::expression const& expr, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::compute_column(table, expr, cudf::get_default_stream(), mr); + return detail::compute_column(table, expr, stream, mr); } } // namespace cudf diff --git a/cpp/src/transform/encode.cu b/cpp/src/transform/encode.cu index 7a044b9f6f7..1c9d52bce1b 100644 --- a/cpp/src/transform/encode.cu +++ b/cpp/src/transform/encode.cu @@ -72,10 +72,10 @@ std::pair, std::unique_ptr> encode(table_view con } // namespace detail std::pair, std::unique_ptr> encode( - cudf::table_view const& input, rmm::device_async_resource_ref mr) + cudf::table_view const& input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::encode(input, cudf::get_default_stream(), mr); + return detail::encode(input, stream, mr); } } // namespace cudf diff --git a/cpp/src/transform/mask_to_bools.cu b/cpp/src/transform/mask_to_bools.cu index adf5db02d9c..be0b80a2633 100644 --- a/cpp/src/transform/mask_to_bools.cu +++ b/cpp/src/transform/mask_to_bools.cu @@ -62,9 +62,10 @@ std::unique_ptr mask_to_bools(bitmask_type const* bitmask, std::unique_ptr mask_to_bools(bitmask_type const* bitmask, size_type begin_bit, size_type end_bit, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::mask_to_bools(bitmask, begin_bit, end_bit, cudf::get_default_stream(), mr); + return detail::mask_to_bools(bitmask, begin_bit, end_bit, stream, mr); } } // namespace cudf diff --git a/cpp/src/transform/nans_to_nulls.cu b/cpp/src/transform/nans_to_nulls.cu index fd4f33c594c..a24ba304004 100644 --- a/cpp/src/transform/nans_to_nulls.cu +++ b/cpp/src/transform/nans_to_nulls.cu @@ -93,10 +93,10 @@ std::pair, cudf::size_type> nans_to_nulls( } // namespace detail std::pair, cudf::size_type> nans_to_nulls( - column_view const& input, rmm::device_async_resource_ref mr) + column_view const& input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::nans_to_nulls(input, cudf::get_default_stream(), mr); + return detail::nans_to_nulls(input, stream, mr); } } // namespace cudf diff --git a/cpp/src/transform/one_hot_encode.cu b/cpp/src/transform/one_hot_encode.cu index 808f2d1b284..46e6e55b0b7 100644 --- a/cpp/src/transform/one_hot_encode.cu +++ b/cpp/src/transform/one_hot_encode.cu @@ -115,9 +115,10 @@ std::pair, table_view> one_hot_encode(column_view const& std::pair, table_view> one_hot_encode(column_view const& input, column_view const& categories, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::one_hot_encode(input, categories, cudf::get_default_stream(), mr); + return detail::one_hot_encode(input, categories, stream, mr); } } // namespace cudf diff --git a/cpp/src/transform/row_bit_count.cu b/cpp/src/transform/row_bit_count.cu index 12a15eb7e34..4530fabf889 100644 --- a/cpp/src/transform/row_bit_count.cu +++ b/cpp/src/transform/row_bit_count.cu @@ -561,23 +561,26 @@ std::unique_ptr row_bit_count(table_view const& t, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - return segmented_row_bit_count(t, 1, stream, mr); + return detail::segmented_row_bit_count(t, 1, stream, mr); } } // namespace detail std::unique_ptr segmented_row_bit_count(table_view const& t, size_type segment_length, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::segmented_row_bit_count(t, segment_length, cudf::get_default_stream(), mr); + return detail::segmented_row_bit_count(t, segment_length, stream, mr); } -std::unique_ptr row_bit_count(table_view const& t, rmm::device_async_resource_ref mr) +std::unique_ptr row_bit_count(table_view const& t, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::row_bit_count(t, cudf::get_default_stream(), mr); + return detail::row_bit_count(t, stream, mr); } } // namespace cudf diff --git a/cpp/src/transform/transform.cpp b/cpp/src/transform/transform.cpp index 98ec44758b9..f5e9048fa0a 100644 --- a/cpp/src/transform/transform.cpp +++ b/cpp/src/transform/transform.cpp @@ -97,10 +97,11 @@ std::unique_ptr transform(column_view const& input, std::string const& unary_udf, data_type output_type, bool is_ptx, + rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::transform(input, unary_udf, output_type, is_ptx, cudf::get_default_stream(), mr); + return detail::transform(input, unary_udf, output_type, is_ptx, stream, mr); } } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 5e85b3e8adf..8c4b0f1e367 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -736,6 +736,7 @@ ConfigureTest( STREAM_MODE testing ) +ConfigureTest(STREAM_TRANSFORM_TEST streams/transform_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_UNARY_TEST streams/unary_test.cpp STREAM_MODE testing) # ################################################################################################## diff --git a/cpp/tests/streams/transform_test.cpp b/cpp/tests/streams/transform_test.cpp new file mode 100644 index 00000000000..9187672221c --- /dev/null +++ b/cpp/tests/streams/transform_test.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +class TransformTest : public cudf::test::BaseFixture {}; + +template +void test_udf(char const udf[], Data data_init, cudf::size_type size, bool is_ptx) +{ + auto all_valid = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return true; }); + auto data_iter = cudf::detail::make_counting_transform_iterator(0, data_init); + cudf::test::fixed_width_column_wrapper in( + data_iter, data_iter + size, all_valid); + cudf::transform( + in, udf, cudf::data_type(cudf::type_to_id()), is_ptx, cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, Transform) +{ + char const* cuda = + R"***( +__device__ inline void fdsf ( + float* C, + float a +) +{ + *C = a*a*a*a; +} +)***"; + + char const* ptx = + R"***( +// +// Generated by NVIDIA NVVM Compiler +// +// Compiler Build ID: CL-24817639 +// Cuda compilation tools, release 10.0, V10.0.130 +// Based on LLVM 3.4svn +// + +.version 6.3 +.target sm_70 +.address_size 64 + + // .globl _ZN8__main__7add$241Ef +.common .global .align 8 .u64 _ZN08NumbaEnv8__main__7add$241Ef; +.common .global .align 8 .u64 _ZN08NumbaEnv5numba7targets7numbers14int_power_impl12$3clocals$3e13int_power$242Efx; + +.visible .func (.param .b32 func_retval0) _ZN8__main__7add$241Ef( + .param .b64 _ZN8__main__7add$241Ef_param_0, + .param .b32 _ZN8__main__7add$241Ef_param_1 +) +{ + .reg .f32 %f<4>; + .reg .b32 %r<2>; + .reg .b64 %rd<2>; + + + ld.param.u64 %rd1, [_ZN8__main__7add$241Ef_param_0]; + ld.param.f32 %f1, [_ZN8__main__7add$241Ef_param_1]; + mul.f32 %f2, %f1, %f1; + mul.f32 %f3, %f2, %f2; + st.f32 [%rd1], %f3; + mov.u32 %r1, 0; + st.param.b32 [func_retval0+0], %r1; + ret; +} +)***"; + + auto data_init = [](cudf::size_type row) { return row % 3; }; + test_udf(cuda, data_init, 500, false); + test_udf(ptx, data_init, 500, true); +} + +TEST_F(TransformTest, ComputeColumn) +{ + auto c_0 = cudf::test::fixed_width_column_wrapper{3, 20, 1, 50}; + auto c_1 = cudf::test::fixed_width_column_wrapper{10, 7, 20, 0}; + auto table = cudf::table_view{{c_0, c_1}}; + auto col_ref_0 = cudf::ast::column_reference(0); + auto col_ref_1 = cudf::ast::column_reference(1); + auto expression = cudf::ast::operation(cudf::ast::ast_operator::ADD, col_ref_0, col_ref_1); + cudf::compute_column(table, expression, cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, BoolsToMask) +{ + std::vector input({1, 0, 1, 0, 1, 0, 1, 0}); + cudf::test::fixed_width_column_wrapper input_column(input.begin(), input.end()); + cudf::bools_to_mask(input_column, cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, MaskToBools) +{ + cudf::mask_to_bools(nullptr, 0, 0, cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, Encode) +{ + cudf::test::fixed_width_column_wrapper input{{1, 2, 3, 2, 3, 2, 1}}; + cudf::encode(cudf::table_view({input}), cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, OneHotEncode) +{ + auto input = cudf::test::fixed_width_column_wrapper{8, 8, 8, 9, 9}; + auto category = cudf::test::fixed_width_column_wrapper{8, 9}; + cudf::one_hot_encode(input, category, cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, NaNsToNulls) +{ + std::vector input = {1, 2, 3, 4, 5}; + std::vector mask = {true, true, true, true, false, false}; + auto input_column = + cudf::test::fixed_width_column_wrapper(input.begin(), input.end(), mask.begin()); + cudf::nans_to_nulls(input_column, cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, RowBitCount) +{ + std::vector strings{"abc", "ï", "", "z", "bananas", "warp", "", "zing"}; + cudf::test::strings_column_wrapper col(strings.begin(), strings.end()); + cudf::row_bit_count(cudf::table_view({col}), cudf::test::get_default_stream()); +} + +TEST_F(TransformTest, SegmentedRowBitCount) +{ + // clang-format off + std::vector const strings { "daïs", "def", "", "z", "bananas", "warp", "", "zing" }; + std::vector const valids { 1, 0, 0, 1, 0, 1, 1, 1 }; + // clang-format on + cudf::test::strings_column_wrapper const col(strings.begin(), strings.end(), valids.begin()); + auto const input = cudf::table_view({col}); + auto constexpr segment_length = 2; + cudf::segmented_row_bit_count(input, segment_length, cudf::test::get_default_stream()); +} From a94512a568bd0351fd20b0c2cbcd6067fd4d504b Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Wed, 7 Aug 2024 22:20:57 -0700 Subject: [PATCH 035/270] Add interop example for `arrow::StringViewArray` to `cudf::column` (#16498) Demonstrates the conversion from an `arrow:StringViewArray` to a `cudf::column` Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) Approvers: - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16498 --- cpp/examples/build.sh | 1 + cpp/examples/interop/CMakeLists.txt | 20 ++++ cpp/examples/interop/interop.cpp | 176 ++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100644 cpp/examples/interop/CMakeLists.txt create mode 100644 cpp/examples/interop/interop.cpp diff --git a/cpp/examples/build.sh b/cpp/examples/build.sh index dce81fb1677..2d6f6f316c7 100755 --- a/cpp/examples/build.sh +++ b/cpp/examples/build.sh @@ -61,3 +61,4 @@ build_example tpch build_example strings build_example nested_types build_example parquet_io +build_example interop diff --git a/cpp/examples/interop/CMakeLists.txt b/cpp/examples/interop/CMakeLists.txt new file mode 100644 index 00000000000..a1f99c1d2fd --- /dev/null +++ b/cpp/examples/interop/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +cmake_minimum_required(VERSION 3.26.4) + +include(../set_cuda_architecture.cmake) + +rapids_cuda_init_architectures(interop_example) +rapids_cuda_set_architectures(RAPIDS) + +project( + interop_example + VERSION 0.0.1 + LANGUAGES CXX CUDA +) + +include(../fetch_dependencies.cmake) + +add_executable(interop interop.cpp) +target_link_libraries(interop PRIVATE cudf::cudf) +target_compile_features(interop PRIVATE cxx_std_17) diff --git a/cpp/examples/interop/interop.cpp b/cpp/examples/interop/interop.cpp new file mode 100644 index 00000000000..8271c3836e4 --- /dev/null +++ b/cpp/examples/interop/interop.cpp @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include +#include + +// Helper functuons to create StringViews +inline arrow::StringViewType::c_type to_inline_string_view(const void* data, int32_t const& size) +{ + arrow::StringViewType::c_type out; + out.inlined = {size, {}}; + memcpy(&out.inlined.data, data, size); + return out; +} +inline arrow::StringViewType::c_type to_inline_string_view(std::string_view const& v) +{ + return to_inline_string_view(v.data(), static_cast(v.size())); +} +inline arrow::StringViewType::c_type to_string_view(const void* data, + int32_t const& size, + int32_t const& buffer_index, + int32_t const& offset) +{ + if (size <= arrow::StringViewType::kInlineSize) { return to_inline_string_view(data, size); } + arrow::StringViewType::c_type out; + out.ref = {size, {}, buffer_index, offset}; + memcpy(&out.ref.prefix, data, sizeof(out.ref.prefix)); + return out; +} +inline arrow::StringViewType::c_type to_string_view(std::string_view const& v, + int32_t const& buffer_index, + int32_t const& offset) +{ + return to_string_view(v.data(), static_cast(v.size()), buffer_index, offset); +} + +/** + * @brief Create a StringViewArray + * + * @param data_buffers The data buffers + * @param views The string views + * @param validate Whether to validate the array + */ +arrow::Result> make_string_view_array( + arrow::BufferVector const& data_buffers, + std::vector const& views, + bool validate = true) +{ + auto const length = static_cast(views.size()); + auto const arr = std::make_shared( + arrow::utf8_view(), length, arrow::Buffer::FromVector(views), std::move(data_buffers)); + if (validate) { RETURN_NOT_OK(arr->ValidateFull()); } + return arr; +} + +/** + * @brief Convert a vector of strings into a vector of the + * constituent chars and a vector of offsets. + * + * @param strings The vector of strings + */ +auto make_chars_and_offsets(std::vector const& strings) +{ + std::vector chars{}; + std::vector offsets(1, 0); + for (auto& str : strings) { + chars.insert(chars.end(), std::cbegin(str), std::cend(str)); + auto const last_offset = static_cast(offsets.back()); + auto const next_offset = last_offset + str.length(); + CUDF_EXPECTS( + next_offset < static_cast(std::numeric_limits::max()), + "Cannot use arrow_string_view_to_cudf_column to build a large strings column"); + offsets.push_back(static_cast(next_offset)); + } + return std::make_tuple(std::move(chars), std::move(offsets)); +}; + +/** + * @brief Convert an Arrow StringViewArray to a cudf::column + * + * @param array The Arrow StringViewArray + * @param stream The CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr arrow_string_view_to_cudf_column( + std::shared_ptr const& array, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) +{ + // Convert the string views into chars and offsets + std::vector strings; + for (auto i = 0; i < array->length(); i++) { + strings.push_back(array->GetString(i)); + } + auto const [chars, offsets] = make_chars_and_offsets(strings); + + // Copy the chars vector to the device + rmm::device_uvector d_chars(chars.size(), stream, mr); + CUDF_CUDA_TRY(cudaMemcpyAsync( + d_chars.data(), chars.data(), chars.size() * sizeof(char), cudaMemcpyDefault, stream.value())); + + // Copy the offsets vector to the device + // and wrap it in a cudf::column + rmm::device_uvector d_offsets(offsets.size(), stream, mr); + CUDF_CUDA_TRY(cudaMemcpyAsync(d_offsets.data(), + offsets.data(), + offsets.size() * sizeof(cudf::size_type), + cudaMemcpyDefault, + stream.value())); + auto offsets_col = + std::make_unique(std::move(d_offsets), rmm::device_buffer{0, stream, mr}, 0); + + // Create a string column out of the chars and offsets + return cudf::make_strings_column(array->length(), + std::move(offsets_col), + d_chars.release(), + 0, + rmm::device_buffer{0, stream, mr}); +} + +int main(int argc, char** argv) +{ + std::vector> data_buffers; + std::vector views; + + // Define the data buffers and string views + auto const buffer_a = + arrow::Buffer::FromString("hello rapids teamapache arrow interopnvidiacudf"); + data_buffers.push_back(buffer_a); + views.push_back(to_string_view("hello rapid steam", 0, 0)); + views.push_back(to_string_view("apache arrow interop", 0, 17)); + views.push_back(to_inline_string_view("nvidia")); + views.push_back(to_inline_string_view("cudf")); + + // Create a StringViewArray + auto const string_view_col = make_string_view_array(data_buffers, views, true).ValueOrDie(); + std::cout << string_view_col->ToString() << std::endl; + + // Convert the StringViewArray to a cudf::column + auto const cudf_col = arrow_string_view_to_cudf_column(string_view_col); + + // Write the cudf::column as CSV + auto const tbl_view = cudf::table_view({cudf_col->view()}); + std::vector const names = {"col_a"}; + + std::vector h_buffer; + cudf::io::csv_writer_options writer_options = + cudf::io::csv_writer_options::builder(cudf::io::sink_info(&h_buffer), tbl_view) + .include_header(not names.empty()) + .names(names); + + cudf::io::write_csv(writer_options); + auto const result = std::string(h_buffer.data(), h_buffer.size()); + std::cout << result << std::endl; + + return 0; +} From cc75b05b426920e6522c49527f8b684f780f38e3 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:00:22 -0400 Subject: [PATCH 036/270] Change IPv4 convert APIs to support UINT32 instead of INT64 (#16489) Changes the integer type for `cudf::strings::ipv4_to_integers` and `cudf::strings::integers_to_ipv4` to use UINT32 types instead of INT64. The INT64 type was originally chosen because libcudf did not support unsigned types at the time. This is a breaking change since the basic input/output type is changed. Closes #16324 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - https://github.com/brandon-b-miller - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/16489 --- cpp/include/cudf/strings/convert/convert_ipv4.hpp | 11 +++-------- cpp/src/strings/convert/convert_ipv4.cu | 14 +++++++------- cpp/tests/strings/ipv4_tests.cpp | 8 ++++---- python/cudf/cudf/core/column/numerical.py | 4 ++-- python/cudf/cudf/tests/test_string.py | 6 ++++-- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/cpp/include/cudf/strings/convert/convert_ipv4.hpp b/cpp/include/cudf/strings/convert/convert_ipv4.hpp index 04a04907c12..97d1dfee017 100644 --- a/cpp/include/cudf/strings/convert/convert_ipv4.hpp +++ b/cpp/include/cudf/strings/convert/convert_ipv4.hpp @@ -44,15 +44,12 @@ namespace strings { * No checking is done on the format. If a string is not in IPv4 format, the resulting * integer is undefined. * - * The resulting 32-bit integer is placed in an int64_t to avoid setting the sign-bit - * in an int32_t type. This could be changed if cudf supported a UINT32 type in the future. - * * Any null entries will result in corresponding null entries in the output column. * * @param input Strings instance for this operation * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory - * @return New INT64 column converted from strings + * @return New UINT32 column converted from strings */ std::unique_ptr ipv4_to_integers( strings_column_view const& input, @@ -68,13 +65,11 @@ std::unique_ptr ipv4_to_integers( * Each input integer is dissected into four integers by dividing the input into 8-bit sections. * These sub-integers are then converted into [0-9] characters and placed between '.' characters. * - * No checking is done on the input integer value. Only the lower 32-bits are used. - * * Any null entries will result in corresponding null entries in the output column. * - * @throw cudf::logic_error if the input column is not INT64 type. + * @throw cudf::logic_error if the input column is not UINT32 type. * - * @param integers Integer (INT64) column to convert + * @param integers Integer (UINT32) column to convert * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory * @return New strings column diff --git a/cpp/src/strings/convert/convert_ipv4.cu b/cpp/src/strings/convert/convert_ipv4.cu index 68a24e000ae..13d6e9bc3ba 100644 --- a/cpp/src/strings/convert/convert_ipv4.cu +++ b/cpp/src/strings/convert/convert_ipv4.cu @@ -46,7 +46,7 @@ namespace { struct ipv4_to_integers_fn { column_device_view const d_strings; - __device__ int64_t operator()(size_type idx) + __device__ uint32_t operator()(size_type idx) { if (d_strings.is_null(idx)) return 0; string_view d_str = d_strings.element(idx); @@ -66,7 +66,7 @@ struct ipv4_to_integers_fn { } } uint32_t result = (ipvals[0] << 24) + (ipvals[1] << 16) + (ipvals[2] << 8) + ipvals[3]; - return static_cast(result); + return result; } }; @@ -79,18 +79,18 @@ std::unique_ptr ipv4_to_integers(strings_column_view const& input, { size_type strings_count = input.size(); if (strings_count == 0) { - return make_numeric_column(data_type{type_id::INT64}, 0, mask_state::UNALLOCATED, stream); + return make_numeric_column(data_type{type_id::UINT32}, 0, mask_state::UNALLOCATED, stream); } auto strings_column = column_device_view::create(input.parent(), stream); // create output column copying the strings' null-mask - auto results = make_numeric_column(data_type{type_id::INT64}, + auto results = make_numeric_column(data_type{type_id::UINT32}, strings_count, cudf::detail::copy_bitmask(input.parent(), stream, mr), input.null_count(), stream, mr); - auto d_results = results->mutable_view().data(); + auto d_results = results->mutable_view().data(); // fill output column with ipv4 integers thrust::transform(rmm::exec_policy(stream), thrust::make_counting_iterator(0), @@ -135,7 +135,7 @@ struct integers_to_ipv4_fn { return; } - auto const ip_number = d_column.element(idx); + auto const ip_number = d_column.element(idx); char* out_ptr = d_chars ? d_chars + d_offsets[idx] : nullptr; int shift_bits = 24; @@ -165,7 +165,7 @@ std::unique_ptr integers_to_ipv4(column_view const& integers, { if (integers.is_empty()) return make_empty_column(type_id::STRING); - CUDF_EXPECTS(integers.type().id() == type_id::INT64, "Input column must be type_id::INT64 type"); + CUDF_EXPECTS(integers.type().id() == type_id::UINT32, "Input column must be UINT32 type"); auto d_column = column_device_view::create(integers, stream); auto [offsets_column, chars] = diff --git a/cpp/tests/strings/ipv4_tests.cpp b/cpp/tests/strings/ipv4_tests.cpp index 3bfe0f9727e..ea3ac439e62 100644 --- a/cpp/tests/strings/ipv4_tests.cpp +++ b/cpp/tests/strings/ipv4_tests.cpp @@ -40,8 +40,8 @@ TEST_F(StringsConvertTest, IPv4ToIntegers) auto strings_view = cudf::strings_column_view(strings); auto results = cudf::strings::ipv4_to_integers(strings_view); - std::vector h_expected{0, 0, 0, 698875905, 2130706433, 700776449, 3232235521}; - cudf::test::fixed_width_column_wrapper expected( + std::vector h_expected{0, 0, 0, 698875905, 2130706433, 700776449, 3232235521}; + cudf::test::fixed_width_column_wrapper expected( h_expected.cbegin(), h_expected.cend(), thrust::make_transform_iterator(h_strings.begin(), @@ -59,8 +59,8 @@ TEST_F(StringsConvertTest, IntegersToIPv4) thrust::make_transform_iterator(h_strings.begin(), [](auto const str) { return str != nullptr; })); - std::vector h_column{3232235521, 167772161, 0, 0, 700055553, 700776449}; - cudf::test::fixed_width_column_wrapper column( + std::vector h_column{3232235521, 167772161, 0, 0, 700055553, 700776449}; + cudf::test::fixed_width_column_wrapper column( h_column.cbegin(), h_column.cend(), thrust::make_transform_iterator(h_strings.begin(), diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index df27134d458..b83d7600c82 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -313,8 +313,8 @@ def normalize_binop_value( return NotImplemented def int2ip(self) -> "cudf.core.column.StringColumn": - if self.dtype != cudf.dtype("int64"): - raise TypeError("Only int64 type can be converted to ip") + if self.dtype != cudf.dtype("uint32"): + raise TypeError("Only uint32 type can be converted to ip") return libcudf.string_casting.int2ip(self) diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index 4bd084a3938..a2a3e874c91 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -2672,7 +2672,9 @@ def test_string_ip4_to_int(): def test_string_int_to_ipv4(): - gsr = cudf.Series([0, None, 0, 698875905, 2130706433, 700776449]) + gsr = cudf.Series([0, None, 0, 698875905, 2130706433, 700776449]).astype( + "uint32" + ) expected = cudf.Series( ["0.0.0.0", None, "0.0.0.0", "41.168.0.1", "127.0.0.1", "41.197.0.1"] ) @@ -2718,7 +2720,7 @@ def test_string_isipv4(): @pytest.mark.parametrize( - "dtype", sorted(list(dtypeutils.NUMERIC_TYPES - {"int64", "uint64"})) + "dtype", sorted(list(dtypeutils.NUMERIC_TYPES - {"uint32"})) ) def test_string_int_to_ipv4_dtype_fail(dtype): gsr = cudf.Series([1, 2, 3, 4, 5]).astype(dtype) From da51cad6c25f54ab344b0aa25e3dc1e4adb4550a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Aug 2024 10:25:11 -0500 Subject: [PATCH 037/270] Improve update-version.sh (#16506) A few small tweaks to `update-version.sh` for alignment across RAPIDS. The `UCX_PY` curl call is unused. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16506 --- ci/release/update-version.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index ad96aff3930..132e58249e6 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -18,18 +18,16 @@ CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}') CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} -#Get . for next version +# Get . for next version NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} -NEXT_UCX_PY_VERSION="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG}).*" # Need to distutils-normalize the versions for some use cases CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}'))") NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") PATCH_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_PATCH}'))") -echo "current is ${CURRENT_SHORT_TAG_PEP440}, next is ${NEXT_SHORT_TAG_PEP440}" echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" @@ -61,7 +59,7 @@ for DEP in "${DEPENDENCIES[@]}"; do sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done for FILE in python/*/pyproject.toml; do - sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE} + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" "${FILE}" done done @@ -77,7 +75,7 @@ sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_ # CI files for FILE in .github/workflows/*.yaml .github/workflows/*.yml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" - sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE}; + sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" "${FILE}" done sed_runner "s/branch-[0-9]+\.[0-9]+/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cudf_polars.sh From 792dd0686f4970c70f9bdba62c54a3de0a495fd5 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 8 Aug 2024 12:56:36 -0400 Subject: [PATCH 038/270] Update pre-commit hooks (#16510) This PR updates pre-commit hooks to the latest versions that are supported without causing style check errors. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16510 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbcd78d051f..1b17eae0842 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -144,7 +144,7 @@ repos: - id: ruff-format files: python/.*$ - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.2.0 + rev: v0.3.1 hooks: - id: verify-copyright exclude: | From 1bbe440ee7ddbc021f945e4156220f9bc270a443 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Aug 2024 12:25:29 -0500 Subject: [PATCH 039/270] Add keep option to distinct nvbench (#16497) This PR adopts some work from @srinivasyadav18 with additional modifications. This is meant to complement #16484. Authors: - Bradley Dice (https://github.com/bdice) - Srinivas Yadav (https://github.com/srinivasyadav18) Approvers: - Yunsong Wang (https://github.com/PointKernel) - Srinivas Yadav (https://github.com/srinivasyadav18) URL: https://github.com/rapidsai/cudf/pull/16497 --- cpp/benchmarks/CMakeLists.txt | 1 + cpp/benchmarks/stream_compaction/distinct.cpp | 45 ++++++++++++------- .../stream_compaction/stable_distinct.cpp | 45 ++++++++++++------- .../stream_compaction_common.cpp | 35 +++++++++++++++ .../stream_compaction_common.hpp | 19 ++++++++ 5 files changed, 113 insertions(+), 32 deletions(-) create mode 100644 cpp/benchmarks/stream_compaction/stream_compaction_common.cpp create mode 100644 cpp/benchmarks/stream_compaction/stream_compaction_common.hpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 7be456ddfba..483b7b0a539 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -162,6 +162,7 @@ ConfigureNVBench( stream_compaction/distinct.cpp stream_compaction/distinct_count.cpp stream_compaction/stable_distinct.cpp + stream_compaction/stream_compaction_common.cpp stream_compaction/unique.cpp stream_compaction/unique_count.cpp ) diff --git a/cpp/benchmarks/stream_compaction/distinct.cpp b/cpp/benchmarks/stream_compaction/distinct.cpp index c04b6516903..d7deebca89a 100644 --- a/cpp/benchmarks/stream_compaction/distinct.cpp +++ b/cpp/benchmarks/stream_compaction/distinct.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -23,15 +24,29 @@ #include +#include + NVBENCH_DECLARE_TYPE_STRINGS(cudf::timestamp_ms, "cudf::timestamp_ms", "cudf::timestamp_ms"); template void nvbench_distinct(nvbench::state& state, nvbench::type_list) { - cudf::size_type const num_rows = state.get_int64("NumRows"); + cudf::size_type const num_rows = state.get_int64("NumRows"); + auto const keep = get_keep(state.get_string("keep")); + cudf::size_type const cardinality = state.get_int64("cardinality"); + + if (cardinality > num_rows) { + state.skip("cardinality > num_rows"); + return; + } - data_profile profile = data_profile_builder().cardinality(0).null_probability(0.01).distribution( - cudf::type_to_id(), distribution_id::UNIFORM, 0, 100); + data_profile profile = data_profile_builder() + .cardinality(cardinality) + .null_probability(0.01) + .distribution(cudf::type_to_id(), + distribution_id::UNIFORM, + static_cast(0), + std::numeric_limits::max()); auto source_column = create_random_column(cudf::type_to_id(), row_count{num_rows}, profile); @@ -40,20 +55,19 @@ void nvbench_distinct(nvbench::state& state, nvbench::type_list) state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { - auto result = cudf::distinct(input_table, - {0}, - cudf::duplicate_keep_option::KEEP_ANY, - cudf::null_equality::EQUAL, - cudf::nan_equality::ALL_EQUAL); + auto result = cudf::distinct( + input_table, {0}, keep, cudf::null_equality::EQUAL, cudf::nan_equality::ALL_EQUAL); }); } -using data_type = nvbench::type_list; +using data_type = nvbench::type_list; NVBENCH_BENCH_TYPES(nvbench_distinct, NVBENCH_TYPE_AXES(data_type)) .set_name("distinct") .set_type_axes_names({"Type"}) - .add_int64_axis("NumRows", {10'000, 100'000, 1'000'000, 10'000'000}); + .add_string_axis("keep", {"any", "first", "last", "none"}) + .add_int64_axis("cardinality", {100, 100'000, 10'000'000, 1'000'000'000}) + .add_int64_axis("NumRows", {100, 100'000, 10'000'000, 1'000'000'000}); template void nvbench_distinct_list(nvbench::state& state, nvbench::type_list) @@ -61,6 +75,7 @@ void nvbench_distinct_list(nvbench::state& state, nvbench::type_list) auto const size = state.get_int64("ColumnSize"); auto const dtype = cudf::type_to_id(); double const null_probability = state.get_float64("null_probability"); + auto const keep = get_keep(state.get_string("keep")); auto builder = data_profile_builder().null_probability(null_probability); if (dtype == cudf::type_id::LIST) { @@ -80,11 +95,8 @@ void nvbench_distinct_list(nvbench::state& state, nvbench::type_list) state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { - auto result = cudf::distinct(*table, - {0}, - cudf::duplicate_keep_option::KEEP_ANY, - cudf::null_equality::EQUAL, - cudf::nan_equality::ALL_EQUAL); + auto result = + cudf::distinct(*table, {0}, keep, cudf::null_equality::EQUAL, cudf::nan_equality::ALL_EQUAL); }); } @@ -92,5 +104,6 @@ NVBENCH_BENCH_TYPES(nvbench_distinct_list, NVBENCH_TYPE_AXES(nvbench::type_list)) .set_name("distinct_list") .set_type_axes_names({"Type"}) + .add_string_axis("keep", {"any", "first", "last", "none"}) .add_float64_axis("null_probability", {0.0, 0.1}) .add_int64_axis("ColumnSize", {100'000'000}); diff --git a/cpp/benchmarks/stream_compaction/stable_distinct.cpp b/cpp/benchmarks/stream_compaction/stable_distinct.cpp index bcee3048013..0a8836c0583 100644 --- a/cpp/benchmarks/stream_compaction/stable_distinct.cpp +++ b/cpp/benchmarks/stream_compaction/stable_distinct.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -23,15 +24,29 @@ #include +#include + NVBENCH_DECLARE_TYPE_STRINGS(cudf::timestamp_ms, "cudf::timestamp_ms", "cudf::timestamp_ms"); template void nvbench_stable_distinct(nvbench::state& state, nvbench::type_list) { - cudf::size_type const num_rows = state.get_int64("NumRows"); + cudf::size_type const num_rows = state.get_int64("NumRows"); + auto const keep = get_keep(state.get_string("keep")); + cudf::size_type const cardinality = state.get_int64("cardinality"); + + if (cardinality > num_rows) { + state.skip("cardinality > num_rows"); + return; + } - data_profile profile = data_profile_builder().cardinality(0).null_probability(0.01).distribution( - cudf::type_to_id(), distribution_id::UNIFORM, 0, 100); + data_profile profile = data_profile_builder() + .cardinality(cardinality) + .null_probability(0.01) + .distribution(cudf::type_to_id(), + distribution_id::UNIFORM, + static_cast(0), + std::numeric_limits::max()); auto source_column = create_random_column(cudf::type_to_id(), row_count{num_rows}, profile); @@ -40,20 +55,19 @@ void nvbench_stable_distinct(nvbench::state& state, nvbench::type_list) state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { - auto result = cudf::stable_distinct(input_table, - {0}, - cudf::duplicate_keep_option::KEEP_ANY, - cudf::null_equality::EQUAL, - cudf::nan_equality::ALL_EQUAL); + auto result = cudf::stable_distinct( + input_table, {0}, keep, cudf::null_equality::EQUAL, cudf::nan_equality::ALL_EQUAL); }); } -using data_type = nvbench::type_list; +using data_type = nvbench::type_list; NVBENCH_BENCH_TYPES(nvbench_stable_distinct, NVBENCH_TYPE_AXES(data_type)) .set_name("stable_distinct") .set_type_axes_names({"Type"}) - .add_int64_axis("NumRows", {10'000, 100'000, 1'000'000, 10'000'000}); + .add_string_axis("keep", {"any", "first", "last", "none"}) + .add_int64_axis("cardinality", {100, 100'000, 10'000'000, 1'000'000'000}) + .add_int64_axis("NumRows", {100, 100'000, 10'000'000, 1'000'000'000}); template void nvbench_stable_distinct_list(nvbench::state& state, nvbench::type_list) @@ -61,6 +75,7 @@ void nvbench_stable_distinct_list(nvbench::state& state, nvbench::type_list(); double const null_probability = state.get_float64("null_probability"); + auto const keep = get_keep(state.get_string("keep")); auto builder = data_profile_builder().null_probability(null_probability); if (dtype == cudf::type_id::LIST) { @@ -80,11 +95,8 @@ void nvbench_stable_distinct_list(nvbench::state& state, nvbench::type_list)) .set_name("stable_distinct_list") .set_type_axes_names({"Type"}) + .add_string_axis("keep", {"any", "first", "last", "none"}) .add_float64_axis("null_probability", {0.0, 0.1}) .add_int64_axis("ColumnSize", {100'000'000}); diff --git a/cpp/benchmarks/stream_compaction/stream_compaction_common.cpp b/cpp/benchmarks/stream_compaction/stream_compaction_common.cpp new file mode 100644 index 00000000000..8cbb2956777 --- /dev/null +++ b/cpp/benchmarks/stream_compaction/stream_compaction_common.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +cudf::duplicate_keep_option get_keep(std::string const& keep_str) +{ + if (keep_str == "any") { + return cudf::duplicate_keep_option::KEEP_ANY; + } else if (keep_str == "first") { + return cudf::duplicate_keep_option::KEEP_FIRST; + } else if (keep_str == "last") { + return cudf::duplicate_keep_option::KEEP_LAST; + } else if (keep_str == "none") { + return cudf::duplicate_keep_option::KEEP_NONE; + } else { + CUDF_FAIL("Unsupported keep option."); + } +} diff --git a/cpp/benchmarks/stream_compaction/stream_compaction_common.hpp b/cpp/benchmarks/stream_compaction/stream_compaction_common.hpp new file mode 100644 index 00000000000..d1ef2b10f41 --- /dev/null +++ b/cpp/benchmarks/stream_compaction/stream_compaction_common.hpp @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2020-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +cudf::duplicate_keep_option get_keep(std::string const& keep_str); From 2c8de625b69bf5f7f3315c45a34bdf9ba45315a9 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Fri, 9 Aug 2024 08:25:58 -0500 Subject: [PATCH 040/270] enable list to be forced as string in JSON reader. (#16472) closes #15278 This PR allows list type also forced as string when mixed type as string is enabled and a user given schema specifies a column as string, in JSON reader. Authors: - Karthikeyan (https://github.com/karthikeyann) - Nghia Truong (https://github.com/ttnghia) Approvers: - Nghia Truong (https://github.com/ttnghia) - Shruti Shivakumar (https://github.com/shrshi) URL: https://github.com/rapidsai/cudf/pull/16472 --- cpp/src/io/json/json_column.cu | 22 ++++--- cpp/tests/io/json/json_test.cpp | 113 ++++++++++++++++++++++---------- 2 files changed, 90 insertions(+), 45 deletions(-) diff --git a/cpp/src/io/json/json_column.cu b/cpp/src/io/json/json_column.cu index 17fa7abdffe..e5e21e054a6 100644 --- a/cpp/src/io/json/json_column.cu +++ b/cpp/src/io/json/json_column.cu @@ -567,22 +567,22 @@ void make_device_json_column(device_span input, thrust::uninitialized_fill(rmm::exec_policy_nosync(stream), v.begin(), v.end(), 0); }; - auto initialize_json_columns = [&](auto i, auto& col) { - if (column_categories[i] == NC_ERR || column_categories[i] == NC_FN) { + auto initialize_json_columns = [&](auto i, auto& col, auto column_category) { + if (column_category == NC_ERR || column_category == NC_FN) { return; - } else if (column_categories[i] == NC_VAL || column_categories[i] == NC_STR) { + } else if (column_category == NC_VAL || column_category == NC_STR) { col.string_offsets.resize(max_row_offsets[i] + 1, stream); col.string_lengths.resize(max_row_offsets[i] + 1, stream); init_to_zero(col.string_offsets); init_to_zero(col.string_lengths); - } else if (column_categories[i] == NC_LIST) { + } else if (column_category == NC_LIST) { col.child_offsets.resize(max_row_offsets[i] + 2, stream); init_to_zero(col.child_offsets); } col.num_rows = max_row_offsets[i] + 1; col.validity = cudf::detail::create_null_mask(col.num_rows, cudf::mask_state::ALL_NULL, stream, mr); - col.type = to_json_col_type(column_categories[i]); + col.type = to_json_col_type(column_category); }; auto reinitialize_as_string = [&](auto i, auto& col) { @@ -764,21 +764,23 @@ void make_device_json_column(device_span input, } } + auto this_column_category = column_categories[this_col_id]; if (is_enabled_mixed_types_as_string) { - // get path of this column, check if it is a struct forced as string, and enforce it + // get path of this column, check if it is a struct/list forced as string, and enforce it auto const nt = tree_path.get_path(this_col_id); std::optional const user_dtype = get_path_data_type(nt, options); - if (column_categories[this_col_id] == NC_STRUCT and user_dtype.has_value() and - user_dtype.value().id() == type_id::STRING) { + if ((column_categories[this_col_id] == NC_STRUCT or + column_categories[this_col_id] == NC_LIST) and + user_dtype.has_value() and user_dtype.value().id() == type_id::STRING) { is_mixed_type_column[this_col_id] = 1; - column_categories[this_col_id] = NC_STR; + this_column_category = NC_STR; } } CUDF_EXPECTS(parent_col.child_columns.count(name) == 0, "duplicate column name: " + name); // move into parent device_json_column col(stream, mr); - initialize_json_columns(this_col_id, col); + initialize_json_columns(this_col_id, col, this_column_category); auto inserted = parent_col.child_columns.try_emplace(name, std::move(col)).second; CUDF_EXPECTS(inserted, "child column insertion failed, duplicate column name in the parent"); if (not replaced) parent_col.column_order.push_back(name); diff --git a/cpp/tests/io/json/json_test.cpp b/cpp/tests/io/json/json_test.cpp index 993ab82f423..0a485e26b71 100644 --- a/cpp/tests/io/json/json_test.cpp +++ b/cpp/tests/io/json/json_test.cpp @@ -2351,7 +2351,7 @@ TEST_F(JsonReaderTest, MapTypes) // Testing function for mixed types in JSON (for spark json reader) auto test_fn = [](std::string_view json_string, bool lines, std::vector types) { std::map dtype_schema{ - {"foo1", {data_type{type_id::STRING}}}, // list won't be a string + {"foo1", {data_type{type_id::STRING}}}, // list forced as a string {"foo2", {data_type{type_id::STRING}}}, // struct forced as a string {"1", {data_type{type_id::STRING}}}, {"2", {data_type{type_id::STRING}}}, @@ -2378,17 +2378,17 @@ TEST_F(JsonReaderTest, MapTypes) test_fn(R"([{ "foo1": [1,2,3], "bar": 123 }, { "foo2": { "a": 1 }, "bar": 456 }])", false, - {type_id::LIST, type_id::INT32, type_id::STRING}); + {type_id::STRING, type_id::INT32, type_id::STRING}); // jsonl test_fn(R"( { "foo1": [1,2,3], "bar": 123 } { "foo2": { "a": 1 }, "bar": 456 })", true, - {type_id::LIST, type_id::INT32, type_id::STRING}); + {type_id::STRING, type_id::INT32, type_id::STRING}); // jsonl-array test_fn(R"([123, [1,2,3]] [456, null, { "a": 1 }])", true, - {type_id::INT64, type_id::LIST, type_id::STRING}); + {type_id::INT64, type_id::STRING, type_id::STRING}); // json-array test_fn(R"([[[1,2,3], null, 123], [null, { "a": 1 }, 456 ]])", @@ -2678,38 +2678,81 @@ TEST_F(JsonReaderTest, JsonNestedDtypeFilter) TEST_F(JsonReaderTest, JSONMixedTypeChildren) { - std::string const json_str = R"( -{ "Root": { "Key": [ { "EE": "A" } ] } } -{ "Root": { "Key": { } } } -{ "Root": { "Key": [{ "YY": 1}] } } -)"; - // Column "EE" is created and destroyed - // Column "YY" should not be created - - cudf::io::json_reader_options options = - cudf::io::json_reader_options::builder(cudf::io::source_info{json_str.c_str(), json_str.size()}) - .lines(true) - .recovery_mode(cudf::io::json_recovery_mode_t::RECOVER_WITH_NULL) - .normalize_single_quotes(true) - .normalize_whitespace(false) - .mixed_types_as_string(true) - .keep_quotes(true); - - auto result = cudf::io::read_json(options); + // struct mixed. + { + std::string const json_str = R"( + { "Root": { "Key": [ { "EE": "A" } ] } } + { "Root": { "Key": { } } } + { "Root": { "Key": [{ "YY": 1}] } } + )"; + // Column "EE" is created and destroyed + // Column "YY" should not be created + + cudf::io::json_reader_options options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{json_str.c_str(), json_str.size()}) + .lines(true) + .recovery_mode(cudf::io::json_recovery_mode_t::RECOVER_WITH_NULL) + .normalize_single_quotes(true) + .normalize_whitespace(false) + .mixed_types_as_string(true) + .keep_quotes(true); + + auto result = cudf::io::read_json(options); + + ASSERT_EQ(result.tbl->num_columns(), 1); + ASSERT_EQ(result.metadata.schema_info.size(), 1); + EXPECT_EQ(result.metadata.schema_info[0].name, "Root"); + ASSERT_EQ(result.metadata.schema_info[0].children.size(), 1); + EXPECT_EQ(result.metadata.schema_info[0].children[0].name, "Key"); + ASSERT_EQ(result.metadata.schema_info[0].children[0].children.size(), 2); + EXPECT_EQ(result.metadata.schema_info[0].children[0].children[0].name, "offsets"); + // types + EXPECT_EQ(result.tbl->get_column(0).type().id(), cudf::type_id::STRUCT); + EXPECT_EQ(result.tbl->get_column(0).child(0).type().id(), cudf::type_id::STRING); + cudf::test::strings_column_wrapper expected( + {R"([ { "EE": "A" } ])", "{ }", R"([{ "YY": 1}])"}); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result.tbl->get_column(0).child(0)); + } - ASSERT_EQ(result.tbl->num_columns(), 1); - ASSERT_EQ(result.metadata.schema_info.size(), 1); - EXPECT_EQ(result.metadata.schema_info[0].name, "Root"); - ASSERT_EQ(result.metadata.schema_info[0].children.size(), 1); - EXPECT_EQ(result.metadata.schema_info[0].children[0].name, "Key"); - ASSERT_EQ(result.metadata.schema_info[0].children[0].children.size(), 2); - EXPECT_EQ(result.metadata.schema_info[0].children[0].children[0].name, "offsets"); - // types - EXPECT_EQ(result.tbl->get_column(0).type().id(), cudf::type_id::STRUCT); - EXPECT_EQ(result.tbl->get_column(0).child(0).type().id(), cudf::type_id::STRING); - cudf::test::strings_column_wrapper expected({R"([ { "EE": "A" } ])", "{ }", R"([{ "YY": 1}])"}); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result.tbl->get_column(0).child(0)); + // list mixed. + { + std::string const json_str = R"( + { "Root": { "Key": [ { "EE": "A" } ] } } + { "Root": { "Key": "abc" } } + { "Root": { "Key": [{ "YY": 1}] } } + )"; + // Column "EE" is created and destroyed + // Column "YY" should not be created + + cudf::io::json_reader_options options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{json_str.c_str(), json_str.size()}) + .lines(true) + .recovery_mode(cudf::io::json_recovery_mode_t::RECOVER_WITH_NULL) + .normalize_single_quotes(true) + .normalize_whitespace(false) + .mixed_types_as_string(true) + .keep_quotes(true); + + auto result = cudf::io::read_json(options); + + ASSERT_EQ(result.tbl->num_columns(), 1); + ASSERT_EQ(result.metadata.schema_info.size(), 1); + EXPECT_EQ(result.metadata.schema_info[0].name, "Root"); + ASSERT_EQ(result.metadata.schema_info[0].children.size(), 1); + EXPECT_EQ(result.metadata.schema_info[0].children[0].name, "Key"); + ASSERT_EQ(result.metadata.schema_info[0].children[0].children.size(), 2); + EXPECT_EQ(result.metadata.schema_info[0].children[0].children[0].name, "offsets"); + // types + EXPECT_EQ(result.tbl->get_column(0).type().id(), cudf::type_id::STRUCT); + EXPECT_EQ(result.tbl->get_column(0).child(0).type().id(), cudf::type_id::STRING); + cudf::test::strings_column_wrapper expected( + {R"([ { "EE": "A" } ])", "\"abc\"", R"([{ "YY": 1}])"}); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result.tbl->get_column(0).child(0)); + } } CUDF_TEST_PROGRAM_MAIN() From 9ec34ad81152a4d7889bdf1f5b92032000b09b8f Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 9 Aug 2024 10:24:31 -0400 Subject: [PATCH 041/270] Remove a deprecated multibyte_split API (#16501) Removes overloaded `cudf::io::text::multibyte_split` API deprecated in 24.08 and is no longer needed. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16501 --- cpp/include/cudf/io/text/multibyte_split.hpp | 20 -------------------- cpp/src/io/text/multibyte_split.cu | 14 -------------- 2 files changed, 34 deletions(-) diff --git a/cpp/include/cudf/io/text/multibyte_split.hpp b/cpp/include/cudf/io/text/multibyte_split.hpp index 8624a386d0f..3a1f9611324 100644 --- a/cpp/include/cudf/io/text/multibyte_split.hpp +++ b/cpp/include/cudf/io/text/multibyte_split.hpp @@ -96,26 +96,6 @@ std::unique_ptr multibyte_split( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); -/** - * @brief Splits the source text into a strings column using a multiple byte delimiter. - * - * @deprecated Since 24.08 - * - * @param source The source input data encoded in UTF-8 - * @param delimiter UTF-8 encoded string for which to find offsets in the source - * @param byte_range The position and size within `source` to produce the column from - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Memory resource to use for the device memory allocation - * @return The strings found by splitting the source by the delimiter within the relevant byte - * range. - */ -[[deprecated]] std::unique_ptr multibyte_split( - data_chunk_source const& source, - std::string const& delimiter, - std::optional byte_range, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); - /** @} */ // end of group } // namespace text diff --git a/cpp/src/io/text/multibyte_split.cu b/cpp/src/io/text/multibyte_split.cu index be2e2b9a79c..97729a091fb 100644 --- a/cpp/src/io/text/multibyte_split.cu +++ b/cpp/src/io/text/multibyte_split.cu @@ -567,20 +567,6 @@ std::unique_ptr multibyte_split(cudf::io::text::data_chunk_source } // namespace detail -// deprecated in 24.08 -std::unique_ptr multibyte_split(cudf::io::text::data_chunk_source const& source, - std::string const& delimiter, - std::optional byte_range, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - return multibyte_split(source, - delimiter, - parse_options{byte_range.value_or(create_byte_range_info_max())}, - stream, - mr); -} - std::unique_ptr multibyte_split(cudf::io::text::data_chunk_source const& source, std::string const& delimiter, parse_options options, From 8009dc800bf79ba5fbacc9658235a212590640ba Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Fri, 9 Aug 2024 09:07:47 -0700 Subject: [PATCH 042/270] Update docs of the TPC-H derived examples (#16423) Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) Approvers: - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/16423 --- .gitignore | 2 + cpp/examples/tpch/README.md | 37 ++++++------ .../tpch/datagen/correct_datatypes.py | 60 +++++++++++++++++++ cpp/examples/tpch/datagen/datagen.sh | 31 ++++++++++ cpp/examples/tpch/datagen/tpch.patch | 33 ++++++++++ 5 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 cpp/examples/tpch/datagen/correct_datatypes.py create mode 100755 cpp/examples/tpch/datagen/datagen.sh create mode 100644 cpp/examples/tpch/datagen/tpch.patch diff --git a/.gitignore b/.gitignore index c89fb49697a..153c7f59744 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,8 @@ Debug build/ cpp/build/ cpp/examples/*/install/ +cpp/examples/*/build/ +cpp/examples/tpch/datagen/datafusion cpp/include/cudf/ipc_generated/*.h cpp/thirdparty/googletest/ diff --git a/cpp/examples/tpch/README.md b/cpp/examples/tpch/README.md index 1ea71ae9824..8c046c3f1e8 100644 --- a/cpp/examples/tpch/README.md +++ b/cpp/examples/tpch/README.md @@ -1,38 +1,39 @@ -# TPC-H Inspired Examples +# TPC-H Derived Examples Implements TPC-H queries using `libcudf`. We leverage the data generator (wrapper around official TPC-H datagen) from [Apache Datafusion](https://github.com/apache/datafusion) for generating data in Parquet format. ## Requirements - Rust +- [libcudf](https://github.com/rapidsai/cudf/blob/branch-24.08/CONTRIBUTING.md#setting-up-your-build-environment) -## Generating the Dataset +## Running Queries -1. Clone the datafusion repository. +1. Build the `libcudf` examples. ```bash -git clone git@github.com:apache/datafusion.git +cd cudf/cpp/examples +./build.sh ``` +The TPC-H query binaries would be built inside `tpch/build`. -2. Run the data generator. The data will be placed in a `data/` subdirectory. +2. Generate the dataset. ```bash -cd datafusion/benchmarks/ -./bench.sh data tpch - -# for scale factor 10, -./bench.sh data tpch10 +cd tpch/datagen +./datagen.sh [scale factor (1/10)] ``` -## Running Queries +The parquet files will be generated in `tpch/datagen/datafusion/benchmarks/data/tpch_sf[scale factor]`. -1. Build the examples. +3. Set these environment variables for optimized runtimes. ```bash -cd cpp/examples -./build.sh +export KVIKIO_COMPAT_MODE="on" +export LIBCUDF_CUFILE_POLICY="KVIKIO" +export CUDA_MODULE_LOADING="EAGER" ``` -The TPC-H query binaries would be built inside `examples/tpch/build`. -2. Execute the queries. +4. Execute the queries. ```bash -./tpch/build/tpch_q1 +./tpch/build/tpch_q[query no] [path to dataset] [memory resource type (cuda/pool/managed/managed_pool)] ``` -A parquet file named `q1.parquet` would be generated holding the results of the query. + +A parquet file named `q[query no].parquet` would be generated containing the results of the query. diff --git a/cpp/examples/tpch/datagen/correct_datatypes.py b/cpp/examples/tpch/datagen/correct_datatypes.py new file mode 100644 index 00000000000..8564774647b --- /dev/null +++ b/cpp/examples/tpch/datagen/correct_datatypes.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import os +import sys + +import pyarrow as pa +import pyarrow.parquet as pq +import pandas as pd + +if __name__ == "__main__": + dataset_path = str(sys.argv[1]) + tables = ["lineitem", "part", "partsupp", "orders", "supplier", "customer", "nation", "region"] + for table in tables: + filepath = os.path.join(dataset_path, f"{table}.parquet") + print("Reading file ", filepath) + + if filepath.endswith("lineitem.parquet"): + df = pd.read_parquet(filepath) + df["l_linenumber"] = df["l_linenumber"].astype("int64") + df["l_quantity"] = df["l_quantity"].astype("int64") + df["l_extendedprice"] = df["l_extendedprice"].astype("float64") + df["l_discount"] = df["l_discount"].astype("float64") + df["l_tax"] = df["l_tax"].astype("float64") + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") + + elif filepath.endswith("part.parquet"): + df = pd.read_parquet(filepath) + df["p_size"] = df["p_size"].astype("int64") + df["p_retailprice"] = df["p_retailprice"].astype("float64") + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") + + elif filepath.endswith("partsupp.parquet"): + df = pd.read_parquet(filepath) + df["ps_availqty"] = df["ps_availqty"].astype("int64") + df["ps_supplycost"] = df["ps_supplycost"].astype("float64") + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") + + elif filepath.endswith("orders.parquet"): + df = pd.read_parquet(filepath) + df["o_totalprice"] = df["o_totalprice"].astype("float64") + df["o_shippriority"] = df["o_shippriority"].astype("int64") + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") + + elif filepath.endswith("supplier.parquet"): + df = pd.read_parquet(filepath) + df["s_acctbal"] = df["s_acctbal"].astype("float64") + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") + + elif filepath.endswith("customer.parquet"): + df = pd.read_parquet(filepath) + df["c_acctbal"] = df["c_acctbal"].astype("float64") + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") + + elif filepath.endswith("nation.parquet"): + df = pd.read_parquet(filepath) + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") + + elif filepath.endswith("region.parquet"): + df = pd.read_parquet(filepath) + pq.write_table(pa.Table.from_pandas(df), filepath, compression="snappy") diff --git a/cpp/examples/tpch/datagen/datagen.sh b/cpp/examples/tpch/datagen/datagen.sh new file mode 100755 index 00000000000..0b03753daea --- /dev/null +++ b/cpp/examples/tpch/datagen/datagen.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -e + +scale_factor=$1 +script_dir=$(pwd) + +# Clone the datafusion repository and apply a patch +# for single threaded data generation so that a +# single parquet file is generated for each table +rm -rf datafusion +git clone https://github.com/apache/datafusion.git datafusion +cd datafusion/ +git checkout 679a85f +git apply ${script_dir}/tpch.patch +cd benchmarks/ + +# Generate the data +# Currently, we support only scale factor 1 and 10 +if [ ${scale_factor} -eq 1 ]; then + ./bench.sh data tpch +elif [ ${scale_factor} -eq 10 ]; then + ./bench.sh data tpch10 +else + echo "Unsupported scale factor" + exit 1 +fi + +# Correct the datatypes of the parquet files +python3 ${script_dir}/correct_datatypes.py data/tpch_sf${scale_factor} diff --git a/cpp/examples/tpch/datagen/tpch.patch b/cpp/examples/tpch/datagen/tpch.patch new file mode 100644 index 00000000000..42727aa9904 --- /dev/null +++ b/cpp/examples/tpch/datagen/tpch.patch @@ -0,0 +1,33 @@ +diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh +index 3b854f6dc..f000f09c0 100755 +--- a/benchmarks/bench.sh ++++ b/benchmarks/bench.sh +@@ -311,6 +311,15 @@ data_tpch() { + $CARGO_COMMAND --bin tpch -- convert --input "${TPCH_DIR}" --output "${TPCH_DIR}" --format parquet + popd > /dev/null + fi ++ ++ cp ${TPCH_DIR}/lineitem/part-0.parquet ${TPCH_DIR}/lineitem.parquet ++ cp ${TPCH_DIR}/orders/part-0.parquet ${TPCH_DIR}/orders.parquet ++ cp ${TPCH_DIR}/part/part-0.parquet ${TPCH_DIR}/part.parquet ++ cp ${TPCH_DIR}/partsupp/part-0.parquet ${TPCH_DIR}/partsupp.parquet ++ cp ${TPCH_DIR}/customer/part-0.parquet ${TPCH_DIR}/customer.parquet ++ cp ${TPCH_DIR}/supplier/part-0.parquet ${TPCH_DIR}/supplier.parquet ++ cp ${TPCH_DIR}/nation/part-0.parquet ${TPCH_DIR}/nation.parquet ++ cp ${TPCH_DIR}/region/part-0.parquet ${TPCH_DIR}/region.parquet + } + + # Runs the tpch benchmark +diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs +index b5204b343..84fd2e78d 100644 +--- a/datafusion/common/src/config.rs ++++ b/datafusion/common/src/config.rs +@@ -250,7 +250,7 @@ config_namespace! { + /// concurrency. + /// + /// Defaults to the number of CPU cores on the system +- pub target_partitions: usize, default = num_cpus::get() ++ pub target_partitions: usize, default = 1 + + /// The default time zone + /// From 4446cf0188c03b82cbec28493aa131027f25dffa Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:43:23 -0500 Subject: [PATCH 043/270] Update json normalization to take device_buffer (#16520) This change updates json normalization calls (quote and whitespace normalization) to take owning buffer of device_buffer as input rather than device_uvector. It makes it easy to hand over a string_column's char buffer to normalization calls. Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - David Wendt (https://github.com/davidwendt) - Shruti Shivakumar (https://github.com/shrshi) URL: https://github.com/rapidsai/cudf/pull/16520 --- cpp/include/cudf/io/detail/json.hpp | 4 ++-- cpp/src/io/json/json_normalization.cu | 20 +++++++++---------- cpp/src/io/json/read_json.cu | 16 +++++++-------- .../io/json/json_quote_normalization_test.cpp | 9 ++++----- .../json_whitespace_normalization_test.cu | 7 +++---- 5 files changed, 27 insertions(+), 29 deletions(-) diff --git a/cpp/include/cudf/io/detail/json.hpp b/cpp/include/cudf/io/detail/json.hpp index 42b10a78ce8..38ba4f675c3 100644 --- a/cpp/include/cudf/io/detail/json.hpp +++ b/cpp/include/cudf/io/detail/json.hpp @@ -61,7 +61,7 @@ void write_json(data_sink* sink, * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource to use for device memory allocation */ -void normalize_single_quotes(datasource::owning_buffer>& indata, +void normalize_single_quotes(datasource::owning_buffer& indata, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); @@ -72,7 +72,7 @@ void normalize_single_quotes(datasource::owning_buffer * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource to use for device memory allocation */ -void normalize_whitespace(datasource::owning_buffer>& indata, +void normalize_whitespace(datasource::owning_buffer& indata, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); } // namespace io::json::detail diff --git a/cpp/src/io/json/json_normalization.cu b/cpp/src/io/json/json_normalization.cu index 760b2214365..cb8b4e97ebb 100644 --- a/cpp/src/io/json/json_normalization.cu +++ b/cpp/src/io/json/json_normalization.cu @@ -298,7 +298,7 @@ struct TransduceToNormalizedWS { namespace detail { -void normalize_single_quotes(datasource::owning_buffer>& indata, +void normalize_single_quotes(datasource::owning_buffer& indata, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -311,22 +311,22 @@ void normalize_single_quotes(datasource::owning_buffer outbuf(indata.size() * 2, stream, mr); + rmm::device_buffer outbuf(indata.size() * 2, stream, mr); rmm::device_scalar outbuf_size(stream, mr); - parser.Transduce(indata.data(), + parser.Transduce(reinterpret_cast(indata.data()), static_cast(indata.size()), - outbuf.data(), + static_cast(outbuf.data()), thrust::make_discard_iterator(), outbuf_size.data(), normalize_quotes::start_state, stream); outbuf.resize(outbuf_size.value(stream), stream); - datasource::owning_buffer> outdata(std::move(outbuf)); + datasource::owning_buffer outdata(std::move(outbuf)); std::swap(indata, outdata); } -void normalize_whitespace(datasource::owning_buffer>& indata, +void normalize_whitespace(datasource::owning_buffer& indata, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -339,18 +339,18 @@ void normalize_whitespace(datasource::owning_buffer normalize_whitespace::TransduceToNormalizedWS{}), stream); - rmm::device_uvector outbuf(indata.size(), stream, mr); + rmm::device_buffer outbuf(indata.size(), stream, mr); rmm::device_scalar outbuf_size(stream, mr); - parser.Transduce(indata.data(), + parser.Transduce(reinterpret_cast(indata.data()), static_cast(indata.size()), - outbuf.data(), + static_cast(outbuf.data()), thrust::make_discard_iterator(), outbuf_size.data(), normalize_whitespace::start_state, stream); outbuf.resize(outbuf_size.value(stream), stream); - datasource::owning_buffer> outdata(std::move(outbuf)); + datasource::owning_buffer outdata(std::move(outbuf)); std::swap(indata, outdata); } diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index 590f70864b1..e0d0497e0a2 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -168,7 +168,7 @@ size_t estimate_size_per_subchunk(size_t chunk_size) * @param stream CUDA stream used for device memory operations and kernel launches * @returns Data source owning buffer enclosing the bytes read */ -datasource::owning_buffer> get_record_range_raw_input( +datasource::owning_buffer get_record_range_raw_input( host_span> sources, json_reader_options const& reader_opts, rmm::cuda_stream_view stream) @@ -200,8 +200,8 @@ datasource::owning_buffer> get_record_range_raw_input( ? total_source_size * estimated_compression_ratio + header_size : std::min(total_source_size, chunk_size + num_subchunks_prealloced * size_per_subchunk) + num_extra_delimiters; - rmm::device_uvector buffer(buffer_size, stream); - device_span bufspan(buffer); + rmm::device_buffer buffer(buffer_size, stream); + device_span bufspan(reinterpret_cast(buffer.data()), buffer.size()); // Offset within buffer indicating first read position std::int64_t buffer_offset = 0; @@ -213,8 +213,8 @@ datasource::owning_buffer> get_record_range_raw_input( chunk_offset == 0 ? 0 : find_first_delimiter(readbufspan, '\n', stream); if (first_delim_pos == -1) { // return empty owning datasource buffer - auto empty_buf = rmm::device_uvector(0, stream); - return datasource::owning_buffer>(std::move(empty_buf)); + auto empty_buf = rmm::device_buffer(0, stream); + return datasource::owning_buffer(std::move(empty_buf)); } else if (!should_load_all_sources) { // Find next delimiter std::int64_t next_delim_pos = -1; @@ -232,12 +232,12 @@ datasource::owning_buffer> get_record_range_raw_input( } if (next_delim_pos < buffer_offset) next_delim_pos = buffer_offset + readbufspan.size(); - return datasource::owning_buffer>( + return datasource::owning_buffer( std::move(buffer), reinterpret_cast(buffer.data()) + first_delim_pos + shift_for_nonzero_offset, next_delim_pos - first_delim_pos - shift_for_nonzero_offset); } - return datasource::owning_buffer>( + return datasource::owning_buffer( std::move(buffer), reinterpret_cast(buffer.data()) + first_delim_pos + shift_for_nonzero_offset, readbufspan.size() - first_delim_pos - shift_for_nonzero_offset); @@ -249,7 +249,7 @@ table_with_metadata read_batch(host_span> sources, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - datasource::owning_buffer> bufview = + datasource::owning_buffer bufview = get_record_range_raw_input(sources, reader_opts, stream); // If input JSON buffer has single quotes and option to normalize single quotes is enabled, diff --git a/cpp/tests/io/json/json_quote_normalization_test.cpp b/cpp/tests/io/json/json_quote_normalization_test.cpp index 55ad0afe499..3a9ba8d9f3b 100644 --- a/cpp/tests/io/json/json_quote_normalization_test.cpp +++ b/cpp/tests/io/json/json_quote_normalization_test.cpp @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include @@ -42,12 +42,11 @@ void run_test(std::string const& host_input, std::string const& expected_host_ou std::make_shared(); auto stream_view = cudf::test::get_default_stream(); - auto device_input = cudf::detail::make_device_uvector_async( - host_input, stream_view, rmm::mr::get_current_device_resource()); + auto device_input = rmm::device_buffer( + host_input.c_str(), host_input.size(), stream_view, rmm::mr::get_current_device_resource()); // Preprocessing FST - cudf::io::datasource::owning_buffer> device_data( - std::move(device_input)); + cudf::io::datasource::owning_buffer device_data(std::move(device_input)); cudf::io::json::detail::normalize_single_quotes(device_data, stream_view, rsc.get()); std::string preprocessed_host_output(device_data.size(), 0); diff --git a/cpp/tests/io/json/json_whitespace_normalization_test.cu b/cpp/tests/io/json/json_whitespace_normalization_test.cu index 8ed5fa81b12..01dd17fab98 100644 --- a/cpp/tests/io/json/json_whitespace_normalization_test.cu +++ b/cpp/tests/io/json/json_whitespace_normalization_test.cu @@ -38,12 +38,11 @@ void run_test(std::string const& host_input, std::string const& expected_host_ou // Prepare cuda stream for data transfers & kernels auto stream_view = cudf::test::get_default_stream(); - auto device_input = cudf::detail::make_device_uvector_async( - host_input, stream_view, rmm::mr::get_current_device_resource()); + auto device_input = rmm::device_buffer( + host_input.c_str(), host_input.size(), stream_view, rmm::mr::get_current_device_resource()); // Preprocessing FST - cudf::io::datasource::owning_buffer> device_data( - std::move(device_input)); + cudf::io::datasource::owning_buffer device_data(std::move(device_input)); cudf::io::json::detail::normalize_whitespace( device_data, stream_view, rmm::mr::get_current_device_resource()); From 16aa0eaa54d00d88f897766d91f9e531f64b0070 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:33:19 -1000 Subject: [PATCH 044/270] Allow DataFrame.sort_values(by=) to select an index level (#16519) closes #14794 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Matthew Murray (https://github.com/Matt711) URL: https://github.com/rapidsai/cudf/pull/16519 --- python/cudf/cudf/core/index.py | 13 ++++++++++++- python/cudf/cudf/core/indexed_frame.py | 26 +++++++++++++++++++++++++- python/cudf/cudf/tests/test_sorting.py | 20 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 094da09ab08..7f40428c1b8 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -5,6 +5,7 @@ import operator import pickle import warnings +from collections.abc import Hashable from functools import cache, cached_property from numbers import Number from typing import TYPE_CHECKING, Any, Literal, MutableMapping, cast @@ -60,7 +61,7 @@ from cudf.utils.utils import _warn_no_dask_cudf, search_range if TYPE_CHECKING: - from collections.abc import Generator, Hashable, Iterable + from collections.abc import Generator, Iterable from datetime import tzinfo @@ -450,6 +451,16 @@ def __getitem__(self, index): return self.start + index * self.step return self._as_int_index()[index] + def _get_columns_by_label(self, labels) -> Index: + # used in .sort_values + if isinstance(labels, Hashable): + if labels == self.name: + return self._as_int_index() + elif is_list_like(labels): + if list(self.names) == list(labels): + return self._as_int_index() + raise KeyError(labels) + @_performance_tracking def equals(self, other) -> bool: if isinstance(other, RangeIndex): diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 24d947a574a..3b44a0f5864 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3592,10 +3592,34 @@ def sort_values( if len(self) == 0: return self + try: + by_in_columns = self._get_columns_by_label(by) + except KeyError: + by_in_columns = None + if self.ndim == 1: + # For Series case, we're never selecting an index level. + by_in_index = None + else: + try: + by_in_index = self.index._get_columns_by_label(by) + except KeyError: + by_in_index = None + + if by_in_columns is not None and by_in_index is not None: + raise ValueError( + f"{by=} appears in the {type(self).__name__} columns " + "and as an index level which is ambiguous." + ) + elif by_in_columns is not None: + by_columns = by_in_columns + elif by_in_index is not None: + by_columns = by_in_index + else: + raise KeyError(by) # argsort the `by` column out = self._gather( GatherMap.from_column_unchecked( - self._get_columns_by_label(by)._get_sorted_inds( + by_columns._get_sorted_inds( ascending=ascending, na_position=na_position ), len(self), diff --git a/python/cudf/cudf/tests/test_sorting.py b/python/cudf/cudf/tests/test_sorting.py index a8ffce6e88b..2cf2259d9ec 100644 --- a/python/cudf/cudf/tests/test_sorting.py +++ b/python/cudf/cudf/tests/test_sorting.py @@ -405,3 +405,23 @@ def test_dataframe_scatter_by_map_empty(): df = DataFrame({"a": [], "b": []}, dtype="float64") scattered = df.scatter_by_map(df["a"]) assert len(scattered) == 0 + + +def test_sort_values_by_index_level(): + df = pd.DataFrame({"a": [1, 3, 2]}, index=pd.Index([1, 3, 2], name="b")) + cudf_df = DataFrame.from_pandas(df) + result = cudf_df.sort_values("b") + expected = df.sort_values("b") + assert_eq(result, expected) + + +def test_sort_values_by_ambiguous(): + df = pd.DataFrame({"a": [1, 3, 2]}, index=pd.Index([1, 3, 2], name="a")) + cudf_df = DataFrame.from_pandas(df) + + assert_exceptions_equal( + lfunc=df.sort_values, + rfunc=cudf_df.sort_values, + lfunc_args_and_kwargs=(["a"], {}), + rfunc_args_and_kwargs=(["a"], {}), + ) From 4cd87d3fdb0de6154504f8486ed49b685a9dceec Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:33:53 -1000 Subject: [PATCH 045/270] Fix `date_range(start, end, freq)` when end-start is divisible by freq (#16516) xref https://github.com/rapidsai/cudf/issues/16507 `date_range` generates its dates via `range`, and the end of this range was calculated via `math.ceil((end - start) / freq)`. If `(end - start) / freq` did not produce a remainder, `math.ceil` would not correctly increment this value by `1` to capture the last date. Instead, this PR uses `math.floor((end - start) / freq) + 1` to always ensure the last date is captured Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16516 --- python/cudf/cudf/core/index.py | 6 ++++-- python/cudf/cudf/core/series.py | 3 +++ python/cudf/cudf/core/tools/datetimes.py | 9 +++++---- python/cudf/cudf/tests/test_datetime.py | 6 ++++++ 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 7f40428c1b8..3eab27bd165 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2414,11 +2414,13 @@ def day_name(self, locale: str | None = None) -> Index: >>> datetime_index = cudf.date_range("2016-12-31", "2017-01-08", freq="D") >>> datetime_index DatetimeIndex(['2016-12-31', '2017-01-01', '2017-01-02', '2017-01-03', - '2017-01-04', '2017-01-05', '2017-01-06', '2017-01-07'], + '2017-01-04', '2017-01-05', '2017-01-06', '2017-01-07', + '2017-01-08'], dtype='datetime64[ns]', freq='D') >>> datetime_index.day_name() Index(['Saturday', 'Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', - 'Friday', 'Saturday'], dtype='object') + 'Friday', 'Saturday', 'Sunday'], + dtype='object') """ day_names = self._column.get_day_names(locale) return Index._from_data({self.name: day_names}) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index de57ac5f290..53675d339ac 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -801,14 +801,17 @@ def dt(self): >>> s.dt.hour 0 12 1 13 + 2 14 dtype: int16 >>> s.dt.second 0 0 1 0 + 2 0 dtype: int16 >>> s.dt.day 0 3 1 3 + 2 3 dtype: int16 Returns diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 2f77778116f..c50a36b68b5 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -951,7 +951,7 @@ def date_range( end = cudf.Scalar(end, dtype=dtype) _is_increment_sequence = end >= start - periods = math.ceil( + periods = math.floor( int(end - start) / _offset_to_nanoseconds_lower_bound(offset) ) @@ -959,9 +959,10 @@ def date_range( # Mismatched sign between (end-start) and offset, return empty # column periods = 0 - elif periods == 0: - # end == start, return exactly 1 timestamp (start) - periods = 1 + else: + # If end == start, periods == 0 and we return exactly 1 timestamp (start). + # Otherwise, since closed="both", we ensure the end point is included. + periods += 1 # We compute `end_estim` (the estimated upper bound of the date # range) below, but don't always use it. We do this to ensure diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 6bc775d2a2c..7be4faa42c3 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -2536,3 +2536,9 @@ def test_dti_methods(method, kwargs): result = getattr(cudf_dti, method)(**kwargs) expected = getattr(pd_dti, method)(**kwargs) assert_eq(result, expected) + + +def test_date_range_start_end_divisible_by_freq(): + result = cudf.date_range("2011-01-01", "2011-01-02", freq="h") + expected = pd.date_range("2011-01-01", "2011-01-02", freq="h") + assert_eq(result, expected) From 45b20d135a290d5f14e291316e94674653f71737 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:22:15 -1000 Subject: [PATCH 046/270] Preserve array name in MultiIndex.from_arrays (#16515) xref https://github.com/rapidsai/cudf/issues/16507 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Matthew Murray (https://github.com/Matt711) URL: https://github.com/rapidsai/cudf/pull/16515 --- python/cudf/cudf/core/multiindex.py | 4 ++++ python/cudf/cudf/tests/test_multiindex.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 9646b34830f..ab88b191570 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1394,12 +1394,16 @@ def from_arrays( raise TypeError(error_msg) codes = [] levels = [] + names_from_arrays = [] for array in arrays: if not (is_list_like(array) or is_column_like(array)): raise TypeError(error_msg) code, level = factorize(array, sort=True) codes.append(code) levels.append(level) + names_from_arrays.append(getattr(array, "name", None)) + if names is None: + names = names_from_arrays return cls( codes=codes, levels=levels, sortorder=sortorder, names=names ) diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index b7314a36e73..a68f4574da3 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -2179,3 +2179,13 @@ def test_unique_level(): result = pd_mi.unique(level=1) expected = cudf_mi.unique(level=1) assert_eq(result, expected) + + +@pytest.mark.parametrize( + "idx", [pd.Index, pd.CategoricalIndex, pd.DatetimeIndex, pd.TimedeltaIndex] +) +def test_from_arrays_infer_names(idx): + arrays = [idx([1], name="foo"), idx([2], name="bar")] + expected = pd.MultiIndex.from_arrays(arrays) + result = cudf.MultiIndex.from_arrays(arrays) + assert_eq(result, expected) From a3dc14fcea938729c7c9468bd6a64331395b2f78 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 12 Aug 2024 07:56:48 -1000 Subject: [PATCH 047/270] Disallow indexing by selecting duplicate labels (#16514) xref https://github.com/rapidsai/cudf/issues/16507 I would say this was a bug before because we would silently return a new DataFrame with just `len(set(column_labels))` when selecting by column. Now this operation raises since duplicate column labels are generally not supported. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - https://github.com/brandon-b-miller URL: https://github.com/rapidsai/cudf/pull/16514 --- python/cudf/cudf/core/column_accessor.py | 4 ++++ python/cudf/cudf/tests/test_indexing.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 819d351b2c4..83596704672 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -530,6 +530,10 @@ def _select_by_label_list_like(self, key: Any) -> ColumnAccessor: ) else: data = {k: self._grouped_data[k] for k in key} + if len(data) != len(key): + raise ValueError( + "Selecting duplicate column labels is not supported." + ) if self.multiindex: data = dict(_to_flat_dict_inner(data)) return self.__class__( diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index 7005cbc6834..716b4dc6acd 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -2361,3 +2361,11 @@ def test_sliced_categorical_as_ordered(): name="a", ) assert_eq(result, expected) + + +def test_duplicate_labels_raises(): + df = cudf.DataFrame([[1, 2]], columns=["a", "b"]) + with pytest.raises(ValueError): + df[["a", "a"]] + with pytest.raises(ValueError): + df.loc[:, ["a", "a"]] From 091cb72294a394deb176600e74c7cb115cfff05a Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:48:02 -0400 Subject: [PATCH 048/270] Remove deprecated public APIs from libcudf (#16524) Removing some more deprecated public libcudf APIs. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/16524 --- cpp/include/cudf/strings/replace.hpp | 12 ------------ cpp/include/cudf/utilities/type_checks.hpp | 19 ------------------- cpp/src/strings/replace/multi.cu | 11 ----------- cpp/src/utilities/type_checks.cpp | 5 ----- 4 files changed, 47 deletions(-) diff --git a/cpp/include/cudf/strings/replace.hpp b/cpp/include/cudf/strings/replace.hpp index 5b4ffb98f99..f450b77ad7a 100644 --- a/cpp/include/cudf/strings/replace.hpp +++ b/cpp/include/cudf/strings/replace.hpp @@ -160,18 +160,6 @@ std::unique_ptr replace_multiple( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); -/** - * @copydoc cudf::strings::replace_multiple - * - * @deprecated since 24.08 - */ -[[deprecated]] std::unique_ptr replace( - strings_column_view const& input, - strings_column_view const& targets, - strings_column_view const& repls, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); - /** @} */ // end of doxygen group } // namespace strings } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/utilities/type_checks.hpp b/cpp/include/cudf/utilities/type_checks.hpp index 4fcbca09d17..aeb5db57830 100644 --- a/cpp/include/cudf/utilities/type_checks.hpp +++ b/cpp/include/cudf/utilities/type_checks.hpp @@ -22,25 +22,6 @@ namespace CUDF_EXPORT cudf { -/** - * @brief Compare the types of two `column_view`s - * - * @deprecated Since 24.06. Use cudf::have_same_types instead. - * - * This function returns true if the type of `lhs` equals that of `rhs`. - * - For fixed point types, the scale is compared. - * - For dictionary types, the type of the keys are compared if both are - * non-empty columns. - * - For lists types, the type of child columns are compared recursively. - * - For struct types, the type of each field are compared in order. - * - For all other types, the `id` of `data_type` is compared. - * - * @param lhs The first `column_view` to compare - * @param rhs The second `column_view` to compare - * @return true if column types match - */ -[[deprecated]] bool column_types_equal(column_view const& lhs, column_view const& rhs); - /** * @brief Compare the type IDs of two `column_view`s * diff --git a/cpp/src/strings/replace/multi.cu b/cpp/src/strings/replace/multi.cu index 2ca22f0e017..b5248700d53 100644 --- a/cpp/src/strings/replace/multi.cu +++ b/cpp/src/strings/replace/multi.cu @@ -533,16 +533,5 @@ std::unique_ptr replace_multiple(strings_column_view const& strings, return detail::replace_multiple(strings, targets, repls, stream, mr); } -// deprecated in 24.08 -std::unique_ptr replace(strings_column_view const& strings, - strings_column_view const& targets, - strings_column_view const& repls, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - CUDF_FUNC_RANGE(); - return detail::replace_multiple(strings, targets, repls, stream, mr); -} - } // namespace strings } // namespace cudf diff --git a/cpp/src/utilities/type_checks.cpp b/cpp/src/utilities/type_checks.cpp index dac981fb532..3095b342748 100644 --- a/cpp/src/utilities/type_checks.cpp +++ b/cpp/src/utilities/type_checks.cpp @@ -139,11 +139,6 @@ bool have_same_types(column_view const& lhs, column_view const& rhs) return type_dispatcher(lhs.type(), columns_equal_fn{}, lhs, rhs); } -bool column_types_equal(column_view const& lhs, column_view const& rhs) -{ - return have_same_types(lhs, rhs); -} - bool have_same_types(column_view const& lhs, scalar const& rhs) { return type_dispatcher(lhs.type(), column_scalar_equal_fn{}, lhs, rhs); From cce00c00b0ae374ee72332aaea5fcd1cc121e85a Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Mon, 12 Aug 2024 14:38:37 -0700 Subject: [PATCH 049/270] Pass batch size to JSON reader using environment variable (#16502) The JSON reader set the batch size to `INT_MAX` bytes since the motivation for implementing a batched JSON reader was to parse source files whose total size is larger than `INT_MAX` (#16138, #16162). However, we can use a much smaller batch size to evaluate the correctness of the reader and speed up tests significantly. This PR focuses on reducing runtime of the batched reader test by setting the batch size to be used by the reader as an environment variable. The runtime of `JsonLargeReaderTest.MultiBatch` in `LARGE_STRINGS_TEST` gtest drops from ~52s to ~3s. Authors: - Shruti Shivakumar (https://github.com/shrshi) Approvers: - Nghia Truong (https://github.com/ttnghia) - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16502 --- cpp/CMakeLists.txt | 1 - cpp/src/io/json/byte_range_info.cu | 37 ---- cpp/src/io/json/read_json.cu | 291 +++++++++++++++----------- cpp/src/io/json/read_json.hpp | 28 ++- cpp/tests/large_strings/json_tests.cu | 20 +- 5 files changed, 204 insertions(+), 173 deletions(-) delete mode 100644 cpp/src/io/json/byte_range_info.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 310bc99b279..eeafc411874 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -392,7 +392,6 @@ add_library( src/io/csv/reader_impl.cu src/io/csv/writer_impl.cu src/io/functions.cpp - src/io/json/byte_range_info.cu src/io/json/json_column.cu src/io/json/json_normalization.cu src/io/json/json_tree.cu diff --git a/cpp/src/io/json/byte_range_info.cu b/cpp/src/io/json/byte_range_info.cu deleted file mode 100644 index 258a40b0dd3..00000000000 --- a/cpp/src/io/json/byte_range_info.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include - -#include - -namespace cudf::io::json::detail { - -// Extract the first character position in the string. -size_type find_first_delimiter(device_span d_data, - char const delimiter, - rmm::cuda_stream_view stream) -{ - auto const first_delimiter_position = - thrust::find(rmm::exec_policy(stream), d_data.begin(), d_data.end(), delimiter); - return first_delimiter_position != d_data.end() ? first_delimiter_position - d_data.begin() : -1; -} - -} // namespace cudf::io::json::detail diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index e0d0497e0a2..2658cbbed2f 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -31,6 +31,7 @@ #include #include +#include #include #include @@ -38,11 +39,14 @@ namespace cudf::io::json::detail { -size_t sources_size(host_span> const sources, - size_t range_offset, - size_t range_size) +namespace { + +// Return total size of sources enclosing the passed range +std::size_t sources_size(host_span> const sources, + std::size_t range_offset, + std::size_t range_size) { - return std::accumulate(sources.begin(), sources.end(), 0ul, [=](size_t sum, auto& source) { + return std::accumulate(sources.begin(), sources.end(), 0ul, [=](std::size_t sum, auto& source) { auto const size = source->size(); // TODO take care of 0, 0, or *, 0 case. return sum + @@ -50,109 +54,55 @@ size_t sources_size(host_span> const sources, }); } +// Return estimated size of subchunk using a heuristic involving the byte range size and the minimum +// subchunk size +std::size_t estimate_size_per_subchunk(std::size_t chunk_size) +{ + auto geometric_mean = [](double a, double b) { return std::sqrt(a * b); }; + // NOTE: heuristic for choosing subchunk size: geometric mean of minimum subchunk size (set to + // 10kb) and the byte range size + return geometric_mean(std::ceil(static_cast(chunk_size) / num_subchunks), + min_subchunk_size); +} + /** - * @brief Read from array of data sources into RMM buffer. The size of the returned device span - can be larger than the number of bytes requested from the list of sources when - the range to be read spans across multiple sources. This is due to the delimiter - characters inserted after the end of each accessed source. + * @brief Return the upper bound on the batch size for the JSON reader. * - * @param buffer Device span buffer to which data is read - * @param sources Array of data sources - * @param compression Compression format of source - * @param range_offset Number of bytes to skip from source start - * @param range_size Number of bytes to read from source - * @param stream CUDA stream used for device memory operations and kernel launches - * @returns A subspan of the input device span containing data read + * The datasources passed to the JSON reader are split into batches demarcated by byte range + * offsets and read iteratively. The batch size is capped at INT_MAX bytes, which is the + * default value returned by the function. This value can be overridden at runtime using the + * environment variable LIBCUDF_JSON_BATCH_SIZE + * + * @return size in bytes */ -device_span ingest_raw_input(device_span buffer, - host_span> sources, - compression_type compression, - size_t range_offset, - size_t range_size, - rmm::cuda_stream_view stream) +std::size_t get_batch_size_upper_bound() { - CUDF_FUNC_RANGE(); - // We append a line delimiter between two files to make sure the last line of file i and the first - // line of file i+1 don't end up on the same JSON line, if file i does not already end with a line - // delimiter. - auto constexpr num_delimiter_chars = 1; - - if (compression == compression_type::NONE) { - auto delimiter_map = cudf::detail::make_empty_host_vector(sources.size(), stream); - std::vector prefsum_source_sizes(sources.size()); - std::vector> h_buffers; - size_t bytes_read = 0; - std::transform_inclusive_scan(sources.begin(), - sources.end(), - prefsum_source_sizes.begin(), - std::plus{}, - [](std::unique_ptr const& s) { return s->size(); }); - auto upper = - std::upper_bound(prefsum_source_sizes.begin(), prefsum_source_sizes.end(), range_offset); - size_t start_source = std::distance(prefsum_source_sizes.begin(), upper); - - auto const total_bytes_to_read = - std::min(range_size, prefsum_source_sizes.back() - range_offset); - range_offset -= start_source ? prefsum_source_sizes[start_source - 1] : 0; - for (size_t i = start_source; i < sources.size() && bytes_read < total_bytes_to_read; i++) { - if (sources[i]->is_empty()) continue; - auto data_size = - std::min(sources[i]->size() - range_offset, total_bytes_to_read - bytes_read); - auto destination = reinterpret_cast(buffer.data()) + bytes_read + - (num_delimiter_chars * delimiter_map.size()); - if (sources[i]->is_device_read_preferred(data_size)) { - bytes_read += sources[i]->device_read(range_offset, data_size, destination, stream); - } else { - h_buffers.emplace_back(sources[i]->host_read(range_offset, data_size)); - auto const& h_buffer = h_buffers.back(); - CUDF_CUDA_TRY(cudaMemcpyAsync( - destination, h_buffer->data(), h_buffer->size(), cudaMemcpyHostToDevice, stream.value())); - bytes_read += h_buffer->size(); - } - range_offset = 0; - delimiter_map.push_back(bytes_read + (num_delimiter_chars * delimiter_map.size())); - } - // Removing delimiter inserted after last non-empty source is read - if (!delimiter_map.empty()) { delimiter_map.pop_back(); } - - // If this is a multi-file source, we scatter the JSON line delimiters between files - if (sources.size() > 1) { - static_assert(num_delimiter_chars == 1, - "Currently only single-character delimiters are supported"); - auto const delimiter_source = thrust::make_constant_iterator('\n'); - auto const d_delimiter_map = cudf::detail::make_device_uvector_async( - delimiter_map, stream, rmm::mr::get_current_device_resource()); - thrust::scatter(rmm::exec_policy_nosync(stream), - delimiter_source, - delimiter_source + d_delimiter_map.size(), - d_delimiter_map.data(), - buffer.data()); - } - stream.synchronize(); - return buffer.first(bytes_read + (delimiter_map.size() * num_delimiter_chars)); - } - // TODO: allow byte range reading from multiple compressed files. - auto remaining_bytes_to_read = std::min(range_size, sources[0]->size() - range_offset); - auto hbuffer = std::vector(remaining_bytes_to_read); - // Single read because only a single compressed source is supported - // Reading to host because decompression of a single block is much faster on the CPU - sources[0]->host_read(range_offset, remaining_bytes_to_read, hbuffer.data()); - auto uncomp_data = decompress(compression, hbuffer); - CUDF_CUDA_TRY(cudaMemcpyAsync(buffer.data(), - reinterpret_cast(uncomp_data.data()), - uncomp_data.size() * sizeof(char), - cudaMemcpyHostToDevice, - stream.value())); - stream.synchronize(); - return buffer.first(uncomp_data.size()); + auto const batch_size_str = std::getenv("LIBCUDF_JSON_BATCH_SIZE"); + int64_t const batch_size = batch_size_str != nullptr ? std::atol(batch_size_str) : 0L; + auto const batch_limit = static_cast(std::numeric_limits::max()); + auto const batch_size_upper_bound = static_cast( + (batch_size > 0 && batch_size < batch_limit) ? batch_size : batch_limit); + return batch_size_upper_bound; } -size_t estimate_size_per_subchunk(size_t chunk_size) +/** + * @brief Extract the first delimiter character position in the string + * + * @param d_data Device span in which to search for delimiter character + * @param delimiter Delimiter character to search for + * @param stream CUDA stream used for device memory operations and kernel launches + * + * @return Position of first delimiter character in device array + */ +size_type find_first_delimiter(device_span d_data, + char const delimiter, + rmm::cuda_stream_view stream) { - auto geometric_mean = [](double a, double b) { return std::sqrt(a * b); }; - // NOTE: heuristic for choosing subchunk size: geometric mean of minimum subchunk size (set to - // 10kb) and the byte range size - return geometric_mean(std::ceil((double)chunk_size / num_subchunks), min_subchunk_size); + auto const first_delimiter_position = + thrust::find(rmm::exec_policy(stream), d_data.begin(), d_data.end(), delimiter); + return first_delimiter_position != d_data.end() + ? static_cast(thrust::distance(d_data.begin(), first_delimiter_position)) + : -1; } /** @@ -175,12 +125,12 @@ datasource::owning_buffer get_record_range_raw_input( { CUDF_FUNC_RANGE(); - size_t const total_source_size = sources_size(sources, 0, 0); + std::size_t const total_source_size = sources_size(sources, 0, 0); auto constexpr num_delimiter_chars = 1; auto const num_extra_delimiters = num_delimiter_chars * (sources.size() - 1); compression_type const reader_compression = reader_opts.get_compression(); - size_t const chunk_offset = reader_opts.get_byte_range_offset(); - size_t chunk_size = reader_opts.get_byte_range_size(); + std::size_t const chunk_offset = reader_opts.get_byte_range_offset(); + std::size_t chunk_size = reader_opts.get_byte_range_size(); CUDF_EXPECTS(total_source_size ? chunk_offset < total_source_size : !chunk_offset, "Invalid offsetting", @@ -188,14 +138,14 @@ datasource::owning_buffer get_record_range_raw_input( auto should_load_all_sources = !chunk_size || chunk_size >= total_source_size - chunk_offset; chunk_size = should_load_all_sources ? total_source_size - chunk_offset : chunk_size; - int const num_subchunks_prealloced = should_load_all_sources ? 0 : max_subchunks_prealloced; - size_t const size_per_subchunk = estimate_size_per_subchunk(chunk_size); + int const num_subchunks_prealloced = should_load_all_sources ? 0 : max_subchunks_prealloced; + std::size_t const size_per_subchunk = estimate_size_per_subchunk(chunk_size); // The allocation for single source compressed input is estimated by assuming a ~4:1 // compression ratio. For uncompressed inputs, we can getter a better estimate using the idea // of subchunks. auto constexpr header_size = 4096; - size_t const buffer_size = + std::size_t const buffer_size = reader_compression != compression_type::NONE ? total_source_size * estimated_compression_ratio + header_size : std::min(total_source_size, chunk_size + num_subchunks_prealloced * size_per_subchunk) + @@ -217,8 +167,8 @@ datasource::owning_buffer get_record_range_raw_input( return datasource::owning_buffer(std::move(empty_buf)); } else if (!should_load_all_sources) { // Find next delimiter - std::int64_t next_delim_pos = -1; - size_t next_subchunk_start = chunk_offset + chunk_size; + std::int64_t next_delim_pos = -1; + std::size_t next_subchunk_start = chunk_offset + chunk_size; while (next_subchunk_start < total_source_size && next_delim_pos < buffer_offset) { buffer_offset += readbufspan.size(); readbufspan = ingest_raw_input(bufspan.last(buffer_size - buffer_offset), @@ -243,6 +193,8 @@ datasource::owning_buffer get_record_range_raw_input( readbufspan.size() - first_delim_pos - shift_for_nonzero_offset); } +// Helper function to read the current batch using byte range offsets and size +// passed table_with_metadata read_batch(host_span> sources, json_reader_options const& reader_opts, rmm::cuda_stream_view stream, @@ -270,6 +222,92 @@ table_with_metadata read_batch(host_span> sources, return device_parse_nested_json(buffer, reader_opts, stream, mr); } +} // anonymous namespace + +device_span ingest_raw_input(device_span buffer, + host_span> sources, + compression_type compression, + std::size_t range_offset, + std::size_t range_size, + rmm::cuda_stream_view stream) +{ + CUDF_FUNC_RANGE(); + // We append a line delimiter between two files to make sure the last line of file i and the first + // line of file i+1 don't end up on the same JSON line, if file i does not already end with a line + // delimiter. + auto constexpr num_delimiter_chars = 1; + + if (compression == compression_type::NONE) { + auto delimiter_map = cudf::detail::make_empty_host_vector(sources.size(), stream); + std::vector prefsum_source_sizes(sources.size()); + std::vector> h_buffers; + std::size_t bytes_read = 0; + std::transform_inclusive_scan(sources.begin(), + sources.end(), + prefsum_source_sizes.begin(), + std::plus{}, + [](std::unique_ptr const& s) { return s->size(); }); + auto upper = + std::upper_bound(prefsum_source_sizes.begin(), prefsum_source_sizes.end(), range_offset); + std::size_t start_source = std::distance(prefsum_source_sizes.begin(), upper); + + auto const total_bytes_to_read = + std::min(range_size, prefsum_source_sizes.back() - range_offset); + range_offset -= start_source ? prefsum_source_sizes[start_source - 1] : 0; + for (std::size_t i = start_source; i < sources.size() && bytes_read < total_bytes_to_read; + i++) { + if (sources[i]->is_empty()) continue; + auto data_size = + std::min(sources[i]->size() - range_offset, total_bytes_to_read - bytes_read); + auto destination = reinterpret_cast(buffer.data()) + bytes_read + + (num_delimiter_chars * delimiter_map.size()); + if (sources[i]->is_device_read_preferred(data_size)) { + bytes_read += sources[i]->device_read(range_offset, data_size, destination, stream); + } else { + h_buffers.emplace_back(sources[i]->host_read(range_offset, data_size)); + auto const& h_buffer = h_buffers.back(); + CUDF_CUDA_TRY(cudaMemcpyAsync( + destination, h_buffer->data(), h_buffer->size(), cudaMemcpyHostToDevice, stream.value())); + bytes_read += h_buffer->size(); + } + range_offset = 0; + delimiter_map.push_back(bytes_read + (num_delimiter_chars * delimiter_map.size())); + } + // Removing delimiter inserted after last non-empty source is read + if (!delimiter_map.empty()) { delimiter_map.pop_back(); } + + // If this is a multi-file source, we scatter the JSON line delimiters between files + if (sources.size() > 1) { + static_assert(num_delimiter_chars == 1, + "Currently only single-character delimiters are supported"); + auto const delimiter_source = thrust::make_constant_iterator('\n'); + auto const d_delimiter_map = cudf::detail::make_device_uvector_async( + delimiter_map, stream, rmm::mr::get_current_device_resource()); + thrust::scatter(rmm::exec_policy_nosync(stream), + delimiter_source, + delimiter_source + d_delimiter_map.size(), + d_delimiter_map.data(), + buffer.data()); + } + stream.synchronize(); + return buffer.first(bytes_read + (delimiter_map.size() * num_delimiter_chars)); + } + // TODO: allow byte range reading from multiple compressed files. + auto remaining_bytes_to_read = std::min(range_size, sources[0]->size() - range_offset); + auto hbuffer = std::vector(remaining_bytes_to_read); + // Single read because only a single compressed source is supported + // Reading to host because decompression of a single block is much faster on the CPU + sources[0]->host_read(range_offset, remaining_bytes_to_read, hbuffer.data()); + auto uncomp_data = decompress(compression, hbuffer); + CUDF_CUDA_TRY(cudaMemcpyAsync(buffer.data(), + reinterpret_cast(uncomp_data.data()), + uncomp_data.size() * sizeof(char), + cudaMemcpyHostToDevice, + stream.value())); + stream.synchronize(); + return buffer.first(uncomp_data.size()); +} + table_with_metadata read_json(host_span> sources, json_reader_options const& reader_opts, rmm::cuda_stream_view stream, @@ -296,15 +334,16 @@ table_with_metadata read_json(host_span> sources, * Note that the batched reader does not work for compressed inputs or for regular * JSON inputs. */ - size_t const total_source_size = sources_size(sources, 0, 0); - size_t chunk_offset = reader_opts.get_byte_range_offset(); - size_t chunk_size = reader_opts.get_byte_range_size(); - chunk_size = !chunk_size ? total_source_size - chunk_offset - : std::min(chunk_size, total_source_size - chunk_offset); + std::size_t const total_source_size = sources_size(sources, 0, 0); + std::size_t chunk_offset = reader_opts.get_byte_range_offset(); + std::size_t chunk_size = reader_opts.get_byte_range_size(); + chunk_size = !chunk_size ? total_source_size - chunk_offset + : std::min(chunk_size, total_source_size - chunk_offset); - size_t const size_per_subchunk = estimate_size_per_subchunk(chunk_size); - size_t const batch_size_ub = - std::numeric_limits::max() - (max_subchunks_prealloced * size_per_subchunk); + std::size_t const size_per_subchunk = estimate_size_per_subchunk(chunk_size); + std::size_t const batch_size_upper_bound = get_batch_size_upper_bound(); + std::size_t const batch_size = + batch_size_upper_bound - (max_subchunks_prealloced * size_per_subchunk); /* * Identify the position (zero-indexed) of starting source file from which to begin @@ -314,10 +353,10 @@ table_with_metadata read_json(host_span> sources, */ // Prefix sum of source file sizes - size_t pref_source_size = 0; + std::size_t pref_source_size = 0; // Starting source file from which to being batching evaluated using byte range offset - size_t const start_source = [chunk_offset, &sources, &pref_source_size]() { - for (size_t src_idx = 0; src_idx < sources.size(); ++src_idx) { + std::size_t const start_source = [chunk_offset, &sources, &pref_source_size]() { + for (std::size_t src_idx = 0; src_idx < sources.size(); ++src_idx) { if (pref_source_size + sources[src_idx]->size() > chunk_offset) { return src_idx; } pref_source_size += sources[src_idx]->size(); } @@ -329,16 +368,16 @@ table_with_metadata read_json(host_span> sources, * batch begins, and `end_bytes_size` gives the terminal bytes position after which reading * stops. */ - size_t pref_bytes_size = chunk_offset; - size_t end_bytes_size = chunk_offset + chunk_size; - std::vector batch_offsets{pref_bytes_size}; - for (size_t i = start_source; i < sources.size() && pref_bytes_size < end_bytes_size;) { + std::size_t pref_bytes_size = chunk_offset; + std::size_t end_bytes_size = chunk_offset + chunk_size; + std::vector batch_offsets{pref_bytes_size}; + for (std::size_t i = start_source; i < sources.size() && pref_bytes_size < end_bytes_size;) { pref_source_size += sources[i]->size(); // If the current source file can subsume multiple batches, we split the file until the // boundary of the last batch exceeds the end of the file (indexed by `pref_source_size`) while (pref_bytes_size < end_bytes_size && - pref_source_size >= std::min(pref_bytes_size + batch_size_ub, end_bytes_size)) { - auto next_batch_size = std::min(batch_size_ub, end_bytes_size - pref_bytes_size); + pref_source_size >= std::min(pref_bytes_size + batch_size, end_bytes_size)) { + auto next_batch_size = std::min(batch_size, end_bytes_size - pref_bytes_size); batch_offsets.push_back(batch_offsets.back() + next_batch_size); pref_bytes_size += next_batch_size; } @@ -356,7 +395,7 @@ table_with_metadata read_json(host_span> sources, // Dispatch individual batches to read_batch and push the resulting table into // partial_tables array. Note that the reader options need to be updated for each // batch to adjust byte range offset and byte range size. - for (size_t i = 0; i < batch_offsets.size() - 1; i++) { + for (std::size_t i = 0; i < batch_offsets.size() - 1; i++) { batched_reader_opts.set_byte_range_offset(batch_offsets[i]); batched_reader_opts.set_byte_range_size(batch_offsets[i + 1] - batch_offsets[i]); partial_tables.emplace_back( diff --git a/cpp/src/io/json/read_json.hpp b/cpp/src/io/json/read_json.hpp index 32de4ebabfa..7e3a920f00d 100644 --- a/cpp/src/io/json/read_json.hpp +++ b/cpp/src/io/json/read_json.hpp @@ -37,6 +37,20 @@ constexpr size_t min_subchunk_size = 10000; constexpr int estimated_compression_ratio = 4; constexpr int max_subchunks_prealloced = 3; +/** + * @brief Read from array of data sources into RMM buffer. The size of the returned device span + can be larger than the number of bytes requested from the list of sources when + the range to be read spans across multiple sources. This is due to the delimiter + characters inserted after the end of each accessed source. + * + * @param buffer Device span buffer to which data is read + * @param sources Array of data sources + * @param compression Compression format of source + * @param range_offset Number of bytes to skip from source start + * @param range_size Number of bytes to read from source + * @param stream CUDA stream used for device memory operations and kernel launches + * @returns A subspan of the input device span containing data read + */ device_span ingest_raw_input(device_span buffer, host_span> sources, compression_type compression, @@ -44,14 +58,20 @@ device_span ingest_raw_input(device_span buffer, size_t range_size, rmm::cuda_stream_view stream); +/** + * @brief Reads and returns the entire data set in batches. + * + * @param sources Input `datasource` objects to read the dataset from + * @param reader_opts Settings for controlling reading behavior + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource to use for device memory allocation + * + * @return cudf::table object that contains the array of cudf::column. + */ table_with_metadata read_json(host_span> sources, json_reader_options const& reader_opts, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); -size_type find_first_delimiter(device_span d_data, - char const delimiter, - rmm::cuda_stream_view stream); - } // namespace io::json::detail } // namespace CUDF_EXPORT cudf diff --git a/cpp/tests/large_strings/json_tests.cu b/cpp/tests/large_strings/json_tests.cu index 49abf7b484d..e34ab991c11 100644 --- a/cpp/tests/large_strings/json_tests.cu +++ b/cpp/tests/large_strings/json_tests.cu @@ -28,13 +28,17 @@ struct JsonLargeReaderTest : public cudf::test::StringsLargeTest {}; TEST_F(JsonLargeReaderTest, MultiBatch) { - std::string json_string = R"( + std::string json_string = R"( { "a": { "y" : 6}, "b" : [1, 2, 3], "c": 11 } { "a": { "y" : 6}, "b" : [4, 5 ], "c": 12 } { "a": { "y" : 6}, "b" : [6 ], "c": 13 } { "a": { "y" : 6}, "b" : [7 ], "c": 14 })"; - constexpr size_t batch_size_ub = std::numeric_limits::max(); - constexpr size_t expected_file_size = 1.5 * static_cast(batch_size_ub); + + std::size_t const batch_size_upper_bound = std::numeric_limits::max() / 16; + // set smaller batch_size to reduce file size and execution time + setenv("LIBCUDF_JSON_BATCH_SIZE", std::to_string(batch_size_upper_bound).c_str(), 1); + + constexpr std::size_t expected_file_size = 1.5 * static_cast(batch_size_upper_bound); std::size_t const log_repetitions = static_cast(std::ceil(std::log2(expected_file_size / json_string.size()))); @@ -66,8 +70,11 @@ TEST_F(JsonLargeReaderTest, MultiBatch) datasources.emplace_back(cudf::io::datasource::create(hb)); } // Test for different chunk sizes - std::vector chunk_sizes{ - batch_size_ub / 4, batch_size_ub / 2, batch_size_ub, static_cast(batch_size_ub * 2)}; + std::vector chunk_sizes{batch_size_upper_bound / 4, + batch_size_upper_bound / 2, + batch_size_upper_bound, + static_cast(batch_size_upper_bound * 2)}; + for (auto chunk_size : chunk_sizes) { auto const tables = split_byte_range_reading(datasources, @@ -86,4 +93,7 @@ TEST_F(JsonLargeReaderTest, MultiBatch) // cannot use EQUAL due to concatenate removing null mask CUDF_TEST_EXPECT_TABLES_EQUIVALENT(current_reader_table.tbl->view(), result->view()); } + + // go back to normal batch_size + unsetenv("LIBCUDF_LARGE_STRINGS_THRESHOLD"); } From e5f8dd33d78a2c964f8d6bac895deb73a9be7aa6 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Mon, 12 Aug 2024 16:52:52 -0500 Subject: [PATCH 050/270] Update the java code to properly deal with lists being returned as strings (#16536) Recently some JSON parsing was updated so lists could be returned as strings. This updates the java code so that when cleaning up the results to match the desired schema that it can handle corner cases associated with lists and structs properly. Tests are covered in the Spark plugin, but I am happy to add some here if we really want to validate that part of this. Authors: - Robert (Bobby) Evans (https://github.com/revans2) Approvers: - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16536 --- java/src/main/java/ai/rapids/cudf/Table.java | 29 +++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/Table.java b/java/src/main/java/ai/rapids/cudf/Table.java index 4e737451ed6..36e342cae13 100644 --- a/java/src/main/java/ai/rapids/cudf/Table.java +++ b/java/src/main/java/ai/rapids/cudf/Table.java @@ -1084,7 +1084,12 @@ private static DidViewChange gatherJSONColumns(Schema schema, TableWithMeta.Nest // The types don't match so just return the input unchanged... return DidViewChange.no(); } else { - String[] foundNames = children.getNames(); + String[] foundNames; + if (children == null) { + foundNames = new String[0]; + } else { + foundNames = children.getNames(); + } HashMap indices = new HashMap<>(); for (int i = 0; i < foundNames.length; i++) { indices.put(foundNames[i], i); @@ -1101,8 +1106,9 @@ private static DidViewChange gatherJSONColumns(Schema schema, TableWithMeta.Nest for (int i = 0; i < columns.length; i++) { String neededColumnName = neededNames[i]; Integer index = indices.get(neededColumnName); + Schema childSchema = schema.getChild(i); if (index != null) { - if (schema.getChild(i).isStructOrHasStructDescendant()) { + if (childSchema.isStructOrHasStructDescendant()) { ColumnView child = cv.getChildColumnView(index); boolean shouldCloseChild = true; try { @@ -1131,8 +1137,23 @@ private static DidViewChange gatherJSONColumns(Schema schema, TableWithMeta.Nest } } else { somethingChanged = true; - try (Scalar s = Scalar.fromNull(types[i])) { - columns[i] = ColumnVector.fromScalar(s, (int) cv.getRowCount()); + if (types[i] == DType.LIST) { + try (Scalar s = Scalar.listFromNull(childSchema.getChild(0).asHostDataType())) { + columns[i] = ColumnVector.fromScalar(s, (int) cv.getRowCount()); + } + } else if (types[i] == DType.STRUCT) { + int numStructChildren = childSchema.getNumChildren(); + HostColumnVector.DataType[] structChildren = new HostColumnVector.DataType[numStructChildren]; + for (int structChildIndex = 0; structChildIndex < numStructChildren; structChildIndex++) { + structChildren[structChildIndex] = childSchema.getChild(structChildIndex).asHostDataType(); + } + try (Scalar s = Scalar.structFromNull(structChildren)) { + columns[i] = ColumnVector.fromScalar(s, (int) cv.getRowCount()); + } + } else { + try (Scalar s = Scalar.fromNull(types[i])) { + columns[i] = ColumnVector.fromScalar(s, (int) cv.getRowCount()); + } } } } From 7178bf2eb34334db909a151926d8112c441b3b09 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 13 Aug 2024 08:45:44 -0400 Subject: [PATCH 051/270] Rework cudf::io::text::byte_range_info class member functions (#16518) Adds `const` declarations to appropriate member functions in class `cudf::io::text::byte_range_info` and moves the ctor implementation to .cpp file. This helps with using the `byte_range_info` objects in `const` variables and inside of `const` functions. Found while working on #15983 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Muhammad Haseeb (https://github.com/mhaseeb123) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16518 --- cpp/include/cudf/io/text/byte_range_info.hpp | 21 ++++++++------------ cpp/src/io/text/byte_range_info.cpp | 7 +++++++ cpp/src/io/text/multibyte_split.cu | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/cpp/include/cudf/io/text/byte_range_info.hpp b/cpp/include/cudf/io/text/byte_range_info.hpp index 7e9256be1d3..5f3c91dc99c 100644 --- a/cpp/include/cudf/io/text/byte_range_info.hpp +++ b/cpp/include/cudf/io/text/byte_range_info.hpp @@ -16,7 +16,6 @@ #pragma once -#include #include #include @@ -40,53 +39,49 @@ class byte_range_info { int64_t _size{}; ///< size in bytes public: - constexpr byte_range_info() = default; + byte_range_info() = default; /** * @brief Constructs a byte_range_info object * * @param offset offset in bytes * @param size size in bytes */ - constexpr byte_range_info(int64_t offset, int64_t size) : _offset(offset), _size(size) - { - CUDF_EXPECTS(offset >= 0, "offset must be non-negative"); - CUDF_EXPECTS(size >= 0, "size must be non-negative"); - } + byte_range_info(int64_t offset, int64_t size); /** * @brief Copy constructor * * @param other byte_range_info object to copy */ - constexpr byte_range_info(byte_range_info const& other) noexcept = default; + byte_range_info(byte_range_info const& other) noexcept = default; /** * @brief Copy assignment operator * * @param other byte_range_info object to copy * @return this object after copying */ - constexpr byte_range_info& operator=(byte_range_info const& other) noexcept = default; + byte_range_info& operator=(byte_range_info const& other) noexcept = default; /** * @brief Get the offset in bytes * * @return Offset in bytes */ - [[nodiscard]] constexpr int64_t offset() { return _offset; } + [[nodiscard]] int64_t offset() const { return _offset; } /** * @brief Get the size in bytes * * @return Size in bytes */ - [[nodiscard]] constexpr int64_t size() { return _size; } + [[nodiscard]] int64_t size() const { return _size; } /** * @brief Returns whether the span is empty. * - * @return true iff the span is empty, i.e. `size() == 0` + * @return true iff the range is empty, i.e. `size() == 0` */ - [[nodiscard]] constexpr bool empty() { return size() == 0; } + [[nodiscard]] bool is_empty() const { return size() == 0; } }; /** diff --git a/cpp/src/io/text/byte_range_info.cpp b/cpp/src/io/text/byte_range_info.cpp index 6a7836ed4e1..fe811739b97 100644 --- a/cpp/src/io/text/byte_range_info.cpp +++ b/cpp/src/io/text/byte_range_info.cpp @@ -16,6 +16,7 @@ #include #include +#include #include @@ -23,6 +24,12 @@ namespace cudf { namespace io { namespace text { +byte_range_info::byte_range_info(int64_t offset, int64_t size) : _offset(offset), _size(size) +{ + CUDF_EXPECTS(offset >= 0, "offset must be non-negative"); + CUDF_EXPECTS(size >= 0, "size must be non-negative"); +} + byte_range_info create_byte_range_info_max() { return {0, std::numeric_limits::max()}; } std::vector create_byte_range_infos_consecutive(int64_t total_bytes, diff --git a/cpp/src/io/text/multibyte_split.cu b/cpp/src/io/text/multibyte_split.cu index 97729a091fb..e3435a24b18 100644 --- a/cpp/src/io/text/multibyte_split.cu +++ b/cpp/src/io/text/multibyte_split.cu @@ -310,7 +310,7 @@ std::unique_ptr multibyte_split(cudf::io::text::data_chunk_source { CUDF_FUNC_RANGE(); - if (byte_range.empty()) { return make_empty_column(type_id::STRING); } + if (byte_range.is_empty()) { return make_empty_column(type_id::STRING); } auto device_delim = cudf::string_scalar(delimiter, true, stream, mr); From 419fb99fa9ac471ae00ebe7787543b8e9cc154b5 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 13 Aug 2024 08:52:30 -0400 Subject: [PATCH 052/270] Fix all-empty input column for strings split APIs (#16466) Fixes specialized behavior for all empty input column on the strings split APIs. Verifying behavior with Pandas `str.split( pat, expand, regex )` `pat=None -- whitespace` `expand=False -- record APIs` `regex=True -- re APIs` - [x] `split` - [x] `split` - whitespace - [x] `rsplit` - [x] `rsplit` - whitespace - [x] `split_record` - [x] `split_record` - whitespace - [x] `rsplit_record` - [x] `rsplit_record` - whitespace - [x] `split_re` - [x] `rsplit_re` - [x] `split_record_re` - [x] `rsplit_record_re` Closes #16453 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Mark Harris (https://github.com/harrism) - Bradley Dice (https://github.com/bdice) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/16466 --- cpp/src/strings/split/split.cuh | 24 ++++++-------- cpp/src/strings/split/split_re.cu | 4 +++ cpp/tests/strings/split_tests.cpp | 47 ++++++++++++++++++++++++--- python/cudf/cudf/tests/test_string.py | 16 +++++++++ 4 files changed, 73 insertions(+), 18 deletions(-) diff --git a/cpp/src/strings/split/split.cuh b/cpp/src/strings/split/split.cuh index 4d7096c02ca..af70367678e 100644 --- a/cpp/src/strings/split/split.cuh +++ b/cpp/src/strings/split/split.cuh @@ -142,7 +142,7 @@ struct base_split_tokenizer { // max_tokens already included in token counts if (d_tokens.size() == 1) { - d_tokens[0] = string_index_pair{d_str.data(), d_str.size_bytes()}; + d_tokens[0] = string_index_pair{(d_str.empty() ? "" : d_str.data()), d_str.size_bytes()}; return; } @@ -357,24 +357,20 @@ std::pair, rmm::device_uvector> split auto const chars_bytes = get_offset_value(input.offsets(), input.offset() + strings_count, stream) - get_offset_value(input.offsets(), input.offset(), stream); - if (chars_bytes == 0) { - auto offsets = cudf::make_column_from_scalar( - numeric_scalar(0, true, stream), strings_count + 1, stream, mr); - auto tokens = rmm::device_uvector(0, stream); - return std::pair{std::move(offsets), std::move(tokens)}; - } auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), input.offset()); // count the number of delimiters in the entire column rmm::device_scalar d_count(0, stream); - constexpr int64_t block_size = 512; - constexpr size_type bytes_per_thread = 4; - auto const num_blocks = util::div_rounding_up_safe( - util::div_rounding_up_safe(chars_bytes, static_cast(bytes_per_thread)), block_size); - count_delimiters_kernel - <<>>( - tokenizer, d_offsets, chars_bytes, d_count.data()); + if (chars_bytes > 0) { + constexpr int64_t block_size = 512; + constexpr size_type bytes_per_thread = 4; + auto const num_blocks = util::div_rounding_up_safe( + util::div_rounding_up_safe(chars_bytes, static_cast(bytes_per_thread)), block_size); + count_delimiters_kernel + <<>>( + tokenizer, d_offsets, chars_bytes, d_count.data()); + } // Create a vector of every delimiter position in the chars column. // These may include overlapping or otherwise out-of-bounds delimiters which diff --git a/cpp/src/strings/split/split_re.cu b/cpp/src/strings/split/split_re.cu index d72ec1085b5..e0aacf07ef0 100644 --- a/cpp/src/strings/split/split_re.cu +++ b/cpp/src/strings/split/split_re.cu @@ -71,6 +71,10 @@ struct token_reader_fn { auto const token_offset = d_token_offsets[idx]; auto const token_count = d_token_offsets[idx + 1] - token_offset; auto const d_result = d_tokens + token_offset; // store tokens here + if (nchars == 0) { + d_result[0] = string_index_pair{"", 0}; + return; + } int64_t token_idx = 0; auto itr = d_str.begin(); diff --git a/cpp/tests/strings/split_tests.cpp b/cpp/tests/strings/split_tests.cpp index 4c020cb4c29..7ece08b19f2 100644 --- a/cpp/tests/strings/split_tests.cpp +++ b/cpp/tests/strings/split_tests.cpp @@ -307,24 +307,46 @@ TEST_F(StringsSplitTest, SplitRecordWhitespaceWithMaxSplit) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); } -TEST_F(StringsSplitTest, SplitRecordAllEmpty) +TEST_F(StringsSplitTest, SplitAllEmpty) { auto input = cudf::test::strings_column_wrapper({"", "", "", ""}); auto sv = cudf::strings_column_view(input); + auto empty = cudf::string_scalar(""); auto delimiter = cudf::string_scalar("s"); + + auto result = cudf::strings::split(sv, delimiter); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view().column(0), input); + result = cudf::strings::rsplit(sv, delimiter); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view().column(0), input); + + // whitespace hits a special case where nothing matches returns an all-null column + auto expected = cudf::test::strings_column_wrapper({"", "", "", ""}, {0, 0, 0, 0}); + result = cudf::strings::split(sv, empty); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view().column(0), expected); + result = cudf::strings::rsplit(sv, empty); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view().column(0), expected); +} + +TEST_F(StringsSplitTest, SplitRecordAllEmpty) +{ + auto input = cudf::test::strings_column_wrapper({"", "", "", ""}); + auto sv = cudf::strings_column_view(input); auto empty = cudf::string_scalar(""); + auto delimiter = cudf::string_scalar("s"); using LCW = cudf::test::lists_column_wrapper; - LCW expected({LCW{}, LCW{}, LCW{}, LCW{}}); + LCW expected({LCW{""}, LCW{""}, LCW{""}, LCW{""}}); + LCW expected_empty({LCW{}, LCW{}, LCW{}, LCW{}}); + auto result = cudf::strings::split_record(sv, delimiter); CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); result = cudf::strings::split_record(sv, empty); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected_empty); result = cudf::strings::rsplit_record(sv, delimiter); CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); result = cudf::strings::rsplit_record(sv, empty); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected_empty); } TEST_F(StringsSplitTest, MultiByteDelimiters) @@ -575,6 +597,23 @@ TEST_F(StringsSplitTest, SplitRegexWordBoundary) } } +TEST_F(StringsSplitTest, SplitRegexAllEmpty) +{ + auto input = cudf::test::strings_column_wrapper({"", "", "", ""}); + auto sv = cudf::strings_column_view(input); + auto prog = cudf::strings::regex_program::create("[ _]"); + + auto result = cudf::strings::split_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view().column(0), input); + result = cudf::strings::rsplit_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view().column(0), input); + + auto rec_result = cudf::strings::split_record_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view().column(0), input); + rec_result = cudf::strings::rsplit_record_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view().column(0), input); +} + TEST_F(StringsSplitTest, RSplitRecord) { std::vector h_strings{ diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index a2a3e874c91..30880f074c0 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -978,6 +978,22 @@ def test_string_split_re(data, pat, n, expand): assert_eq(expect, got) +@pytest.mark.parametrize("pat", [None, "\\s+"]) +@pytest.mark.parametrize("regex", [False, True]) +@pytest.mark.parametrize("expand", [False, True]) +def test_string_split_all_empty(pat, regex, expand): + ps = pd.Series(["", "", "", ""], dtype="str") + gs = cudf.Series(["", "", "", ""], dtype="str") + + expect = ps.str.split(pat=pat, expand=expand, regex=regex) + got = gs.str.split(pat=pat, expand=expand, regex=regex) + + if isinstance(got, cudf.DataFrame): + assert_eq(expect, got, check_column_type=False) + else: + assert_eq(expect, got) + + @pytest.mark.parametrize( "str_data", [[], ["a", "b", "c", "d", "e"], [None, None, None, None, None]] ) From 3a791cb8a83ca2cf446a910cb94d5a4e3edf2b9f Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 13 Aug 2024 08:56:43 -0400 Subject: [PATCH 053/270] Remove unneeded pair-iterator benchmark (#16511) Removes the pair-iterator benchmark logic. The remaining benchmarks use the null-replacement-iterator which uses the libcudf pair-iterator internally. There is no need for benchmarking this unique iterator pattern that is not used by libcudf. The `cpp/benchmarks/iterator/iterator.cu` failed to compile with gcc 12 because the sum-reduce function cannot resolve adding `thrust::pair` objects together likely due to some recent changes in CCCL. Regardless, adding `thrust::pair` objects is not something we need to benchmark. The existing benchmark benchmarks libcudf's usage of the internal pair-iterator correctly. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Yunsong Wang (https://github.com/PointKernel) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16511 --- cpp/benchmarks/iterator/iterator.cu | 77 ----------------------------- 1 file changed, 77 deletions(-) diff --git a/cpp/benchmarks/iterator/iterator.cu b/cpp/benchmarks/iterator/iterator.cu index ada7a9bd73d..fd0cebb12ea 100644 --- a/cpp/benchmarks/iterator/iterator.cu +++ b/cpp/benchmarks/iterator/iterator.cu @@ -30,7 +30,6 @@ #include #include #include -#include #include #include @@ -161,68 +160,6 @@ void BM_iterator(benchmark::State& state) sizeof(TypeParam)); } -// operator+ defined for pair iterator reduction -template -__device__ thrust::pair operator+(thrust::pair lhs, thrust::pair rhs) -{ - return thrust::pair{lhs.first * lhs.second + rhs.first * rhs.second, - lhs.second + rhs.second}; -} -// ----------------------------------------------------------------------------- -template -void pair_iterator_bench_cub(cudf::column_view& col, - rmm::device_uvector>& result) -{ - thrust::pair init{0, false}; - auto d_col = cudf::column_device_view::create(col); - int num_items = col.size(); - auto begin = d_col->pair_begin(); - reduce_by_cub(result.begin(), begin, num_items, init); -} - -template -void pair_iterator_bench_thrust(cudf::column_view& col, - rmm::device_uvector>& result) -{ - thrust::pair init{0, false}; - auto d_col = cudf::column_device_view::create(col); - auto d_in = d_col->pair_begin(); - auto d_end = d_in + col.size(); - thrust::reduce(thrust::device, d_in, d_end, init, cudf::DeviceSum{}); -} - -template -void BM_pair_iterator(benchmark::State& state) -{ - cudf::size_type const column_size{(cudf::size_type)state.range(0)}; - using T = TypeParam; - auto num_gen = thrust::counting_iterator(0); - auto null_gen = - thrust::make_transform_iterator(num_gen, [](cudf::size_type row) { return row % 2 == 0; }); - - cudf::test::fixed_width_column_wrapper wrap_hasnull_F(num_gen, num_gen + column_size); - cudf::test::fixed_width_column_wrapper wrap_hasnull_T( - num_gen, num_gen + column_size, null_gen); - cudf::column_view hasnull_F = wrap_hasnull_F; - cudf::column_view hasnull_T = wrap_hasnull_T; - - // Initialize dev_result to false - auto dev_result = cudf::detail::make_zeroed_device_uvector_sync>( - 1, cudf::get_default_stream(), rmm::mr::get_current_device_resource()); - for (auto _ : state) { - cuda_event_timer raii(state, true); // flush_l2_cache = true, stream = 0 - if (cub_or_thrust) { - pair_iterator_bench_cub(hasnull_T, - dev_result); // driven by pair iterator with nulls - } else { - pair_iterator_bench_thrust(hasnull_T, - dev_result); // driven by pair iterator with nulls - } - } - state.SetBytesProcessed(static_cast(state.iterations()) * column_size * - sizeof(TypeParam)); -} - #define ITER_BM_BENCHMARK_DEFINE(name, type, cub_or_thrust, raw_or_iterator) \ BENCHMARK_DEFINE_F(Iterator, name)(::benchmark::State & state) \ { \ @@ -238,17 +175,3 @@ ITER_BM_BENCHMARK_DEFINE(double_cub_raw, double, true, true); ITER_BM_BENCHMARK_DEFINE(double_cub_iter, double, true, false); ITER_BM_BENCHMARK_DEFINE(double_thrust_raw, double, false, true); ITER_BM_BENCHMARK_DEFINE(double_thrust_iter, double, false, false); - -#define PAIRITER_BM_BENCHMARK_DEFINE(name, type, cub_or_thrust) \ - BENCHMARK_DEFINE_F(Iterator, name)(::benchmark::State & state) \ - { \ - BM_pair_iterator(state); \ - } \ - BENCHMARK_REGISTER_F(Iterator, name) \ - ->RangeMultiplier(10) \ - ->Range(1000, 10000000) \ - ->UseManualTime() \ - ->Unit(benchmark::kMillisecond); - -PAIRITER_BM_BENCHMARK_DEFINE(double_cub_pair, double, true); -PAIRITER_BM_BENCHMARK_DEFINE(double_thrust_pair, double, false); From 3801f811ab7713e4cb9cc3bb34d282f8a04e71e4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Aug 2024 12:40:40 -0500 Subject: [PATCH 054/270] Remove hardcoded versions from workflows. (#16540) This PR removes hardcoded Python versions from CI workflows. It is a prerequisite for dropping Python 3.9. See https://github.com/rapidsai/build-planning/issues/88. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16540 --- .github/workflows/pandas-tests.yaml | 3 ++- .github/workflows/pr.yaml | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pandas-tests.yaml b/.github/workflows/pandas-tests.yaml index cf0c2b377dd..10c803f7921 100644 --- a/.github/workflows/pandas-tests.yaml +++ b/.github/workflows/pandas-tests.yaml @@ -19,7 +19,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c2e7f64f952..ea8a1762b2c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -187,6 +187,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/cudf_pandas_scripts/run_tests.sh @@ -196,7 +197,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. From 5780c4d8fb5afac2e04988a2ff5531f94c22d3a3 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Tue, 13 Aug 2024 13:46:31 -0700 Subject: [PATCH 055/270] Register `read_parquet` and `read_csv` with dask-expr (#16535) After https://github.com/dask/dask-expr/pull/1114, Dask cuDF must register specific `read_parquet` and `read_csv` functions to be used when query-planning is enabled (the default). **This PR is required for CI to pass with dask>2024.8.0** **NOTE**: It probably doesn't make sense to add specific tests for this change. Once the 2014.7.1 dask pin is removed, all `dask_cudf` tests using `read_parquet` and `read_csv` will fail without this change... Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Mads R. B. Kristensen (https://github.com/madsbk) - Benjamin Zaitlen (https://github.com/quasiben) URL: https://github.com/rapidsai/cudf/pull/16535 --- python/dask_cudf/dask_cudf/backends.py | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 2b1f745fc04..01bab30190a 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -667,6 +667,41 @@ def from_dict( constructor=constructor, ) + @staticmethod + def read_parquet(*args, engine=None, **kwargs): + import dask_expr as dx + + from dask_cudf.io.parquet import CudfEngine + + return _default_backend( + dx.read_parquet, *args, engine=CudfEngine, **kwargs + ) + + @staticmethod + def read_csv( + path, + *args, + header="infer", + dtype_backend=None, + storage_options=None, + **kwargs, + ): + import dask_expr as dx + from fsspec.utils import stringify_path + + if not isinstance(path, str): + path = stringify_path(path) + return dx.new_collection( + dx.io.csv.ReadCSV( + path, + dtype_backend=dtype_backend, + storage_options=storage_options, + kwargs=kwargs, + header=header, + dataframe_backend="cudf", + ) + ) + @staticmethod def read_json(*args, **kwargs): from dask_cudf.io.json import read_json as read_json_impl From cf3fabf7d090dcd983080e3c844002ebb7280e77 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 13 Aug 2024 22:59:47 +0200 Subject: [PATCH 056/270] Ensure comparisons with pyints and integer series always succeed (#16532) When Python integers are compared to a series of integers, the result can always be correctly defined no matter the values of the Python integer. This was always a very mild issue. But with NumPy 2 behavior not upcasting the computation result type based on the value anymore, even things like: ``` cudf.Series([1, 2, 3], dtype="int8") < 1000 ``` would fail. (Similar paths could be taken for other integer scalars, but there would be mostly nice for performance.) N.B. NumPy/pandas also support exact comparisons when mixing e.g. uint64 and int64. This is another rare exception that cudf currently does not support. Closes gh-16282 Authors: - Sebastian Berg (https://github.com/seberg) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16532 --- python/cudf/cudf/core/column/numerical.py | 54 +++++++++++++++++------ python/cudf/cudf/tests/test_binops.py | 41 +++++++++++++++++ 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index b83d7600c82..bbc74ef349e 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -199,16 +199,53 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: np.bool_: np.float32, } + out_dtype = None if op in {"__truediv__", "__rtruediv__"}: # Division with integer types results in a suitable float. if truediv_type := int_float_dtype_mapping.get(self.dtype.type): return self.astype(truediv_type)._binaryop(other, op) + elif op in { + "__lt__", + "__gt__", + "__le__", + "__ge__", + "__eq__", + "__ne__", + }: + out_dtype = "bool" + + # If `other` is a Python integer and it is out-of-bounds + # promotion could fail but we can trivially define the result + # in terms of `notnull` or `NULL_NOT_EQUALS`. + if type(other) is int and self.dtype.kind in "iu": # noqa: E721 + truthiness = None + iinfo = np.iinfo(self.dtype) + if iinfo.min > other: + truthiness = op in {"__ne__", "__gt__", "__ge__"} + elif iinfo.max < other: + truthiness = op in {"__ne__", "__lt__", "__le__"} + + # Compare with minimum value so that the result is true/false + if truthiness is True: + other = iinfo.min + op = "__ge__" + elif truthiness is False: + other = iinfo.min + op = "__lt__" + + elif op in {"NULL_EQUALS", "NULL_NOT_EQUALS"}: + out_dtype = "bool" reflect, op = self._check_reflected_op(op) if (other := self._wrap_binop_normalization(other)) is NotImplemented: return NotImplemented - out_dtype = self.dtype - if other is not None: + + if out_dtype is not None: + pass # out_dtype was already set to bool + if other is None: + # not a binary operator, so no need to promote + out_dtype = self.dtype + elif out_dtype is None: out_dtype = np.result_type(self.dtype, other.dtype) if op in {"__mod__", "__floordiv__"}: tmp = self if reflect else other @@ -225,17 +262,6 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: out_dtype = cudf.dtype("float64") elif is_scalar(tmp) and tmp == 0: out_dtype = cudf.dtype("float64") - if op in { - "__lt__", - "__gt__", - "__le__", - "__ge__", - "__eq__", - "__ne__", - "NULL_EQUALS", - "NULL_NOT_EQUALS", - }: - out_dtype = "bool" if op in {"__and__", "__or__", "__xor__"}: if self.dtype.kind == "f" or other.dtype.kind == "f": @@ -247,7 +273,7 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: if self.dtype.kind == "b" or other.dtype.kind == "b": out_dtype = "bool" - if ( + elif ( op == "__pow__" and self.dtype.kind in "iu" and (is_integer(other) or other.dtype.kind in "iu") diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 503b1a975b4..4256ec872e6 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -290,6 +290,47 @@ def test_series_compare(cmpop, obj_class, dtype): np.testing.assert_equal(result3.to_numpy(), cmpop(arr1, arr2)) +@pytest.mark.parametrize( + "dtype,val", + [("int8", 200), ("int32", 2**32), ("uint8", -128), ("uint64", -1)], +) +@pytest.mark.parametrize( + "op", + [ + operator.eq, + operator.ne, + operator.lt, + operator.le, + operator.gt, + operator.ge, + ], +) +@pytest.mark.parametrize("reverse", [False, True]) +def test_series_compare_integer(dtype, val, op, reverse): + # Tests that these actually work, even though they are out of bound. + force_cast_val = np.array(val).astype(dtype) + sr = Series( + [np.iinfo(dtype).min, np.iinfo(dtype).max, force_cast_val, None], + dtype=dtype, + ) + + if reverse: + _op = op + + def op(x, y): + return _op(y, x) + + # We expect the same result as comparing to a value within range (e.g. 0) + # except that a NULL value evaluates to False + if op(0, val): + expected = Series([True, True, True, None]) + else: + expected = Series([False, False, False, None]) + + res = op(sr, val) + assert_eq(res, expected) + + def _series_compare_nulls_typegen(): return [ *combinations_with_replacement(DATETIME_TYPES, 2), From 1f0d0c93f315f64698ffcc80082926896facf13a Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:07:22 -0400 Subject: [PATCH 057/270] Change cudf::empty_like to not include offsets for empty strings columns (#16529) Fixes `cudf::empty_like` to only create empty child columns for nested types. The empty child columns are needed to store the types for consistency with `cudf::make_empty_column`. Closes #16490 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/16529 --- cpp/src/copying/copy.cpp | 6 ++++++ cpp/tests/copying/pack_tests.cpp | 6 ++++-- cpp/tests/replace/replace_nulls_tests.cpp | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cpp/src/copying/copy.cpp b/cpp/src/copying/copy.cpp index 98ee6aa8f68..bac8dbe5d95 100644 --- a/cpp/src/copying/copy.cpp +++ b/cpp/src/copying/copy.cpp @@ -143,6 +143,12 @@ std::unique_ptr empty_like(column_view const& input) { CUDF_FUNC_RANGE(); + // test_dataframe.py passes an EMPTY column type here; + // this causes is_nested to throw an error since it uses the type-dispatcher + if ((input.type().id() == type_id::EMPTY) || !cudf::is_nested(input.type())) { + return make_empty_column(input.type()); + } + std::vector> children; std::transform(input.child_begin(), input.child_end(), diff --git a/cpp/tests/copying/pack_tests.cpp b/cpp/tests/copying/pack_tests.cpp index ea4408efa6a..8a50e071cb9 100644 --- a/cpp/tests/copying/pack_tests.cpp +++ b/cpp/tests/copying/pack_tests.cpp @@ -573,6 +573,8 @@ TEST_F(PackUnpackTest, SlicedEmpty) cudf::table_view t({a, b, c, d}); - auto sliced = cudf::split(t, {0}); - this->run_test(sliced[0]); + auto sliced = cudf::split(t, {0}); + auto packed = cudf::pack(t); + auto unpacked = cudf::unpack(packed); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(t, unpacked); } diff --git a/cpp/tests/replace/replace_nulls_tests.cpp b/cpp/tests/replace/replace_nulls_tests.cpp index 9603ea44a76..fcee27305f2 100644 --- a/cpp/tests/replace/replace_nulls_tests.cpp +++ b/cpp/tests/replace/replace_nulls_tests.cpp @@ -674,7 +674,7 @@ TEST_F(ReplaceDictionaryTest, ReplaceNullsEmpty) cudf::test::fixed_width_column_wrapper input_empty_w({}); auto input_empty = cudf::dictionary::encode(input_empty_w); auto result = cudf::replace_nulls(input_empty->view(), input_empty->view()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), input_empty->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view(), input_empty->view()); } TEST_F(ReplaceDictionaryTest, ReplaceNullsNoNulls) From c20d6b3a3588c70d985e0d737fed844a9c0c6426 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:07:51 -0400 Subject: [PATCH 058/270] Remove unneeded output size parameter from internal count_matches utility (#16531) Removes `output_size` parameter from `cudf::strings::detail::count_matches` utility since the output size should equal the input size from the first parameter. This also removes an unnecessary `assert()` call. The parameter became unnecessary as part of the large strings work. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Shruti Shivakumar (https://github.com/shrshi) URL: https://github.com/rapidsai/cudf/pull/16531 --- cpp/src/strings/contains.cu | 2 +- cpp/src/strings/count_matches.cu | 9 +++------ cpp/src/strings/count_matches.hpp | 2 -- cpp/src/strings/extract/extract_all.cu | 2 +- cpp/src/strings/search/findall.cu | 2 +- cpp/src/strings/split/split_re.cu | 6 +++--- 6 files changed, 9 insertions(+), 14 deletions(-) diff --git a/cpp/src/strings/contains.cu b/cpp/src/strings/contains.cu index 718ac41e36c..79d241205df 100644 --- a/cpp/src/strings/contains.cu +++ b/cpp/src/strings/contains.cu @@ -112,7 +112,7 @@ std::unique_ptr count_re(strings_column_view const& input, auto const d_strings = column_device_view::create(input.parent(), stream); - auto result = count_matches(*d_strings, *d_prog, input.size(), stream, mr); + auto result = count_matches(*d_strings, *d_prog, stream, mr); if (input.has_nulls()) { result->set_null_mask(cudf::detail::copy_bitmask(input.parent(), stream, mr), input.null_count()); diff --git a/cpp/src/strings/count_matches.cu b/cpp/src/strings/count_matches.cu index e8672ea5335..4ad3a75baf7 100644 --- a/cpp/src/strings/count_matches.cu +++ b/cpp/src/strings/count_matches.cu @@ -60,18 +60,15 @@ struct count_fn { std::unique_ptr count_matches(column_device_view const& d_strings, reprog_device& d_prog, - size_type output_size, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - assert(output_size >= d_strings.size() and "Unexpected output size"); - auto results = make_numeric_column( - data_type{type_to_id()}, output_size, mask_state::UNALLOCATED, stream, mr); + data_type{type_to_id()}, d_strings.size(), mask_state::UNALLOCATED, stream, mr); - if (d_strings.size() == 0) return results; + if (d_strings.size() == 0) { return results; } - auto d_results = results->mutable_view().data(); + auto d_results = results->mutable_view().data(); launch_transform_kernel(count_fn{d_strings}, d_prog, d_results, d_strings.size(), stream); diff --git a/cpp/src/strings/count_matches.hpp b/cpp/src/strings/count_matches.hpp index 4a5efac37fd..eab9863b975 100644 --- a/cpp/src/strings/count_matches.hpp +++ b/cpp/src/strings/count_matches.hpp @@ -37,14 +37,12 @@ class reprog_device; * * @param d_strings Device view of the input strings column. * @param d_prog Regex instance to evaluate on each string. - * @param output_size Number of rows for the output column. * @param stream CUDA stream used for device memory operations and kernel launches. * @param mr Device memory resource used to allocate the returned column's device memory. * @return Integer column of match counts */ std::unique_ptr count_matches(column_device_view const& d_strings, reprog_device& d_prog, - size_type output_size, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); diff --git a/cpp/src/strings/extract/extract_all.cu b/cpp/src/strings/extract/extract_all.cu index 27691068d5a..897eba58833 100644 --- a/cpp/src/strings/extract/extract_all.cu +++ b/cpp/src/strings/extract/extract_all.cu @@ -119,7 +119,7 @@ std::unique_ptr extract_all_record(strings_column_view const& input, // Get the match counts for each string. // This column will become the output lists child offsets column. - auto counts = count_matches(*d_strings, *d_prog, strings_count, stream, mr); + auto counts = count_matches(*d_strings, *d_prog, stream, mr); auto d_counts = counts->mutable_view().data(); // Compute null output rows diff --git a/cpp/src/strings/search/findall.cu b/cpp/src/strings/search/findall.cu index 0d0962258cf..2f7e7352458 100644 --- a/cpp/src/strings/search/findall.cu +++ b/cpp/src/strings/search/findall.cu @@ -104,7 +104,7 @@ std::unique_ptr findall(strings_column_view const& input, auto d_prog = regex_device_builder::create_prog_device(prog, stream); // Create lists offsets column - auto const sizes = count_matches(*d_strings, *d_prog, strings_count, stream, mr); + auto const sizes = count_matches(*d_strings, *d_prog, stream, mr); auto [offsets, total_matches] = cudf::detail::make_offsets_child_column( sizes->view().begin(), sizes->view().end(), stream, mr); auto const d_offsets = offsets->view().data(); diff --git a/cpp/src/strings/split/split_re.cu b/cpp/src/strings/split/split_re.cu index e0aacf07ef0..d273c93ec12 100644 --- a/cpp/src/strings/split/split_re.cu +++ b/cpp/src/strings/split/split_re.cu @@ -210,8 +210,8 @@ std::unique_ptr
split_re(strings_column_view const& input, auto d_strings = column_device_view::create(input.parent(), stream); // count the number of delimiters matched in each string - auto const counts = count_matches( - *d_strings, *d_prog, strings_count, stream, rmm::mr::get_current_device_resource()); + auto const counts = + count_matches(*d_strings, *d_prog, stream, rmm::mr::get_current_device_resource()); // get the split tokens from the input column; this also converts the counts into offsets auto [tokens, offsets] = @@ -275,7 +275,7 @@ std::unique_ptr split_record_re(strings_column_view const& input, auto d_strings = column_device_view::create(input.parent(), stream); // count the number of delimiters matched in each string - auto counts = count_matches(*d_strings, *d_prog, strings_count, stream, mr); + auto counts = count_matches(*d_strings, *d_prog, stream, mr); // get the split tokens from the input column; this also converts the counts into offsets auto [tokens, offsets] = From bf3372b1aa02939db32b2df62ab816a0eb9abdde Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 14 Aug 2024 12:06:29 -0500 Subject: [PATCH 059/270] Switch python version to `3.10` in `cudf.pandas` pandas test scripts (#16559) python 3.9 support was recently dropped in rapids, hence changing the python version to 3.10 Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16559 --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index 6cf70a2347f..5dbb4ba991c 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -12,7 +12,7 @@ RAPIDS_FULL_VERSION=$(<./VERSION) rapids-logger "Github job name: ${GH_JOB_NAME}" rapids-logger "Rapids version: ${RAPIDS_FULL_VERSION}" -PY_VER="39" +PY_VER="310" MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION}-results.json From d684ae0e80d179d4d711c00278d00b5f66625303 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:36:51 -1000 Subject: [PATCH 060/270] Raise NotImplementedError for Series.rename that's not a scalar (#16525) xref https://github.com/rapidsai/cudf/issues/16507 Raising a `NotImplementedError` gives a chance for this work in `cudf.pandas` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16525 --- python/cudf/cudf/core/series.py | 4 ++++ python/cudf/cudf/tests/test_series.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 53675d339ac..822b966364f 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3589,6 +3589,10 @@ def rename( raise NotImplementedError("level is currently not supported.") if errors != "ignore": raise NotImplementedError("errors is currently not supported.") + if not is_scalar(index): + raise NotImplementedError( + ".rename does not currently support relabeling the index." + ) out_data = self._data.copy(deep=copy) return Series._from_data(out_data, self.index, name=index) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 6a1887afb1f..c7aea563535 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2289,6 +2289,13 @@ def test_series_rename(initial_name, name): assert_eq(actual, expected) +@pytest.mark.parametrize("index", [lambda x: x * 2, {1: 2}]) +def test_rename_index_not_supported(index): + ser = cudf.Series(range(2)) + with pytest.raises(NotImplementedError): + ser.rename(index=index) + + @pytest.mark.parametrize( "data", [ From 0253e976ede25d954c607663da61b445e213523f Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:27:52 -0400 Subject: [PATCH 061/270] [FEA] Support named aggregations in `df.groupby().agg()` (#16528) Closes #15967 Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16528 --- python/cudf/cudf/core/column_accessor.py | 7 +--- python/cudf/cudf/core/groupby/groupby.py | 41 ++++++++++++------- python/cudf/cudf/tests/groupby/test_agg.py | 30 ++++++++++++++ .../cudf/cudf/tests/test_column_accessor.py | 4 ++ python/cudf/cudf/tests/test_dataframe.py | 1 - 5 files changed, 62 insertions(+), 21 deletions(-) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 83596704672..48bc84070b1 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -610,7 +610,7 @@ def _pad_key(self, key: Any, pad_value="") -> Any: return key + (pad_value,) * (self.nlevels - len(key)) def rename_levels( - self, mapper: Mapping[Any, Any] | Callable, level: int | None + self, mapper: Mapping[Any, Any] | Callable, level: int | None = None ) -> ColumnAccessor: """ Rename the specified levels of the given ColumnAccessor @@ -653,10 +653,7 @@ def rename_column(x): return x if level is None: - raise NotImplementedError( - "Renaming columns with a MultiIndex and level=None is" - "not supported" - ) + level = 0 new_col_names = (rename_column(k) for k in self.keys()) else: diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 92c4b73ceaa..9b71ea57f1f 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -548,7 +548,7 @@ def _groupby(self): ) @_performance_tracking - def agg(self, func, *args, engine=None, engine_kwargs=None, **kwargs): + def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): """ Apply aggregation(s) to the groups. @@ -648,11 +648,10 @@ def agg(self, func, *args, engine=None, engine_kwargs=None, **kwargs): raise NotImplementedError( "Passing args to func is currently not supported." ) - if kwargs: - raise NotImplementedError( - "Passing kwargs to func is currently not supported." - ) - column_names, columns, normalized_aggs = self._normalize_aggs(func) + + column_names, columns, normalized_aggs = self._normalize_aggs( + func, **kwargs + ) orig_dtypes = tuple(c.dtype for c in columns) # Note: When there are no key columns, the below produces @@ -1266,11 +1265,11 @@ def _grouped(self, *, include_groups: bool = True): return (group_names, offsets, grouped_keys, grouped_values) def _normalize_aggs( - self, aggs: MultiColumnAggType + self, aggs: MultiColumnAggType, **kwargs ) -> tuple[Iterable[Any], tuple[ColumnBase, ...], list[list[AggType]]]: """ Normalize aggs to a list of list of aggregations, where `out[i]` - is a list of aggregations for column `self.obj[i]`. We support three + is a list of aggregations for column `self.obj[i]`. We support four different form of `aggs` input here: - A single agg, such as "sum". This agg is applied to all value columns. @@ -1279,18 +1278,30 @@ def _normalize_aggs( - A mapping of column name to aggs, such as {"a": ["sum"], "b": ["mean"]}, the aggs are applied to specified column. + - Pairs of column name and agg tuples passed as kwargs + eg. col1=("a", "sum"), col2=("b", "prod"). The output column names are + the keys. The aggs are applied to the corresponding column in the tuple. Each agg can be string or lambda functions. """ aggs_per_column: Iterable[AggType | Iterable[AggType]] - if isinstance(aggs, dict): - column_names, aggs_per_column = aggs.keys(), aggs.values() - columns = tuple(self.obj._data[col] for col in column_names) + # TODO: Remove isinstance condition when the legacy dask_cudf API is removed. + # See https://github.com/rapidsai/cudf/pull/16528#discussion_r1715482302 for information. + if aggs or isinstance(aggs, dict): + if isinstance(aggs, dict): + column_names, aggs_per_column = aggs.keys(), aggs.values() + columns = tuple(self.obj._data[col] for col in column_names) + else: + values = self.grouping.values + column_names = values._column_names + columns = values._columns + aggs_per_column = (aggs,) * len(columns) + elif not aggs and kwargs: + column_names, aggs_per_column = kwargs.keys(), kwargs.values() + columns = tuple(self.obj._data[x[0]] for x in kwargs.values()) + aggs_per_column = tuple(x[1] for x in kwargs.values()) else: - values = self.grouping.values - column_names = values._column_names - columns = values._columns - aggs_per_column = (aggs,) * len(columns) + raise TypeError("Must provide at least one aggregation function.") # is_list_like performs type narrowing but type-checkers don't # know it. One could add a TypeGuard annotation to diff --git a/python/cudf/cudf/tests/groupby/test_agg.py b/python/cudf/cudf/tests/groupby/test_agg.py index f8847f02d5a..99e7523031b 100644 --- a/python/cudf/cudf/tests/groupby/test_agg.py +++ b/python/cudf/cudf/tests/groupby/test_agg.py @@ -3,6 +3,7 @@ import pytest import cudf +from cudf.testing import assert_eq @pytest.mark.parametrize( @@ -26,3 +27,32 @@ def test_series_agg(attr): pd_agg = getattr(pdf.groupby(["a"])["a"], attr)("count") assert agg.ndim == pd_agg.ndim + + +@pytest.mark.parametrize("func", ["sum", "prod", "mean", "count"]) +@pytest.mark.parametrize("attr", ["agg", "aggregate"]) +def test_dataframe_agg(attr, func): + df = cudf.DataFrame({"a": [1, 2, 1, 2], "b": [0, 0, 0, 0]}) + pdf = df.to_pandas() + + agg = getattr(df.groupby("a"), attr)(func) + pd_agg = getattr(pdf.groupby(["a"]), attr)(func) + + assert_eq(agg, pd_agg) + + agg = getattr(df.groupby("a"), attr)({"b": func}) + pd_agg = getattr(pdf.groupby(["a"]), attr)({"b": func}) + + assert_eq(agg, pd_agg) + + agg = getattr(df.groupby("a"), attr)([func]) + pd_agg = getattr(pdf.groupby(["a"]), attr)([func]) + + assert_eq(agg, pd_agg) + + agg = getattr(df.groupby("a"), attr)(foo=("b", func), bar=("a", func)) + pd_agg = getattr(pdf.groupby(["a"]), attr)( + foo=("b", func), bar=("a", func) + ) + + assert_eq(agg, pd_agg) diff --git a/python/cudf/cudf/tests/test_column_accessor.py b/python/cudf/cudf/tests/test_column_accessor.py index e84e1433c10..2d7bc809d4d 100644 --- a/python/cudf/cudf/tests/test_column_accessor.py +++ b/python/cudf/cudf/tests/test_column_accessor.py @@ -362,6 +362,10 @@ def test_replace_level_values_MultiColumn(): got = ca.rename_levels(mapper={"a": "f"}, level=0) check_ca_equal(expect, got) + # passing without level kwarg assumes level=0 + got = ca.rename_levels(mapper={"a": "f"}) + check_ca_equal(expect, got) + def test_clear_nrows_empty_before(): ca = ColumnAccessor({}) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 2c59253d500..89eb5a12c71 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -9409,7 +9409,6 @@ def test_rename_for_level_RangeIndex_dataframe(): assert_eq(expect, got) -@pytest_xfail(reason="level=None not implemented yet") def test_rename_for_level_is_None_MC(): gdf = cudf.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) gdf.columns = pd.MultiIndex.from_tuples([("a", 1), ("a", 2), ("b", 1)]) From 19846b6c0ac40fc91ad28573af04ac7403754acb Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:15:03 -1000 Subject: [PATCH 062/270] Disallow cudf.Index accepting column in favor of ._from_column (#16549) Similar to https://github.com/rapidsai/cudf/pull/16454, this PR disallows the public `cudf.Index` accepting a private `ColumnBase` object in favor of `_from_column` (which was added in the linked PR) Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16549 --- python/cudf/cudf/_lib/parquet.pyx | 4 +- python/cudf/cudf/_lib/utils.pyx | 6 +- python/cudf/cudf/api/types.py | 2 +- python/cudf/cudf/core/_base_index.py | 2 +- python/cudf/cudf/core/algorithms.py | 6 +- python/cudf/cudf/core/column/categorical.py | 8 +- python/cudf/cudf/core/column/datetime.py | 10 +- python/cudf/cudf/core/column/methods.py | 6 +- python/cudf/cudf/core/column/string.py | 2 +- python/cudf/cudf/core/cut.py | 2 +- python/cudf/cudf/core/dataframe.py | 8 +- python/cudf/cudf/core/dtypes.py | 14 +- python/cudf/cudf/core/groupby/groupby.py | 9 +- python/cudf/cudf/core/index.py | 238 ++++++++++++-------- python/cudf/cudf/core/indexed_frame.py | 24 +- python/cudf/cudf/core/multiindex.py | 7 +- python/cudf/cudf/core/resample.py | 4 +- python/cudf/cudf/core/series.py | 4 +- python/cudf/cudf/core/tools/datetimes.py | 16 +- python/cudf/cudf/testing/testing.py | 8 +- python/cudf/cudf/tests/test_multiindex.py | 4 +- python/cudf/cudf/tests/test_string.py | 2 +- 22 files changed, 232 insertions(+), 154 deletions(-) diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index 4a4b13b0b31..0fffb6ade58 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -222,7 +222,7 @@ cdef object _process_metadata(object df, if len(filtered_idx) > 0: idx = cudf.concat(filtered_idx) else: - idx = cudf.Index(cudf.core.column.column_empty(0)) + idx = cudf.Index._from_column(cudf.core.column.column_empty(0)) else: start = range_index_meta["start"] + skip_rows stop = range_index_meta["stop"] @@ -240,7 +240,7 @@ cdef object _process_metadata(object df, index_data = df[index_col] actual_index_names = list(index_col_names.values()) if len(index_data._data) == 1: - idx = cudf.Index( + idx = cudf.Index._from_column( index_data._data.columns[0], name=actual_index_names[0] ) diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index f136cd997a7..267432a0182 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -93,12 +93,12 @@ cpdef generate_pandas_metadata(table, index): materialize_index = False if index is not False: for level, name in enumerate(table._index.names): - if isinstance(table._index, cudf.core.multiindex.MultiIndex): + if isinstance(table._index, cudf.MultiIndex): idx = table.index.get_level_values(level) else: idx = table.index - if isinstance(idx, cudf.core.index.RangeIndex): + if isinstance(idx, cudf.RangeIndex): if index is None: descr = { "kind": "range", @@ -110,7 +110,7 @@ cpdef generate_pandas_metadata(table, index): else: materialize_index = True # When `index=True`, RangeIndex needs to be materialized. - materialized_idx = cudf.Index(idx._values, name=idx.name) + materialized_idx = idx._as_int_index() descr = _index_level_name( index_name=materialized_idx.name, level=level, diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 294ae2fd985..9c436dfad18 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -249,7 +249,7 @@ def _union_categoricals( new_categories=sorted_categories ) - return cudf.Index(result_col) + return cudf.CategoricalIndex._from_column(result_col) def is_bool_dtype(arr_or_dtype): diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index c91514202c5..d13351c49dd 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -1979,7 +1979,7 @@ def from_pandas(cls, index: pd.Index, nan_as_null=no_default): name=index.name, ) else: - return cudf.Index( + return cudf.Index._from_column( column.as_column(index, nan_as_null=nan_as_null), name=index.name, ) diff --git a/python/cudf/cudf/core/algorithms.py b/python/cudf/cudf/core/algorithms.py index 6c69fbd2637..e27d6ec8d3e 100644 --- a/python/cudf/cudf/core/algorithms.py +++ b/python/cudf/cudf/core/algorithms.py @@ -8,7 +8,7 @@ import numpy as np from cudf.core.column import as_column -from cudf.core.index import RangeIndex, ensure_index +from cudf.core.index import Index, RangeIndex from cudf.core.scalar import Scalar from cudf.options import get_option from cudf.utils.dtypes import can_convert_to_column @@ -112,7 +112,9 @@ def factorize(values, sort=False, use_na_sentinel=True, size_hint=None): dtype="int64" if get_option("mode.pandas_compatible") else None, ).values - return labels, cats.values if return_cupy_array else ensure_index(cats) + return labels, cats.values if return_cupy_array else Index._from_column( + cats + ) def _interpolation(column: ColumnBase, index: BaseIndex) -> ColumnBase: diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 6fa69eb9cc1..d25983842f9 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -601,11 +601,13 @@ def __setitem__(self, key, value): to_add_categories = 0 else: if cudf.api.types.is_scalar(value): - arr = [value] + arr = column.as_column(value, length=1, nan_as_null=False) else: - arr = value + arr = column.as_column(value, nan_as_null=False) to_add_categories = len( - cudf.Index(arr, nan_as_null=False).difference(self.categories) + cudf.Index._from_column(arr).difference( + cudf.Index._from_column(self.categories) + ) ) if to_add_categories > 0: diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index ce67ce81e6b..1dbc94384d3 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -250,6 +250,10 @@ def __contains__(self, item: ScalarLike) -> bool: def time_unit(self) -> str: return np.datetime_data(self.dtype)[0] + @property + def quarter(self) -> ColumnBase: + return libcudf.datetime.extract_quarter(self) + @property def year(self) -> ColumnBase: return self.get_dt_field("year") @@ -308,7 +312,7 @@ def is_quarter_start(self) -> ColumnBase: @property def is_year_end(self) -> ColumnBase: day_of_year = self.day_of_year - leap_dates = libcudf.datetime.is_leap_year(self) + leap_dates = self.is_leap_year leap = day_of_year == cudf.Scalar(366) non_leap = day_of_year == cudf.Scalar(365) @@ -316,6 +320,10 @@ def is_year_end(self) -> ColumnBase: False ) + @property + def is_leap_year(self) -> ColumnBase: + return libcudf.datetime.is_leap_year(self) + @property def is_year_start(self) -> ColumnBase: return (self.day_of_year == 1).fillna(False) diff --git a/python/cudf/cudf/core/column/methods.py b/python/cudf/cudf/core/column/methods.py index 8c46d238057..05a0ab2e09a 100644 --- a/python/cudf/cudf/core/column/methods.py +++ b/python/cudf/cudf/core/column/methods.py @@ -65,8 +65,8 @@ def _return_or_inplace( """ if inplace: self._parent._mimic_inplace( - self._parent.__class__._from_data( - {self._parent.name: new_col} + type(self._parent)._from_column( + new_col, name=self._parent.name ), inplace=True, ) @@ -92,6 +92,6 @@ def _return_or_inplace( index=self._parent.index if retain_index else None, ) elif isinstance(self._parent, cudf.BaseIndex): - return cudf.Index(new_col, name=self._parent.name) + return cudf.Index._from_column(new_col, name=self._parent.name) else: return self._parent._mimic_inplace(new_col, inplace=False) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 1a4b558749d..a710a9f46c2 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -4693,7 +4693,7 @@ def character_tokenize(self) -> SeriesOrIndex: result_col, name=self._parent.name, index=index ) elif isinstance(self._parent, cudf.BaseIndex): - return cudf.Index(result_col, name=self._parent.name) + return cudf.Index._from_column(result_col, name=self._parent.name) else: return result_col diff --git a/python/cudf/cudf/core/cut.py b/python/cudf/cudf/core/cut.py index 197f46ee9fe..a4ceea266b4 100644 --- a/python/cudf/cudf/core/cut.py +++ b/python/cudf/cudf/core/cut.py @@ -292,7 +292,7 @@ def cut( ) # we return a categorical index, as we don't have a Categorical method - categorical_index = cudf.CategoricalIndex._from_data({None: col}) + categorical_index = cudf.CategoricalIndex._from_column(col) if isinstance(orig_x, (pd.Series, cudf.Series)): # if we have a series input we return a series output diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index a53c7bcc63c..3033abd53f5 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -326,7 +326,7 @@ def _getitem_tuple_arg(self, arg): range(len(tmp_arg[0])) ) }, - index=cudf.Index(tmp_arg[0]), + index=cudf.Index._from_column(tmp_arg[0]), ) columns_df[cantor_name] = column.as_column( range(len(columns_df)) @@ -1758,7 +1758,7 @@ def _concat( for cols in columns: table_index = None if 1 == first_data_column_position: - table_index = cudf.Index(cols[0]) + table_index = cudf.Index._from_column(cols[0]) elif first_data_column_position > 1: table_index = cudf.MultiIndex._from_data( data=dict( @@ -1810,7 +1810,7 @@ def _concat( if not isinstance(out.index, MultiIndex) and isinstance( out.index.dtype, cudf.CategoricalDtype ): - out = out.set_index(cudf.Index(out.index._values)) + out = out.set_index(out.index) for name, col in out._data.items(): out._data[name] = col._with_type_metadata( tables[0]._data[name].dtype @@ -3007,7 +3007,7 @@ def set_index( and not isinstance(keys[0], (cudf.MultiIndex, pd.MultiIndex)) ): # Don't turn single level MultiIndex into an Index - idx = cudf.Index(data_to_add[0], name=names[0]) + idx = cudf.Index._from_column(data_to_add[0], name=names[0]) else: idx = MultiIndex._from_data(dict(enumerate(data_to_add))) idx.names = names diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 27afec18b4e..6d532e01cba 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -182,7 +182,7 @@ def __init__(self, categories=None, ordered: bool = False) -> None: self._ordered = ordered @property - def categories(self) -> "cudf.core.index.Index": + def categories(self) -> cudf.Index: """ An ``Index`` containing the unique categories allowed. @@ -194,10 +194,12 @@ def categories(self) -> "cudf.core.index.Index": Index(['b', 'a'], dtype='object') """ if self._categories is None: - return cudf.Index( - cudf.core.column.column_empty(0, dtype="object", masked=False) + col = cudf.core.column.column_empty( + 0, dtype="object", masked=False ) - return cudf.Index(self._categories, copy=False) + else: + col = self._categories + return cudf.Index._from_column(col) @property def type(self): @@ -259,7 +261,9 @@ def to_pandas(self) -> pd.CategoricalDtype: categories = self._categories.to_pandas() return pd.CategoricalDtype(categories=categories, ordered=self.ordered) - def _init_categories(self, categories: Any): + def _init_categories( + self, categories: Any + ) -> cudf.core.column.ColumnBase | None: if categories is None: return categories if len(categories) == 0 and not isinstance( diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 9b71ea57f1f..4f283d41b17 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -403,8 +403,7 @@ def indices(self) -> dict[ScalarLike, cp.ndarray]: if len(group_keys) > 1: index = cudf.MultiIndex.from_arrays(group_keys) else: - (group_keys,) = group_keys - index = cudf.Index(group_keys) + index = cudf.Index._from_column(group_keys[0]) return dict( zip(index.to_pandas(), cp.split(indices.values, offsets[1:-1])) ) @@ -2583,7 +2582,7 @@ def _mimic_pandas_order( # corresponding output rows in pandas, to do that here # expand the result by reindexing. ri = cudf.RangeIndex(0, len(self.obj)) - result.index = cudf.Index(ordering) + result.index = cudf.Index._from_column(ordering) # This reorders and expands result = result.reindex(ri) else: @@ -3154,7 +3153,9 @@ def keys(self): dict(zip(range(nkeys), self._key_columns)) )._set_names(self.names) else: - return cudf.Index(self._key_columns[0], name=self.names[0]) + return cudf.Index._from_column( + self._key_columns[0], name=self.names[0] + ) @property def values(self) -> cudf.core.frame.Frame: diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 3eab27bd165..c55f86d48e1 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -18,7 +18,6 @@ import cudf from cudf import _lib as libcudf -from cudf._lib.datetime import extract_quarter, is_leap_year from cudf._lib.filling import sequence from cudf._lib.search import search_sorted from cudf._lib.types import size_type_dtype @@ -819,22 +818,23 @@ def sort_values( @_performance_tracking def _gather(self, gather_map, nullify=False, check_bounds=True): gather_map = cudf.core.column.as_column(gather_map) - return cudf.Index._from_data( - {self.name: self._values.take(gather_map, nullify, check_bounds)} + return cudf.Index._from_column( + self._column.take(gather_map, nullify, check_bounds), + name=self.name, ) @_performance_tracking def _apply_boolean_mask(self, boolean_mask): - return cudf.Index._from_data( - {self.name: self._values.apply_boolean_mask(boolean_mask)} + return cudf.Index._from_column( + self._column.apply_boolean_mask(boolean_mask), name=self.name ) def repeat(self, repeats, axis=None): return self._as_int_index().repeat(repeats, axis) def _split(self, splits): - return cudf.Index._from_data( - {self.name: self._as_int_index()._split(splits)} + return cudf.Index._from_column( + self._as_int_index()._split(splits), name=self.name ) def _binaryop(self, other, op: str): @@ -1087,10 +1087,13 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): def _from_column( cls, column: ColumnBase, *, name: Hashable = None ) -> Self: - ca = cudf.core.column_accessor.ColumnAccessor( - {name: column}, verify=False - ) - return _index_from_data(ca) + if cls is Index: + ca = cudf.core.column_accessor.ColumnAccessor( + {name: column}, verify=False + ) + return _index_from_data(ca) + else: + return super()._from_column(column, name=name) @classmethod @_performance_tracking @@ -1223,8 +1226,8 @@ def _concat(cls, objs): if all(isinstance(obj, RangeIndex) for obj in non_empties): result = _concat_range_index(non_empties) else: - data = concat_columns([o._values for o in non_empties]) - result = Index(data) + data = concat_columns([o._column for o in non_empties]) + result = Index._from_column(data) names = {obj.name for obj in objs} if len(names) == 1: @@ -1491,7 +1494,7 @@ def __repr__(self): def __getitem__(self, index): res = self._get_elements_from_column(index) if isinstance(res, ColumnBase): - res = Index(res, name=self.name) + res = Index._from_column(res, name=self.name) return res @property # type: ignore @@ -1610,8 +1613,8 @@ def _clean_nulls_from_index(self): if isinstance(self, (DatetimeIndex, TimedeltaIndex)) else str(cudf.NA) ) - return cudf.Index( - self._values.astype("str").fillna(fill_value), + return cudf.Index._from_column( + self._column.astype("str").fillna(fill_value), name=self.name, ) @@ -1866,6 +1869,17 @@ def _from_data( result._freq = _validate_freq(freq) return result + @classmethod + @_performance_tracking + def _from_column( + cls, column: ColumnBase, *, name: Hashable = None, freq: Any = None + ) -> Self: + if column.dtype.kind != "M": + raise ValueError("column must have a datetime type.") + result = super()._from_column(column, name=name) + result._freq = _validate_freq(freq) + return result + def __getitem__(self, index): value = super().__getitem__(index) if cudf.get_option("mode.pandas_compatible") and isinstance( @@ -1923,8 +1937,8 @@ def strftime(self, date_format: str) -> Index: date_format : str Date format string (e.g. "%Y-%m-%d"). """ - return Index._from_data( - {self.name: self._column.strftime(date_format)} + return Index._from_column( + self._column.strftime(date_format), name=self.name ) @property @@ -1989,7 +2003,9 @@ def to_pydatetime(self) -> np.ndarray: return self.to_pandas().to_pydatetime() def to_julian_date(self) -> Index: - return Index._from_data({self.name: self._column.to_julian_date()}) + return Index._from_column( + self._column.to_julian_date(), name=self.name + ) def to_period(self, freq) -> pd.PeriodIndex: return self.to_pandas().to_period(freq=freq) @@ -2000,7 +2016,9 @@ def normalize(self) -> Self: Currently not implemented. """ - return type(self)._from_data({self.name: self._column.normalize()}) + return type(self)._from_column( + self._column.normalize(), name=self.name + ) @property def time(self) -> np.ndarray: @@ -2084,7 +2102,7 @@ def days_in_month(self) -> Index: """ Get the total number of days in the month that the date falls on. """ - return Index._from_data({self.name: self._column.days_in_month}) + return Index._from_column(self._column.days_in_month, name=self.name) daysinmonth = days_in_month @@ -2093,7 +2111,7 @@ def day_of_week(self) -> Index: """ Get the day of week that the date falls on. """ - return Index._from_data({self.name: self._column.day_of_week}) + return Index._from_column(self._column.day_of_week, name=self.name) @property # type: ignore @_performance_tracking @@ -2234,15 +2252,15 @@ def microsecond(self): >>> datetime_index.microsecond Index([0, 1, 2], dtype='int32') """ # noqa: E501 - return Index( + return Index._from_column( ( # Need to manually promote column to int32 because # pandas-matching binop behaviour requires that this # __mul__ returns an int16 column. - self._values.get_dt_field("millisecond").astype("int32") + self._column.get_dt_field("millisecond").astype("int32") * cudf.Scalar(1000, dtype="int32") ) - + self._values.get_dt_field("microsecond"), + + self._column.get_dt_field("microsecond"), name=self.name, ) @@ -2374,7 +2392,7 @@ def is_leap_year(self) -> cupy.ndarray: ndarray Booleans indicating if dates belong to a leap year. """ - res = is_leap_year(self._values).fillna(False) + res = self._column.is_leap_year.fillna(False) return cupy.asarray(res) @property # type: ignore @@ -2400,8 +2418,7 @@ def quarter(self): >>> gIndex.quarter Index([2, 4], dtype='int8') """ - res = extract_quarter(self._values) - return Index(res, dtype="int8") + return Index._from_column(self._column.quarter.astype("int8")) @_performance_tracking def day_name(self, locale: str | None = None) -> Index: @@ -2423,7 +2440,7 @@ def day_name(self, locale: str | None = None) -> Index: dtype='object') """ day_names = self._column.get_day_names(locale) - return Index._from_data({self.name: day_names}) + return Index._from_column(day_names, name=self.name) @_performance_tracking def month_name(self, locale: str | None = None) -> Index: @@ -2442,7 +2459,7 @@ def month_name(self, locale: str | None = None) -> Index: Index(['December', 'January', 'January', 'January', 'January', 'February'], dtype='object') """ month_names = self._column.get_month_names(locale) - return Index._from_data({self.name: month_names}) + return Index._from_column(month_names, name=self.name) @_performance_tracking def isocalendar(self) -> cudf.DataFrame: @@ -2481,14 +2498,14 @@ def to_pandas( @_performance_tracking def _get_dt_field(self, field: str) -> Index: """Return an Index of a numerical component of the DatetimeIndex.""" - out_column = self._values.get_dt_field(field) + out_column = self._column.get_dt_field(field) out_column = NumericalColumn( data=out_column.base_data, dtype=out_column.dtype, mask=out_column.base_mask, offset=out_column.offset, ) - return Index(out_column, name=self.name) + return Index._from_column(out_column, name=self.name) def _is_boolean(self): return False @@ -2522,9 +2539,7 @@ def ceil(self, freq): >>> gIndex.ceil("T") DatetimeIndex(['2020-05-31 08:06:00', '1999-12-31 18:41:00'], dtype='datetime64[ns]') """ # noqa: E501 - out_column = self._values.ceil(freq) - - return self.__class__._from_data({self.name: out_column}) + return type(self)._from_column(self._column.ceil(freq), name=self.name) @_performance_tracking def floor(self, freq): @@ -2555,9 +2570,9 @@ def floor(self, freq): >>> gIndex.floor("T") DatetimeIndex(['2020-05-31 08:59:00', '1999-12-31 18:44:00'], dtype='datetime64[ns]') """ # noqa: E501 - out_column = self._values.floor(freq) - - return self.__class__._from_data({self.name: out_column}) + return type(self)._from_column( + self._column.floor(freq), name=self.name + ) @_performance_tracking def round(self, freq): @@ -2595,9 +2610,9 @@ def round(self, freq): >>> dt_idx.round('T') DatetimeIndex(['2001-01-01 00:05:00', '2001-01-01 00:05:00', '2001-01-01 00:05:00'], dtype='datetime64[ns]') """ # noqa: E501 - out_column = self._values.round(freq) - - return self.__class__._from_data({self.name: out_column}) + return type(self)._from_column( + self._column.round(freq), name=self.name + ) def tz_localize( self, @@ -2647,8 +2662,8 @@ def tz_localize( to 'NaT'. """ # noqa: E501 result_col = self._column.tz_localize(tz, ambiguous, nonexistent) - return DatetimeIndex._from_data( - {self.name: result_col}, freq=self._freq + return DatetimeIndex._from_column( + result_col, name=self.name, freq=self._freq ) def tz_convert(self, tz: str | None): @@ -2684,7 +2699,7 @@ def tz_convert(self, tz: str | None): dtype='datetime64[ns, Europe/London]') """ # noqa: E501 result_col = self._column.tz_convert(tz) - return DatetimeIndex._from_data({self.name: result_col}) + return DatetimeIndex._from_column(result_col, name=self.name) def repeat(self, repeats, axis=None): res = super().repeat(repeats, axis=axis) @@ -2794,6 +2809,15 @@ def __init__( super().__init__(data, name=name) + @classmethod + @_performance_tracking + def _from_column( + cls, column: ColumnBase, *, name: Hashable = None, freq: Any = None + ) -> Self: + if column.dtype.kind != "m": + raise ValueError("column must have a timedelta type.") + return super()._from_column(column, name=name) + def __getitem__(self, index): value = super().__getitem__(index) if cudf.get_option("mode.pandas_compatible") and isinstance( @@ -2876,7 +2900,7 @@ def ceil(self, freq: str) -> Self: This method is currently not implemented. """ - return type(self)._from_data({self.name: self._column.ceil(freq)}) + return type(self)._from_column(self._column.ceil(freq), name=self.name) def floor(self, freq: str) -> Self: """ @@ -2884,7 +2908,9 @@ def floor(self, freq: str) -> Self: This method is currently not implemented. """ - return type(self)._from_data({self.name: self._column.floor(freq)}) + return type(self)._from_column( + self._column.floor(freq), name=self.name + ) def round(self, freq: str) -> Self: """ @@ -2892,41 +2918,51 @@ def round(self, freq: str) -> Self: This method is currently not implemented. """ - return type(self)._from_data({self.name: self._column.round(freq)}) + return type(self)._from_column( + self._column.round(freq), name=self.name + ) @property # type: ignore @_performance_tracking - def days(self): + def days(self) -> cudf.Index: """ Number of days for each element. """ # Need to specifically return `int64` to avoid overflow. - return Index(self._values.days, name=self.name, dtype="int64") + return Index._from_column( + self._column.days.astype("int64"), name=self.name + ) @property # type: ignore @_performance_tracking - def seconds(self): + def seconds(self) -> cudf.Index: """ Number of seconds (>= 0 and less than 1 day) for each element. """ - return Index(self._values.seconds, name=self.name, dtype="int32") + return Index._from_column( + self._column.seconds.astype("int32"), name=self.name + ) @property # type: ignore @_performance_tracking - def microseconds(self): + def microseconds(self) -> cudf.Index: """ Number of microseconds (>= 0 and less than 1 second) for each element. """ - return Index(self._values.microseconds, name=self.name, dtype="int32") + return Index._from_column( + self._column.microseconds.astype("int32"), name=self.name + ) @property # type: ignore @_performance_tracking - def nanoseconds(self): + def nanoseconds(self) -> cudf.Index: """ Number of nanoseconds (>= 0 and less than 1 microsecond) for each element. """ - return Index(self._values.nanoseconds, name=self.name, dtype="int32") + return Index._from_column( + self._column.nanoseconds.astype("int32"), name=self.name + ) @property # type: ignore @_performance_tracking @@ -3061,17 +3097,26 @@ def __init__( data = data.as_ordered(ordered=False) super().__init__(data, name=name) + @classmethod + @_performance_tracking + def _from_column( + cls, column: ColumnBase, *, name: Hashable = None, freq: Any = None + ) -> Self: + if not isinstance(column.dtype, cudf.CategoricalDtype): + raise ValueError("column must have a categorial type.") + return super()._from_column(column, name=name) + @property def ordered(self) -> bool: return self._column.ordered @property # type: ignore @_performance_tracking - def codes(self): + def codes(self) -> cudf.Index: """ The category codes of this categorical. """ - return Index(self._values.codes) + return Index._from_column(self._column.codes) @property # type: ignore @_performance_tracking @@ -3094,24 +3139,24 @@ def add_categories(self, new_categories) -> Self: `new_categories` will be included at the last/highest place in the categories and will be unused directly after this call. """ - return type(self)._from_data( - {self.name: self._column.add_categories(new_categories)} + return type(self)._from_column( + self._column.add_categories(new_categories), name=self.name ) def as_ordered(self) -> Self: """ Set the Categorical to be ordered. """ - return type(self)._from_data( - {self.name: self._column.as_ordered(ordered=True)} + return type(self)._from_column( + self._column.as_ordered(ordered=True), name=self.name ) def as_unordered(self) -> Self: """ Set the Categorical to be unordered. """ - return type(self)._from_data( - {self.name: self._column.as_ordered(ordered=False)} + return type(self)._from_column( + self._column.as_ordered(ordered=False), name=self.name ) def remove_categories(self, removals) -> Self: @@ -3125,8 +3170,8 @@ def remove_categories(self, removals) -> Self: removals : category or list of categories The categories which should be removed. """ - return type(self)._from_data( - {self.name: self._column.remove_categories(removals)} + return type(self)._from_column( + self._column.remove_categories(removals), name=self.name ) def remove_unused_categories(self) -> Self: @@ -3135,8 +3180,8 @@ def remove_unused_categories(self) -> Self: This method is currently not supported. """ - return type(self)._from_data( - {self.name: self._column.remove_unused_categories()} + return type(self)._from_column( + self._column.remove_unused_categories(), name=self.name ) def rename_categories(self, new_categories) -> Self: @@ -3145,8 +3190,8 @@ def rename_categories(self, new_categories) -> Self: This method is currently not supported. """ - return type(self)._from_data( - {self.name: self._column.rename_categories(new_categories)} + return type(self)._from_column( + self._column.rename_categories(new_categories), name=self.name ) def reorder_categories(self, new_categories, ordered=None) -> Self: @@ -3164,12 +3209,9 @@ def reorder_categories(self, new_categories, ordered=None) -> Self: Whether or not the categorical is treated as a ordered categorical. If not given, do not change the ordered information. """ - return type(self)._from_data( - { - self.name: self._column.reorder_categories( - new_categories, ordered=ordered - ) - } + return type(self)._from_column( + self._column.reorder_categories(new_categories, ordered=ordered), + name=self.name, ) def set_categories( @@ -3191,12 +3233,11 @@ def set_categories( considered as a rename of the old categories or as reordered categories. """ - return type(self)._from_data( - { - self.name: self._column.set_categories( - new_categories, ordered=ordered, rename=rename - ) - } + return type(self)._from_column( + self._column.set_categories( + new_categories, ordered=ordered, rename=rename + ), + name=self.name, ) @@ -3411,6 +3452,15 @@ def __init__( def closed(self): return self.dtype.closed + @classmethod + @_performance_tracking + def _from_column( + cls, column: ColumnBase, *, name: Hashable = None, freq: Any = None + ) -> Self: + if not isinstance(column.dtype, cudf.IntervalDtype): + raise ValueError("column must have a interval type.") + return super()._from_column(column, name=name) + @classmethod @_performance_tracking def from_breaks( @@ -3593,8 +3643,8 @@ def set_closed( Whether the intervals are closed on the left-side, right-side, both or neither. """ - return type(self)._from_data( - {self.name: self._column.set_closed(closed)} + return type(self)._from_column( + self._column.set_closed(closed), name=self.name ) def to_tuples(self, na_tuple: bool = True) -> pd.Index: @@ -3680,15 +3730,7 @@ def as_index( elif isinstance(arbitrary, BaseIndex): idx = arbitrary.copy(deep=copy).rename(name) elif isinstance(arbitrary, ColumnBase): - idx = _index_from_data({name: arbitrary}) - elif isinstance(arbitrary, cudf.Series): - return as_index( - arbitrary._column, - nan_as_null=nan_as_null, - copy=copy, - name=name, - dtype=dtype, - ) + raise ValueError("Use cudf.Index._from_column instead.") elif isinstance(arbitrary, (pd.RangeIndex, range)): idx = RangeIndex( start=arbitrary.start, @@ -3708,11 +3750,9 @@ def as_index( elif isinstance(arbitrary, cudf.DataFrame) or is_scalar(arbitrary): raise ValueError("Index data must be 1-dimensional and list-like") else: - return as_index( + return Index._from_column( column.as_column(arbitrary, dtype=dtype, nan_as_null=nan_as_null), - copy=copy, name=name, - dtype=dtype, ) if dtype is not None: idx = idx.astype(dtype) @@ -3749,7 +3789,9 @@ def _concat_range_index(indexes: list[RangeIndex]) -> BaseIndex: elif step is None: # First non-empty index had only one element if obj.start == start: - result = Index(concat_columns([x._values for x in indexes])) + result = Index._from_column( + concat_columns([x._column for x in indexes]) + ) return result step = obj.start - start @@ -3757,7 +3799,9 @@ def _concat_range_index(indexes: list[RangeIndex]) -> BaseIndex: next_ is not None and obj.start != next_ ) if non_consecutive: - result = Index(concat_columns([x._values for x in indexes])) + result = Index._from_column( + concat_columns([x._column for x in indexes]) + ) return result if step is not None: next_ = obj[-1] + step diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 3b44a0f5864..8be9f0ad78e 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -182,11 +182,16 @@ def _indices_from_labels(obj, labels): ) else: labels = labels.astype(obj.index.dtype) + idx_labels = cudf.Index._from_column(labels) + else: + idx_labels = labels # join is not guaranteed to maintain the index ordering # so we will sort it with its initial ordering which is stored # in column "__" - lhs = cudf.DataFrame({"__": as_column(range(len(labels)))}, index=labels) + lhs = cudf.DataFrame( + {"__": as_column(range(len(idx_labels)))}, index=idx_labels + ) rhs = cudf.DataFrame({"_": as_column(range(len(obj)))}, index=obj.index) return lhs.join(rhs).sort_values(by=["__", "_"])["_"] @@ -6642,7 +6647,11 @@ def _drop_rows_by_labels( # 3. Use "leftanti" join to drop # TODO: use internal API with "leftanti" and specify left and right # join keys to bypass logic check - to_join = cudf.DataFrame(index=cudf.Index(labels, name=level)) + if isinstance(labels, ColumnBase): + join_index = cudf.Index._from_column(labels, name=level) + else: + join_index = cudf.Index(labels, name=level) + to_join = cudf.DataFrame(index=join_index) join_res = working_df.join(to_join, how="leftanti") # 4. Reconstruct original layout, and rename @@ -6669,12 +6678,11 @@ def _drop_rows_by_labels( if errors == "raise" and not labels.isin(obj.index).all(): raise KeyError("One or more values not found in axis") - key_df = cudf.DataFrame._from_data( - data={}, - index=cudf.Index( - labels, name=getattr(labels, "name", obj.index.name) - ), - ) + if isinstance(labels, ColumnBase): + idx = cudf.Index._from_column(labels, name=obj.index.name) + else: + idx = cudf.Index(labels, name=labels.name) + key_df = cudf.DataFrame._from_data(data={}, index=idx) if isinstance(obj, cudf.DataFrame): res = obj.join(key_df, how="leftanti") else: diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index ab88b191570..a66e2936e3b 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -811,8 +811,9 @@ def _index_and_downcast(self, result, index, index_key): # it into an Index and name the final index values according # to that column's name. *_, last_column = index._data.columns - out_index = cudf.Index(last_column) - out_index.name = index.names[-1] + out_index = cudf.Index._from_column( + last_column, name=index.names[-1] + ) index = out_index elif out_index._num_columns > 1: # Otherwise pop the leftmost levels, names, and codes from the @@ -1061,7 +1062,7 @@ def get_level_values(self, level): raise KeyError(f"Level not found: '{level}'") else: level_idx = colnames.index(level) - level_values = cudf.Index( + level_values = cudf.Index._from_column( self._data[level], name=self.names[level_idx] ) return level_values diff --git a/python/cudf/cudf/core/resample.py b/python/cudf/cudf/core/resample.py index 715bbf89b15..e0aee28bfeb 100644 --- a/python/cudf/cudf/core/resample.py +++ b/python/cudf/cudf/core/resample.py @@ -145,7 +145,9 @@ def copy(self, deep=True): def keys(self): index = super().keys if self._freq is not None and isinstance(index, cudf.DatetimeIndex): - return cudf.DatetimeIndex._from_data(index._data, freq=self._freq) + return cudf.DatetimeIndex._from_column( + index._column, name=index.name, freq=self._freq + ) return index def serialize(self): diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 822b966364f..2fb4fde6552 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3245,8 +3245,8 @@ def value_counts( interval_col = IntervalColumn.from_struct_column( res.index._column._get_decategorized_column() ) - res.index = cudf.IntervalIndex._from_data( - {res.index.name: interval_col} + res.index = cudf.IntervalIndex._from_column( + interval_col, name=res.index.name ) res.name = result_name return res diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index c50a36b68b5..a92bf420147 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -18,7 +18,6 @@ ) from cudf.api.types import is_integer, is_scalar from cudf.core import column -from cudf.core.column_accessor import ColumnAccessor from cudf.core.index import ensure_index # https://github.com/pandas-dev/pandas/blob/2.2.x/pandas/core/tools/datetimes.py#L1112 @@ -288,8 +287,7 @@ def to_datetime( utc=utc, ) if isinstance(arg, (cudf.BaseIndex, pd.Index)): - ca = ColumnAccessor({arg.name: col}, verify=False) - return cudf.DatetimeIndex._from_data(ca) + return cudf.DatetimeIndex._from_column(col, name=arg.name) elif isinstance(arg, (cudf.Series, pd.Series)): return cudf.Series._from_column( col, name=arg.name, index=ensure_index(arg.index) @@ -297,7 +295,7 @@ def to_datetime( elif is_scalar(arg): return col.element_indexing(0) else: - return cudf.Index(col) + return cudf.Index._from_column(col) except Exception as e: if errors == "raise": raise e @@ -900,7 +898,9 @@ def date_range( end = cudf.Scalar(end, dtype=dtype).value.astype("int64") arr = np.linspace(start=start, stop=end, num=periods) result = cudf.core.column.as_column(arr).astype("datetime64[ns]") - return cudf.DatetimeIndex._from_data({name: result}).tz_localize(tz) + return cudf.DatetimeIndex._from_column(result, name=name).tz_localize( + tz + ) # The code logic below assumes `freq` is defined. It is first normalized # into `DateOffset` for further computation with timestamps. @@ -1001,9 +1001,9 @@ def date_range( "datetime64[ns]" ) - return cudf.DatetimeIndex._from_data({name: res}, freq=freq).tz_localize( - tz - ) + return cudf.DatetimeIndex._from_column( + res, name=name, freq=freq + ).tz_localize(tz) def _has_fixed_frequency(freq: DateOffset) -> bool: diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index c2072d90e98..31ad24a4664 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -398,8 +398,12 @@ def assert_index_equal( ) for level in range(left.nlevels): - llevel = cudf.Index(left._columns[level], name=left.names[level]) - rlevel = cudf.Index(right._columns[level], name=right.names[level]) + llevel = cudf.Index._from_column( + left._columns[level], name=left.names[level] + ) + rlevel = cudf.Index._from_column( + right._columns[level], name=right.names[level] + ) mul_obj = f"MultiIndex level [{level}]" assert_index_equal( llevel, diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index a68f4574da3..b1e095e8853 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -167,7 +167,9 @@ def test_string_index(): pdf.index = stringIndex.to_pandas() gdf.index = stringIndex assert_eq(pdf, gdf) - stringIndex = cudf.Index(as_column(["a", "b", "c", "d", "e"]), name="name") + stringIndex = cudf.Index._from_column( + as_column(["a", "b", "c", "d", "e"]), name="name" + ) pdf.index = stringIndex.to_pandas() gdf.index = stringIndex assert_eq(pdf, gdf) diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index 30880f074c0..cc88cc79769 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -1092,7 +1092,7 @@ def test_string_index(): pdf.index = stringIndex.to_pandas() gdf.index = stringIndex assert_eq(pdf, gdf) - stringIndex = cudf.Index( + stringIndex = cudf.Index._from_column( cudf.core.column.as_column(["a", "b", "c", "d", "e"]), name="name" ) pdf.index = stringIndex.to_pandas() From 89863a3b791250a2285b90d2c13f51f009638f44 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:22:31 -1000 Subject: [PATCH 063/270] Align public utility function signatures with pandas 2.x (#16565) The following function signatures have a breaking change * `concat` * `get_dummies` * `date_range` Additionally deprecates the `cat` argument in `get_dummies` (doesn't exist in pandas and not tested), and fixes a bug in `interval_range` where `names` was not being respected Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16565 --- python/cudf/cudf/__init__.py | 2 + python/cudf/cudf/core/index.py | 4 +- python/cudf/cudf/core/reshape.py | 74 ++++++++++++++----- python/cudf/cudf/core/tools/datetimes.py | 12 +-- python/cudf/cudf/core/tools/numeric.py | 9 ++- .../cudf/cudf/tests/indexes/test_interval.py | 6 ++ python/cudf/cudf/tests/test_onehot.py | 6 ++ 7 files changed, 84 insertions(+), 29 deletions(-) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index e14815a1b0d..77ae0791b81 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -97,6 +97,7 @@ "DatetimeIndex", "Decimal32Dtype", "Decimal64Dtype", + "Decimal128Dtype", "Grouper", "Index", "IntervalDtype", @@ -126,6 +127,7 @@ "isclose", "melt", "merge", + "option_context", "pivot", "pivot_table", "read_avro", diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index c55f86d48e1..d02633a97fa 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3350,14 +3350,14 @@ def interval_range( if len(right_col) == 0 or len(left_col) == 0: dtype = IntervalDtype("int64", closed) data = column.column_empty_like_same_mask(left_col, dtype) - return IntervalIndex(data, closed=closed) + return IntervalIndex(data, closed=closed, name=name) interval_col = IntervalColumn( dtype=IntervalDtype(left_col.dtype, closed), size=len(left_col), children=(left_col, right_col), ) - return IntervalIndex(interval_col, closed=closed) + return IntervalIndex(interval_col, closed=closed, name=name) class IntervalIndex(Index): diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 52a55760d4a..df471692702 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -118,7 +118,17 @@ def _normalize_series_and_dataframe(objs, axis): objs[idx] = obj.to_frame(name=name) -def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): +def concat( + objs, + axis=0, + join="outer", + ignore_index=False, + keys=None, + levels=None, + names=None, + verify_integrity=False, + sort=None, +): """Concatenate DataFrames, Series, or Indices row-wise. Parameters @@ -132,6 +142,21 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): ignore_index : bool, default False Set True to ignore the index of the *objs* and provide a default range index instead. + keys : sequence, default None + If multiple levels passed, should contain tuples. Construct + hierarchical index using the passed keys as the outermost level. + Currently not supported. + levels : list of sequences, default None + Specific levels (unique values) to use for constructing a + MultiIndex. Otherwise they will be inferred from the keys. + Currently not supported. + names : list, default None + Names for the levels in the resulting hierarchical index. + Currently not supported. + verify_integrity : bool, default False + Check whether the new concatenated axis contains duplicates. This can + be very expensive relative to the actual data concatenation. + Currently not supported. sort : bool, default False Sort non-concatenation axis if it is not already aligned. @@ -243,6 +268,12 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): 0 a 1 c 3 1 b 2 d 4 """ + if keys is not None: + raise NotImplementedError("keys is currently not supported") + if levels is not None: + raise NotImplementedError("levels is currently not supported") + if names is not None: + raise NotImplementedError("names is currently not supported") # TODO: Do we really need to have different error messages for an empty # list and a list of None? if not objs: @@ -260,7 +291,7 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): f"Can only concatenate dictionary input along axis=1, not {axis}" ) objs = {k: obj for k, obj in objs.items() if obj is not None} - keys = list(objs) + keys_objs = list(objs) objs = list(objs.values()) if any(isinstance(o, cudf.BaseIndex) for o in objs): raise TypeError( @@ -268,7 +299,7 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): ) else: objs = [obj for obj in objs if obj is not None] - keys = None + keys_objs = None if not objs: raise ValueError("All objects passed were None") @@ -317,8 +348,8 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): result = obj.to_frame() else: result = obj.copy(deep=True) - if keys is not None and isinstance(result, cudf.DataFrame): - k = keys[0] + if keys_objs is not None and isinstance(result, cudf.DataFrame): + k = keys_objs[0] result.columns = cudf.MultiIndex.from_tuples( [ (k, *c) if isinstance(c, tuple) else (k, c) @@ -370,7 +401,7 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): objs = _align_objs(objs, how=join, sort=sort) df.index = objs[0].index - if keys is None: + if keys_objs is None: for o in objs: for name, col in o._data.items(): if name in df._data: @@ -408,9 +439,9 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): "label types in cuDF at this time. You must convert " "the labels to the same type." ) - for k, o in zip(keys, objs): + for k, o in zip(keys_objs, objs): for name, col in o._data.items(): - # if only series, then only keep keys as column labels + # if only series, then only keep keys_objs as column labels # if the existing column is multiindex, prepend it # to handle cases where dfs and srs are concatenated if only_series: @@ -426,7 +457,7 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): else: df[col_label] = col - if keys is None: + if keys_objs is None: df.columns = result_columns.unique() if ignore_index: df.columns = cudf.RangeIndex(len(result_columns.unique())) @@ -666,7 +697,7 @@ def _tile(A, reps): def get_dummies( - df, + data, prefix=None, prefix_sep="_", dummy_na=False, @@ -681,7 +712,7 @@ def get_dummies( Parameters ---------- - df : array-like, Series, or DataFrame + data : array-like, Series, or DataFrame Data of which to get dummy indicators. prefix : str, dict, or sequence, optional Prefix to append. Either a str (to apply a constant prefix), dict @@ -759,17 +790,22 @@ def get_dummies( if cats is None: cats = {} + else: + warnings.warn( + "cats is deprecated and will be removed in a future version.", + FutureWarning, + ) if sparse: raise NotImplementedError("sparse is not supported yet") if drop_first: raise NotImplementedError("drop_first is not supported yet") - if isinstance(df, cudf.DataFrame): + if isinstance(data, cudf.DataFrame): encode_fallback_dtypes = ["object", "category"] if columns is None or len(columns) == 0: - columns = df.select_dtypes( + columns = data.select_dtypes( include=encode_fallback_dtypes )._column_names @@ -796,33 +832,33 @@ def get_dummies( # If we have no columns to encode, we need to drop # fallback columns(if any) if len(columns) == 0: - return df.select_dtypes(exclude=encode_fallback_dtypes) + return data.select_dtypes(exclude=encode_fallback_dtypes) else: result_data = { col_name: col - for col_name, col in df._data.items() + for col_name, col in data._data.items() if col_name not in columns } for name in columns: if name not in cats: unique = _get_unique( - column=df._data[name], dummy_na=dummy_na + column=data._data[name], dummy_na=dummy_na ) else: unique = as_column(cats[name]) col_enc_data = _one_hot_encode_column( - column=df._data[name], + column=data._data[name], categories=unique, prefix=prefix_map.get(name, prefix), prefix_sep=prefix_sep_map.get(name, prefix_sep), dtype=dtype, ) result_data.update(col_enc_data) - return cudf.DataFrame._from_data(result_data, index=df.index) + return cudf.DataFrame._from_data(result_data, index=data.index) else: - ser = cudf.Series(df) + ser = cudf.Series(data) unique = _get_unique(column=ser._column, dummy_na=dummy_na) data = _one_hot_encode_column( column=ser._column, diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index a92bf420147..7197560b5a4 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -785,7 +785,7 @@ def date_range( tz=None, normalize: bool = False, name=None, - closed: Literal["left", "right", "both", "neither"] = "both", + inclusive: Literal["left", "right", "both", "neither"] = "both", *, unit: str | None = None, ): @@ -823,7 +823,7 @@ def date_range( name : str, default None Name of the resulting DatetimeIndex - closed : {"left", "right", "both", "neither"}, default "both" + inclusive : {"left", "right", "both", "neither"}, default "both" Whether to set each bound as closed or open. Currently only "both" is supported @@ -839,7 +839,7 @@ def date_range( ----- Of the four parameters `start`, `end`, `periods`, and `freq`, exactly three must be specified. If `freq` is omitted, the resulting DatetimeIndex will - have periods linearly spaced elements between start and end (closed on both + have periods linearly spaced elements between start and end (inclusive on both sides). cudf supports `freq` specified with either fixed-frequency offset @@ -866,8 +866,8 @@ def date_range( '2026-04-23 08:00:00'], dtype='datetime64[ns]') """ - if closed != "both": - raise NotImplementedError(f"{closed=} is currently unsupported.") + if inclusive != "both": + raise NotImplementedError(f"{inclusive=} is currently unsupported.") if unit is not None: raise NotImplementedError(f"{unit=} is currently unsupported.") if normalize is not False: @@ -961,7 +961,7 @@ def date_range( periods = 0 else: # If end == start, periods == 0 and we return exactly 1 timestamp (start). - # Otherwise, since closed="both", we ensure the end point is included. + # Otherwise, since inclusive="both", we ensure the end point is included. periods += 1 # We compute `end_estim` (the estimated upper bound of the date diff --git a/python/cudf/cudf/core/tools/numeric.py b/python/cudf/cudf/core/tools/numeric.py index 8b95f6f6a04..6cecf3fa170 100644 --- a/python/cudf/cudf/core/tools/numeric.py +++ b/python/cudf/cudf/core/tools/numeric.py @@ -20,7 +20,7 @@ from cudf.core.column import ColumnBase -def to_numeric(arg, errors="raise", downcast=None): +def to_numeric(arg, errors="raise", downcast=None, dtype_backend=None): """ Convert argument into numerical types. @@ -48,6 +48,8 @@ def to_numeric(arg, errors="raise", downcast=None): Note that downcast behavior is decoupled from parsing. Errors encountered during downcast is raised regardless of ``errors`` parameter. + dtype_backend : None + Not implemented. Returns ------- @@ -93,7 +95,10 @@ def to_numeric(arg, errors="raise", downcast=None): For example ``[1, 'a']``. A ``TypeError`` will be raised when such input is received, regardless of ``errors`` parameter. """ - + if dtype_backend is not None: + raise NotImplementedError( + "dtype_backend is not currently implemented." + ) if errors not in {"raise", "ignore", "coerce"}: raise ValueError("invalid error value specified") elif errors == "ignore": diff --git a/python/cudf/cudf/tests/indexes/test_interval.py b/python/cudf/cudf/tests/indexes/test_interval.py index 3b3a9f96543..a567c27f584 100644 --- a/python/cudf/cudf/tests/indexes/test_interval.py +++ b/python/cudf/cudf/tests/indexes/test_interval.py @@ -401,3 +401,9 @@ def test_from_tuples(): result = cudf.IntervalIndex.from_tuples(data, closed="left", name="a") expected = pd.IntervalIndex.from_tuples(data, closed="left", name="a") assert_eq(result, expected) + + +def test_interval_range_name(): + expected = pd.interval_range(start=0, periods=5, freq=2, name="foo") + result = cudf.interval_range(start=0, periods=5, freq=2, name="foo") + assert_eq(result, expected) diff --git a/python/cudf/cudf/tests/test_onehot.py b/python/cudf/cudf/tests/test_onehot.py index 154e1e19072..cc17dc46e0a 100644 --- a/python/cudf/cudf/tests/test_onehot.py +++ b/python/cudf/cudf/tests/test_onehot.py @@ -155,3 +155,9 @@ def test_get_dummies_array_like_with_nan(): actual = cudf.get_dummies(ser, dummy_na=True, prefix="a", prefix_sep="_") assert_eq(expected, actual) + + +def test_get_dummies_cats_deprecated(): + df = cudf.DataFrame(range(3)) + with pytest.warns(FutureWarning): + cudf.get_dummies(df, cats={0: [0, 1, 2]}) From 2bcb7ecd2c077b3989ced1b8be8727e1b71f93b1 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:24:48 -1000 Subject: [PATCH 064/270] Fix `.replace(Index, Index)` raising a TypeError (#16513) Since `cudf.Index` is list-like, passing this to `.replace` should act like replacing a list of values with a corresponding list of values. Discovered while working on https://github.com/rapidsai/cuml/pull/6019 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16513 --- python/cudf/cudf/core/indexed_frame.py | 14 +++++++------- python/cudf/cudf/tests/test_replace.py | 6 ++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 8be9f0ad78e..ae7369c80d1 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -6469,7 +6469,7 @@ def _get_replacement_values_for_columns( to_replace_columns = {col: [to_replace] for col in columns_dtype_map} values_columns = {col: [value] for col in columns_dtype_map} elif cudf.api.types.is_list_like(to_replace) or isinstance( - to_replace, ColumnBase + to_replace, (ColumnBase, BaseIndex) ): if is_scalar(value): to_replace_columns = {col: to_replace for col in columns_dtype_map} @@ -6483,7 +6483,9 @@ def _get_replacement_values_for_columns( ) for col in columns_dtype_map } - elif cudf.api.types.is_list_like(value): + elif cudf.api.types.is_list_like( + value + ) or cudf.utils.dtypes.is_column_like(value): if len(to_replace) != len(value): raise ValueError( f"Replacement lists must be " @@ -6495,9 +6497,6 @@ def _get_replacement_values_for_columns( col: to_replace for col in columns_dtype_map } values_columns = {col: value for col in columns_dtype_map} - elif cudf.utils.dtypes.is_column_like(value): - to_replace_columns = {col: to_replace for col in columns_dtype_map} - values_columns = {col: value for col in columns_dtype_map} else: raise TypeError( "value argument must be scalar, list-like or Series" @@ -6592,12 +6591,13 @@ def _get_replacement_values_for_columns( return all_na_columns, to_replace_columns, values_columns -def _is_series(obj): +def _is_series(obj: Any) -> bool: """ Checks if the `obj` is of type `cudf.Series` instead of checking for isinstance(obj, cudf.Series) + to avoid circular imports. """ - return isinstance(obj, Frame) and obj.ndim == 1 and obj.index is not None + return isinstance(obj, IndexedFrame) and obj.ndim == 1 @_performance_tracking diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 1973fe6fb41..e5ee0127a74 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -1378,3 +1378,9 @@ def test_fillna_nan_and_null(): result = ser.fillna(2.2) expected = cudf.Series([2.2, 2.2, 1.1]) assert_eq(result, expected) + + +def test_replace_with_index_objects(): + result = cudf.Series([1, 2]).replace(cudf.Index([1]), cudf.Index([2])) + expected = pd.Series([1, 2]).replace(pd.Index([1]), pd.Index([2])) + assert_eq(result, expected) From ac42bc870a65d807784cae63e25b9e9ca788eb23 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Thu, 15 Aug 2024 09:37:43 -0400 Subject: [PATCH 065/270] Hide all gtest symbols in cudftestutil (#16546) By hiding the gtest symbols in cudftestutil it allows consumers of the library to build with a differing version of gtest without issue. Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Marcus D. Hanwell (https://github.com/cryos) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16546 --- cpp/cmake/thirdparty/get_gtest.cmake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 10e6b026d9a..ec8cbd8c568 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -16,9 +16,18 @@ function(find_and_configure_gtest) include(${rapids-cmake-dir}/cpm/gtest.cmake) + # Mark all the non explicit googletest symbols as hidden. This ensures that libcudftestutil can be + # used by consumers with a different shared gtest. + set(gtest_hide_internal_symbols ON) + # Find or install GoogleTest rapids_cpm_gtest(BUILD_STATIC) + # Mark all the explicit googletest symbols as hidden. This ensures that libcudftestutil can be + # used by consumers with a different shared gtest. + if(TARGET gtest) + target_compile_definitions(gtest PUBLIC "$") + endif() endfunction() find_and_configure_gtest() From f4a9b1c5016e254ebf2de55ac9946af6420ebff5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:14:06 -1000 Subject: [PATCH 066/270] Use more idomatic cudf APIs in dask_cudf meta generation (#16487) Namely: * Avoiding `cudf.core` imports by checking public column `.dtype`s * Using more straightforward cudf APIs to construct meta objects Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16487 --- python/dask_cudf/dask_cudf/backends.py | 124 ++++++++++++------------- 1 file changed, 58 insertions(+), 66 deletions(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 01bab30190a..82ea2ac033a 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -55,37 +55,31 @@ @meta_nonempty.register(cudf.BaseIndex) @_dask_cudf_performance_tracking def _nonempty_index(idx): - if isinstance(idx, cudf.core.index.RangeIndex): - return cudf.core.index.RangeIndex(2, name=idx.name) - elif isinstance(idx, cudf.core.index.DatetimeIndex): - start = "1970-01-01" - data = np.array([start, "1970-01-02"], dtype=idx.dtype) + """Return a non-empty cudf.Index as metadata.""" + # TODO: IntervalIndex, TimedeltaIndex? + if isinstance(idx, cudf.RangeIndex): + return cudf.RangeIndex(2, name=idx.name) + elif isinstance(idx, cudf.DatetimeIndex): + data = np.array(["1970-01-01", "1970-01-02"], dtype=idx.dtype) values = cudf.core.column.as_column(data) - return cudf.core.index.DatetimeIndex(values, name=idx.name) - elif isinstance(idx, cudf.core.index.CategoricalIndex): - key = tuple(idx._data.keys()) - assert len(key) == 1 - categories = idx._data[key[0]].categories - codes = [0, 0] - ordered = idx._data[key[0]].ordered + return cudf.DatetimeIndex(values, name=idx.name) + elif isinstance(idx, cudf.CategoricalIndex): values = cudf.core.column.build_categorical_column( - categories=categories, codes=codes, ordered=ordered + categories=idx.categories, codes=[0, 0], ordered=idx.ordered ) - return cudf.core.index.CategoricalIndex(values, name=idx.name) - elif isinstance(idx, cudf.core.multiindex.MultiIndex): + return cudf.CategoricalIndex(values, name=idx.name) + elif isinstance(idx, cudf.MultiIndex): levels = [meta_nonempty(lev) for lev in idx.levels] - codes = [[0, 0] for i in idx.levels] - return cudf.core.multiindex.MultiIndex( - levels=levels, codes=codes, names=idx.names - ) - elif isinstance(idx._column, cudf.core.column.StringColumn): + codes = [[0, 0]] * idx.nlevels + return cudf.MultiIndex(levels=levels, codes=codes, names=idx.names) + elif is_string_dtype(idx.dtype): return cudf.Index(["cat", "dog"], name=idx.name) - elif isinstance(idx, cudf.core.index.Index): - return cudf.core.index.Index( - np.arange(2, dtype=idx.dtype), name=idx.name - ) + elif isinstance(idx, cudf.Index): + return cudf.Index(np.arange(2, dtype=idx.dtype), name=idx.name) - raise TypeError(f"Don't know how to handle index of type {type(idx)}") + raise TypeError( + f"Don't know how to handle index of type {type(idx).__name__}" + ) def _nest_list_data(data, leaf_type): @@ -101,50 +95,49 @@ def _nest_list_data(data, leaf_type): @_dask_cudf_performance_tracking -def _get_non_empty_data(s): - """Return a non empty column as metadata.""" - if isinstance(s, cudf.core.column.CategoricalColumn): +def _get_non_empty_data( + s: cudf.core.column.ColumnBase, +) -> cudf.core.column.ColumnBase: + """Return a non-empty column as metadata from a column.""" + if isinstance(s.dtype, cudf.CategoricalDtype): categories = ( - s.categories if len(s.categories) else [UNKNOWN_CATEGORIES] + s.categories if len(s.categories) else [UNKNOWN_CATEGORIES] # type: ignore[attr-defined] ) codes = cudf.core.column.as_column( 0, dtype=cudf._lib.types.size_type_dtype, length=2, ) - ordered = s.ordered - data = cudf.core.column.build_categorical_column( + ordered = s.ordered # type: ignore[attr-defined] + return cudf.core.column.build_categorical_column( categories=categories, codes=codes, ordered=ordered ) - elif isinstance(s, cudf.core.column.ListColumn): + elif isinstance(s.dtype, cudf.ListDtype): leaf_type = s.dtype.leaf_type if is_string_dtype(leaf_type): data = ["cat", "dog"] else: data = np.array([0, 1], dtype=leaf_type).tolist() data = _nest_list_data(data, s.dtype) * 2 - data = cudf.core.column.as_column(data, dtype=s.dtype) - elif isinstance(s, cudf.core.column.StructColumn): + return cudf.core.column.as_column(data, dtype=s.dtype) + elif isinstance(s.dtype, cudf.StructDtype): + # Handles IntervalColumn struct_dtype = s.dtype - data = [{key: None for key in struct_dtype.fields.keys()}] * 2 - data = cudf.core.column.as_column(data, dtype=s.dtype) + struct_data = [{key: None for key in struct_dtype.fields.keys()}] * 2 + return cudf.core.column.as_column(struct_data, dtype=s.dtype) elif is_string_dtype(s.dtype): - data = cudf.core.column.as_column(pa.array(["cat", "dog"])) + return cudf.core.column.as_column(pa.array(["cat", "dog"])) elif isinstance(s.dtype, pd.DatetimeTZDtype): - from cudf.utils.dtypes import get_time_unit - - data = cudf.date_range("2001-01-01", periods=2, freq=get_time_unit(s)) - data = data.tz_localize(str(s.dtype.tz))._column + date_data = cudf.date_range("2001-01-01", periods=2, freq=s.time_unit) # type: ignore[attr-defined] + return date_data.tz_localize(str(s.dtype.tz))._column + elif s.dtype.kind in "fiubmM": + return cudf.core.column.as_column( + np.arange(start=0, stop=2, dtype=s.dtype) + ) else: - if pd.api.types.is_numeric_dtype(s.dtype): - data = cudf.core.column.as_column( - cp.arange(start=0, stop=2, dtype=s.dtype) - ) - else: - data = cudf.core.column.as_column( - cp.arange(start=0, stop=2, dtype="int64") - ).astype(s.dtype) - return data + raise TypeError( + f"Don't know how to handle column of type {type(s).__name__}" + ) @meta_nonempty.register(cudf.Series) @@ -162,24 +155,25 @@ def _nonempty_series(s, idx=None): def meta_nonempty_cudf(x): idx = meta_nonempty(x.index) columns_with_dtype = dict() - res = cudf.DataFrame(index=idx) - for col in x._data.names: - dtype = str(x._data[col].dtype) - if dtype in ("list", "struct", "category"): + res = {} + for col_label, col in x._data.items(): + dtype = col.dtype + if isinstance( + dtype, + (cudf.ListDtype, cudf.StructDtype, cudf.CategoricalDtype), + ): # 1. Not possible to hash and store list & struct types # as they can contain different levels of nesting or # fields. - # 2. Not possible to has `category` types as + # 2. Not possible to hash `category` types as # they often contain an underlying types to them. - res._data[col] = _get_non_empty_data(x._data[col]) + res[col_label] = _get_non_empty_data(col) else: if dtype not in columns_with_dtype: - columns_with_dtype[dtype] = cudf.core.column.as_column( - _get_non_empty_data(x._data[col]) - ) - res._data[col] = columns_with_dtype[dtype] + columns_with_dtype[dtype] = _get_non_empty_data(col) + res[col_label] = columns_with_dtype[dtype] - return res + return cudf.DataFrame._from_data(res, index=idx) @make_meta_dispatch.register((cudf.Series, cudf.DataFrame)) @@ -197,9 +191,7 @@ def make_meta_cudf_index(x, index=None): @_dask_cudf_performance_tracking def _empty_series(name, dtype, index=None): if isinstance(dtype, str) and dtype == "category": - return cudf.Series( - [UNKNOWN_CATEGORIES], dtype=dtype, name=name, index=index - ).iloc[:0] + dtype = cudf.CategoricalDtype(categories=[UNKNOWN_CATEGORIES]) return cudf.Series([], dtype=dtype, name=name, index=index) @@ -337,7 +329,7 @@ def percentile_cudf(a, q, interpolation="linear"): if isinstance(q, Iterator): q = list(q) - if cudf.api.types._is_categorical_dtype(a.dtype): + if isinstance(a.dtype, cudf.CategoricalDtype): result = cp.percentile(a.cat.codes, q, interpolation=interpolation) return ( @@ -346,7 +338,7 @@ def percentile_cudf(a, q, interpolation="linear"): ), n, ) - if np.issubdtype(a.dtype, np.datetime64): + if a.dtype.kind == "M": result = a.quantile( [i / 100.0 for i in q], interpolation=interpolation ) From 1e220b708582c73d128c53f3279d4588167a310f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:58:45 -1000 Subject: [PATCH 067/270] Return Interval object in pandas compat mode for IntervalIndex reductions (#16523) xref https://github.com/rapidsai/cudf/issues/16507 In non pandas compat mode, I think this still makes sense to return a `dict` since that's the "scalar" type of a cudf struct/interval type, but in pandas compat mode we should match pandas and return an Interval. Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16523 --- python/cudf/cudf/_lib/reduce.pyx | 6 +++++- python/cudf/cudf/core/column/interval.py | 14 ++++++++++++++ python/cudf/cudf/tests/test_interval.py | 11 +++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/_lib/reduce.pyx b/python/cudf/cudf/_lib/reduce.pyx index 64634b7a6f9..511bba20ef5 100644 --- a/python/cudf/cudf/_lib/reduce.pyx +++ b/python/cudf/cudf/_lib/reduce.pyx @@ -61,7 +61,11 @@ def reduce(reduction_op, Column incol, dtype=None, **kwargs): result, dtype=col_dtype.__class__(precision, scale), ).value - return DeviceScalar.from_pylibcudf(result).value + scalar = DeviceScalar.from_pylibcudf(result).value + if isinstance(col_dtype, cudf.StructDtype): + # TODO: Utilize column_metadata in libcudf to maintain field labels + return dict(zip(col_dtype.fields.keys(), scalar.values())) + return scalar @acquire_spill_lock() diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index b2f79ef0c65..d9fc96a9f3e 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -11,6 +11,7 @@ from cudf.core.dtypes import IntervalDtype if TYPE_CHECKING: + from cudf._typing import ScalarLike from cudf.core.column import ColumnBase @@ -186,3 +187,16 @@ def element_indexing(self, index: int): if cudf.get_option("mode.pandas_compatible"): return pd.Interval(**result, closed=self.dtype.closed) return result + + def _reduce( + self, + op: str, + skipna: bool | None = None, + min_count: int = 0, + *args, + **kwargs, + ) -> ScalarLike: + result = super()._reduce(op, skipna, min_count, *args, **kwargs) + if cudf.get_option("mode.pandas_compatible"): + return pd.Interval(**result, closed=self.dtype.closed) + return result diff --git a/python/cudf/cudf/tests/test_interval.py b/python/cudf/cudf/tests/test_interval.py index 5eeea87d8e0..2d194107658 100644 --- a/python/cudf/cudf/tests/test_interval.py +++ b/python/cudf/cudf/tests/test_interval.py @@ -194,3 +194,14 @@ def test_intervaldtype_eq_string_with_attributes(): dtype = cudf.IntervalDtype("int64", closed="left") assert dtype == "interval" assert dtype == "interval[int64, left]" + + +def test_reduction_return_interval_pandas_compatible(): + ii = pd.IntervalIndex.from_tuples( + [("2017-01-03", "2017-01-04")], dtype="interval[datetime64[ns], right]" + ) + cudf_ii = cudf.IntervalIndex.from_pandas(ii) + with cudf.option_context("mode.pandas_compatible", True): + result = cudf_ii.min() + expected = ii.min() + assert result == expected From 50841355812685e0e48d1577b8384399cdad5a0f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:59:58 -1000 Subject: [PATCH 068/270] Make NumericalColumn.__init__ strict (#16457) This PR makes `NumericalBaseColumn.__init__` and its subclasses strict putting restrictions on `data`, `dtype`, `size` and `children` so these columns cannot be constructed into to an invalid state. It also aligns the signature with the base class. xref https://github.com/rapidsai/cudf/issues/16469 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16457 --- python/cudf/cudf/_lib/column.pyx | 2 + python/cudf/cudf/core/column/column.py | 22 ++--- python/cudf/cudf/core/column/decimal.py | 92 ++++++++++++++++++- python/cudf/cudf/core/column/numerical.py | 13 ++- .../cudf/cudf/core/column/numerical_base.py | 29 +++++- 5 files changed, 134 insertions(+), 24 deletions(-) diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index e030147fdd3..f0c07dfbc1b 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -88,6 +88,8 @@ cdef class Column: object null_count=None, object children=() ): + if size < 0: + raise ValueError("size must be >=0") self._size = size self._distinct_count = {} self._dtype = dtype diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index a7d2cb441dd..9785c3e5517 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1652,23 +1652,19 @@ def build_column( null_count=null_count, ) elif isinstance(dtype, StructDtype): - if size is None: - raise TypeError("Must specify size") return cudf.core.column.StructColumn( data=data, dtype=dtype, - size=size, + size=size, # type: ignore[arg-type] offset=offset, mask=mask, null_count=null_count, children=children, ) elif isinstance(dtype, cudf.Decimal64Dtype): - if size is None: - raise TypeError("Must specify size") return cudf.core.column.Decimal64Column( - data=data, - size=size, + data=data, # type: ignore[arg-type] + size=size, # type: ignore[arg-type] offset=offset, dtype=dtype, mask=mask, @@ -1676,11 +1672,9 @@ def build_column( children=children, ) elif isinstance(dtype, cudf.Decimal32Dtype): - if size is None: - raise TypeError("Must specify size") return cudf.core.column.Decimal32Column( - data=data, - size=size, + data=data, # type: ignore[arg-type] + size=size, # type: ignore[arg-type] offset=offset, dtype=dtype, mask=mask, @@ -1688,11 +1682,9 @@ def build_column( children=children, ) elif isinstance(dtype, cudf.Decimal128Dtype): - if size is None: - raise TypeError("Must specify size") return cudf.core.column.Decimal128Column( - data=data, - size=size, + data=data, # type: ignore[arg-type] + size=size, # type: ignore[arg-type] offset=offset, dtype=dtype, mask=mask, diff --git a/python/cudf/cudf/core/column/decimal.py b/python/cudf/cudf/core/column/decimal.py index 6a7f338b065..3b979ef2e97 100644 --- a/python/cudf/cudf/core/column/decimal.py +++ b/python/cudf/cudf/core/column/decimal.py @@ -31,14 +31,38 @@ if TYPE_CHECKING: from cudf._typing import ColumnBinaryOperand, ColumnLike, Dtype, ScalarLike + from cudf.core.buffer import Buffer class DecimalBaseColumn(NumericalBaseColumn): """Base column for decimal32, decimal64 or decimal128 columns""" - dtype: DecimalDtype _VALID_BINARY_OPERATIONS = BinaryOperand._SUPPORTED_BINARY_OPERATIONS + def __init__( + self, + data: Buffer, + size: int, + dtype: DecimalDtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple = (), + ): + if not isinstance(size, int): + raise ValueError("Must specify an integer size") + if not isinstance(dtype, DecimalDtype): + raise ValueError(f"{dtype=} must be a DecimalDtype instance") + super().__init__( + data=data, + size=size, + dtype=dtype, + mask=mask, + offset=offset, + null_count=null_count, + children=children, + ) + @property def __cuda_array_interface__(self): raise NotImplementedError( @@ -205,7 +229,27 @@ def as_numerical_column( class Decimal32Column(DecimalBaseColumn): - dtype: Decimal32Dtype + def __init__( + self, + data: Buffer, + size: int, + dtype: Decimal32Dtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple = (), + ): + if not isinstance(dtype, Decimal32Dtype): + raise ValueError(f"{dtype=} must be a Decimal32Dtype instance") + super().__init__( + data=data, + size=size, + dtype=dtype, + mask=mask, + offset=offset, + null_count=null_count, + children=children, + ) @classmethod def from_arrow(cls, data: pa.Array): @@ -266,7 +310,27 @@ def _with_type_metadata( class Decimal128Column(DecimalBaseColumn): - dtype: Decimal128Dtype + def __init__( + self, + data: Buffer, + size: int, + dtype: Decimal128Dtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple = (), + ): + if not isinstance(dtype, Decimal128Dtype): + raise ValueError(f"{dtype=} must be a Decimal128Dtype instance") + super().__init__( + data=data, + size=size, + dtype=dtype, + mask=mask, + offset=offset, + null_count=null_count, + children=children, + ) @classmethod def from_arrow(cls, data: pa.Array): @@ -287,7 +351,27 @@ def _with_type_metadata( class Decimal64Column(DecimalBaseColumn): - dtype: Decimal64Dtype + def __init__( + self, + data: Buffer, + size: int, + dtype: Decimal64Dtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple = (), + ): + if not isinstance(dtype, Decimal64Dtype): + raise ValueError(f"{dtype=} must be a Decimal64Dtype instance") + super().__init__( + data=data, + size=size, + dtype=dtype, + mask=mask, + offset=offset, + null_count=null_count, + children=children, + ) def __setitem__(self, key, value): if isinstance(value, np.integer): diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index bbc74ef349e..16e78ef35ef 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -61,25 +61,30 @@ class NumericalColumn(NumericalBaseColumn): def __init__( self, data: Buffer, - dtype: DtypeObj, + size: int | None, + dtype: np.dtype, mask: Buffer | None = None, - size: int | None = None, # TODO: make this non-optional offset: int = 0, null_count: int | None = None, + children: tuple = (), ): - dtype = cudf.dtype(dtype) + if not (isinstance(dtype, np.dtype) and dtype.kind in "iufb"): + raise ValueError( + "dtype must be a floating, integer or boolean numpy dtype." + ) if data.size % dtype.itemsize: raise ValueError("Buffer size must be divisible by element size") if size is None: size = (data.size // dtype.itemsize) - offset super().__init__( - data, + data=data, size=size, dtype=dtype, mask=mask, offset=offset, null_count=null_count, + children=children, ) def _clear_cache(self): diff --git a/python/cudf/cudf/core/column/numerical_base.py b/python/cudf/cudf/core/column/numerical_base.py index f41010062c8..3b8dd05c13a 100644 --- a/python/cudf/cudf/core/column/numerical_base.py +++ b/python/cudf/cudf/core/column/numerical_base.py @@ -9,16 +9,19 @@ import cudf from cudf import _lib as libcudf +from cudf.core.buffer import Buffer from cudf.core.column import ColumnBase from cudf.core.missing import NA from cudf.core.mixins import Scannable if TYPE_CHECKING: from cudf._typing import ScalarLike + from cudf.core.column.decimal import DecimalDtype class NumericalBaseColumn(ColumnBase, Scannable): - """A column composed of numerical data. + """ + A column composed of numerical (bool, integer, float, decimal) data. This class encodes a standard interface for different types of columns containing numerical types of data. In particular, mathematical operations @@ -42,6 +45,30 @@ class NumericalBaseColumn(ColumnBase, Scannable): "cummax", } + def __init__( + self, + data: Buffer, + size: int, + dtype: DecimalDtype | np.dtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple = (), + ): + if not isinstance(data, Buffer): + raise ValueError("data must be a Buffer instance.") + if len(children) != 0: + raise ValueError(f"{type(self).__name__} must have no children.") + super().__init__( + data=data, + size=size, + dtype=dtype, + mask=mask, + offset=offset, + null_count=null_count, + children=children, + ) + def _can_return_nan(self, skipna: bool | None = None) -> bool: return not skipna and self.has_nulls() From 155eddedc0e2b68d203cfbc318172396f4293d98 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Aug 2024 14:00:57 -1000 Subject: [PATCH 069/270] Make Timedelta/DatetimeColumn.__init__ strict (#16464) This PR makes Datetime/TimedeltaColumn.__init__ and its subclasses strict putting restrictions on data, dtype, size and children so these columns cannot be constructed into to an invalid state. It also aligns the signature with the base class. xref https://github.com/rapidsai/cudf/issues/16469 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16464 --- python/cudf/cudf/core/column/column.py | 12 ++----- python/cudf/cudf/core/column/datetime.py | 43 ++++++++++++++++------- python/cudf/cudf/core/column/timedelta.py | 17 +++++---- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 9785c3e5517..b0e33e8b9ce 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1592,10 +1592,8 @@ def build_column( children=children, ) elif dtype.type is np.datetime64: - if data is None: - raise TypeError("Must specify data buffer") return cudf.core.column.DatetimeColumn( - data=data, + data=data, # type: ignore[arg-type] dtype=dtype, mask=mask, size=size, @@ -1603,10 +1601,8 @@ def build_column( null_count=null_count, ) elif isinstance(dtype, pd.DatetimeTZDtype): - if data is None: - raise TypeError("Must specify data buffer") return cudf.core.column.datetime.DatetimeTZColumn( - data=data, + data=data, # type: ignore[arg-type] dtype=dtype, mask=mask, size=size, @@ -1614,10 +1610,8 @@ def build_column( null_count=null_count, ) elif dtype.type is np.timedelta64: - if data is None: - raise TypeError("Must specify data buffer") return cudf.core.column.TimeDeltaColumn( - data=data, + data=data, # type: ignore[arg-type] dtype=dtype, mask=mask, size=size, diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 1dbc94384d3..d0ea4612a1b 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -24,6 +24,7 @@ get_compatible_timezone, get_tz_data, ) +from cudf.core.buffer import Buffer from cudf.core.column import ColumnBase, as_column, column, string from cudf.core.column.timedelta import _unit_to_nanoseconds_conversion from cudf.utils.dtypes import _get_base_dtype @@ -34,10 +35,8 @@ ColumnBinaryOperand, DatetimeLikeScalar, Dtype, - DtypeObj, ScalarLike, ) - from cudf.core.buffer import Buffer from cudf.core.column.numerical import NumericalColumn if PANDAS_GE_220: @@ -207,30 +206,39 @@ class DatetimeColumn(column.ColumnBase): def __init__( self, data: Buffer, - dtype: DtypeObj, + size: int | None, + dtype: np.dtype | pd.DatetimeTZDtype, mask: Buffer | None = None, - size: int | None = None, # TODO: make non-optional offset: int = 0, null_count: int | None = None, + children: tuple = (), ): - dtype = cudf.dtype(dtype) - if dtype.kind != "M": - raise TypeError(f"{self.dtype} is not a supported datetime type") - + if not isinstance(data, Buffer): + raise ValueError("data must be a Buffer.") + dtype = self._validate_dtype_instance(dtype) if data.size % dtype.itemsize: raise ValueError("Buffer size must be divisible by element size") if size is None: size = data.size // dtype.itemsize size = size - offset + if len(children) != 0: + raise ValueError(f"{type(self).__name__} must have no children.") super().__init__( - data, + data=data, size=size, dtype=dtype, mask=mask, offset=offset, null_count=null_count, + children=children, ) + @staticmethod + def _validate_dtype_instance(dtype: np.dtype) -> np.dtype: + if not (isinstance(dtype, np.dtype) and dtype.kind == "M"): + raise ValueError("dtype must be a datetime, numpy dtype") + return dtype + def __contains__(self, item: ScalarLike) -> bool: try: ts = pd.Timestamp(item).as_unit(self.time_unit) @@ -858,21 +866,30 @@ class DatetimeTZColumn(DatetimeColumn): def __init__( self, data: Buffer, + size: int | None, dtype: pd.DatetimeTZDtype, mask: Buffer | None = None, - size: int | None = None, offset: int = 0, null_count: int | None = None, + children: tuple = (), ): super().__init__( data=data, - dtype=_get_base_dtype(dtype), - mask=mask, size=size, + dtype=dtype, + mask=mask, offset=offset, null_count=null_count, + children=children, ) - self._dtype = get_compatible_timezone(dtype) + + @staticmethod + def _validate_dtype_instance( + dtype: pd.DatetimeTZDtype, + ) -> pd.DatetimeTZDtype: + if not isinstance(dtype, pd.DatetimeTZDtype): + raise ValueError("dtype must be a pandas.DatetimeTZDtype") + return get_compatible_timezone(dtype) def to_pandas( self, diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index ba0dc4779bb..6b6f3e517a8 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -75,28 +75,33 @@ class TimeDeltaColumn(ColumnBase): def __init__( self, data: Buffer, - dtype: Dtype, - size: int | None = None, # TODO: make non-optional + size: int | None, + dtype: np.dtype, mask: Buffer | None = None, offset: int = 0, null_count: int | None = None, + children: tuple = (), ): - dtype = cudf.dtype(dtype) - if dtype.kind != "m": - raise TypeError(f"{self.dtype} is not a supported duration type") + if not isinstance(data, Buffer): + raise ValueError("data must be a Buffer.") + if not (isinstance(dtype, np.dtype) and dtype.kind == "m"): + raise ValueError("dtype must be a timedelta numpy dtype.") if data.size % dtype.itemsize: raise ValueError("Buffer size must be divisible by element size") if size is None: size = data.size // dtype.itemsize size = size - offset + if len(children) != 0: + raise ValueError("TimedeltaColumn must have no children.") super().__init__( - data, + data=data, size=size, dtype=dtype, mask=mask, offset=offset, null_count=null_count, + children=children, ) def __contains__(self, item: DatetimeLikeScalar) -> bool: From f955dd76b47779d4f527efe25de417b1acbff4a7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 15 Aug 2024 17:13:58 -0700 Subject: [PATCH 070/270] Rewrite remaining Python Arrow interop conversions using the C Data Interface (#16548) This PR rewrites all remaining parts of the Python interop code previously using Arrow C++ types to instead use the C Data Interface. With this change, we no longer require pyarrow in that part of the Cython code. There are further improvements that we should make to streamline the internals, but I would like to keep this changeset minimal since getting it merged unblocks progress on multiple fronts so that we can progress further in parallel. Contributes to #15193 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/16548 --- cpp/src/interop/arrow_utilities.cpp | 1 + cpp/src/interop/to_arrow_schema.cpp | 5 +- python/cudf/cudf/_lib/CMakeLists.txt | 6 +- .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 5 +- python/cudf/cudf/_lib/pylibcudf/interop.pyx | 188 +++++++++--------- .../cudf/_lib/pylibcudf/libcudf/interop.pxd | 53 +++-- .../cudf/cudf/pylibcudf_tests/common/utils.py | 6 +- 7 files changed, 146 insertions(+), 118 deletions(-) diff --git a/cpp/src/interop/arrow_utilities.cpp b/cpp/src/interop/arrow_utilities.cpp index 4292552a800..3776daf41aa 100644 --- a/cpp/src/interop/arrow_utilities.cpp +++ b/cpp/src/interop/arrow_utilities.cpp @@ -98,6 +98,7 @@ ArrowType id_to_arrow_type(cudf::type_id id) ArrowType id_to_arrow_storage_type(cudf::type_id id) { switch (id) { + case cudf::type_id::TIMESTAMP_DAYS: return NANOARROW_TYPE_INT32; case cudf::type_id::TIMESTAMP_SECONDS: case cudf::type_id::TIMESTAMP_MILLISECONDS: case cudf::type_id::TIMESTAMP_MICROSECONDS: diff --git a/cpp/src/interop/to_arrow_schema.cpp b/cpp/src/interop/to_arrow_schema.cpp index b98ca8a7bed..5afed772656 100644 --- a/cpp/src/interop/to_arrow_schema.cpp +++ b/cpp/src/interop/to_arrow_schema.cpp @@ -170,8 +170,9 @@ int dispatch_to_arrow_type::operator()(column_view input, NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(out, NANOARROW_TYPE_LIST)); auto child = input.child(cudf::lists_column_view::child_column_index); ArrowSchemaInit(out->children[0]); - auto child_meta = - metadata.children_meta.empty() ? column_metadata{"element"} : metadata.children_meta[0]; + auto child_meta = metadata.children_meta.empty() + ? column_metadata{"element"} + : metadata.children_meta[cudf::lists_column_view::child_column_index]; out->flags = input.has_nulls() ? ARROW_FLAG_NULLABLE : 0; NANOARROW_RETURN_NOT_OK(ArrowSchemaSetName(out->children[0], child_meta.name.c_str())); diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 38b7e9ebe04..d32a2d8e3f8 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -64,9 +64,13 @@ rapids_cython_create_modules( target_link_libraries(strings_udf PUBLIC cudf_strings_udf) -set(targets_using_arrow_headers interop avro csv orc json parquet) +set(targets_using_arrow_headers avro csv orc json parquet) link_to_pyarrow_headers("${targets_using_arrow_headers}") +include(${rapids-cmake-dir}/export/find_package_root.cmake) +include(../../../../cpp/cmake/thirdparty/get_nanoarrow.cmake) +target_link_libraries(interop PUBLIC nanoarrow) + add_subdirectory(io) add_subdirectory(nvtext) add_subdirectory(pylibcudf) diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index df4591baa71..da32d530928 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -52,7 +52,10 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX pylibcudf_ ASSOCIATED_TARGETS cudf ) -link_to_pyarrow_headers(pylibcudf_interop) + +include(${rapids-cmake-dir}/export/find_package_root.cmake) +include(../../../../../cpp/cmake/thirdparty/get_nanoarrow.cmake) +target_link_libraries(pylibcudf_interop PUBLIC nanoarrow) add_subdirectory(libcudf) add_subdirectory(strings) diff --git a/python/cudf/cudf/_lib/pylibcudf/interop.pyx b/python/cudf/cudf/_lib/pylibcudf/interop.pyx index adf7e1fd7e8..caa19724786 100644 --- a/python/cudf/cudf/_lib/pylibcudf/interop.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/interop.pyx @@ -1,11 +1,10 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. -from cpython cimport pycapsule -from cython.operator cimport dereference -from libcpp.memory cimport shared_ptr, unique_ptr +from cpython.pycapsule cimport PyCapsule_GetPointer, PyCapsule_New +from libc.stdlib cimport free +from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector -from pyarrow cimport lib as pa from dataclasses import dataclass, field from functools import singledispatch @@ -18,23 +17,14 @@ from cudf._lib.pylibcudf.libcudf.interop cimport ( ArrowArrayStream, ArrowSchema, column_metadata, - from_arrow as cpp_from_arrow, from_arrow_column as cpp_from_arrow_column, from_arrow_stream as cpp_from_arrow_stream, - to_arrow as cpp_to_arrow, -) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport ( - fixed_point_scalar, - scalar, + to_arrow_host_raw, + to_arrow_schema_raw, ) from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.wrappers.decimals cimport ( - decimal32, - decimal64, - decimal128, - scale_type, -) +from . cimport copying from .column cimport Column from .scalar cimport Scalar from .table cimport Table @@ -109,7 +99,9 @@ def from_arrow(pyarrow_object, *, DataType data_type=None): Union[Table, Scalar] The converted object of type corresponding to the input type in cudf. """ - raise TypeError("from_arrow only accepts Table and Scalar objects") + raise TypeError( + f"Unsupported type {type(pyarrow_object)} for conversion from arrow" + ) @from_arrow.register(pa.DataType) @@ -133,7 +125,7 @@ def _from_arrow_table(pyarrow_object, *, DataType data_type=None): raise ValueError("data_type may not be passed for tables") stream = pyarrow_object.__arrow_c_stream__() cdef ArrowArrayStream* c_stream = ( - pycapsule.PyCapsule_GetPointer(stream, "arrow_array_stream") + PyCapsule_GetPointer(stream, "arrow_array_stream") ) cdef unique_ptr[table] c_result @@ -146,54 +138,17 @@ def _from_arrow_table(pyarrow_object, *, DataType data_type=None): @from_arrow.register(pa.Scalar) def _from_arrow_scalar(pyarrow_object, *, DataType data_type=None): - cdef shared_ptr[pa.CScalar] arrow_scalar = pa.pyarrow_unwrap_scalar(pyarrow_object) - - cdef unique_ptr[scalar] c_result - with nogil: - c_result = move(cpp_from_arrow(dereference(arrow_scalar))) - - cdef Scalar result = Scalar.from_libcudf(move(c_result)) - - if result.type().id() != type_id.DECIMAL128: - if data_type is not None: - raise ValueError( - "dtype may not be passed for non-decimal types" - ) - return result - - if data_type is None: - raise ValueError( - "Decimal scalars must be constructed with a dtype" - ) - - cdef type_id tid = data_type.id() - - if tid == type_id.DECIMAL32: - result.c_obj.reset( - new fixed_point_scalar[decimal32]( - ( - result.c_obj.get() - ).value(), - scale_type(-pyarrow_object.type.scale), - result.c_obj.get().is_valid() - ) - ) - elif tid == type_id.DECIMAL64: - result.c_obj.reset( - new fixed_point_scalar[decimal64]( - ( - result.c_obj.get() - ).value(), - scale_type(-pyarrow_object.type.scale), - result.c_obj.get().is_valid() - ) - ) - elif tid != type_id.DECIMAL128: - raise ValueError( - "Decimal scalars may only be cast to decimals" - ) - - return result + if isinstance(pyarrow_object.type, pa.ListType) and pyarrow_object.as_py() is None: + # pyarrow doesn't correctly handle None values for list types, so + # we have to create this one manually. + # https://github.com/apache/arrow/issues/40319 + pa_array = pa.array([None], type=pyarrow_object.type) + else: + pa_array = pa.array([pyarrow_object]) + return copying.get_element( + from_arrow(pa_array, data_type=data_type), + 0, + ) @from_arrow.register(pa.Array) @@ -204,10 +159,10 @@ def _from_arrow_column(pyarrow_object, *, DataType data_type=None): schema, array = pyarrow_object.__arrow_c_array__() cdef ArrowSchema* c_schema = ( - pycapsule.PyCapsule_GetPointer(schema, "arrow_schema") + PyCapsule_GetPointer(schema, "arrow_schema") ) cdef ArrowArray* c_array = ( - pycapsule.PyCapsule_GetPointer(array, "arrow_array") + PyCapsule_GetPointer(array, "arrow_array") ) cdef unique_ptr[column] c_result @@ -238,7 +193,7 @@ def to_arrow(cudf_object, metadata=None): Union[pyarrow.Array, pyarrow.Table, pyarrow.Scalar] The converted object of type corresponding to the input type in PyArrow. """ - raise TypeError("to_arrow only accepts Table and Scalar objects") + raise TypeError(f"Unsupported type {type(cudf_object)} for conversion to arrow") @to_arrow.register(DataType) @@ -281,46 +236,83 @@ def _to_arrow_datatype(cudf_object, **kwargs): ) -@to_arrow.register(Table) -def _to_arrow_table(cudf_object, metadata=None): +cdef void _release_schema(object schema_capsule) noexcept: + """Release the ArrowSchema object stored in a PyCapsule.""" + cdef ArrowSchema* schema = PyCapsule_GetPointer( + schema_capsule, 'arrow_schema' + ) + if schema.release != NULL: + schema.release(schema) + + free(schema) + + +cdef void _release_array(object array_capsule) noexcept: + """Release the ArrowArray object stored in a PyCapsule.""" + cdef ArrowArray* array = PyCapsule_GetPointer( + array_capsule, 'arrow_array' + ) + if array.release != NULL: + array.release(array) + + free(array) + + +def _table_to_schema(Table tbl, metadata): if metadata is None: - metadata = [ColumnMetadata() for _ in range(len(cudf_object.columns()))] + metadata = [ColumnMetadata() for _ in range(len(tbl.columns()))] metadata = [ColumnMetadata(m) if isinstance(m, str) else m for m in metadata] - cdef vector[column_metadata] c_table_metadata - cdef shared_ptr[pa.CTable] c_table_result + + cdef vector[column_metadata] c_metadata + c_metadata.reserve(len(metadata)) for meta in metadata: - c_table_metadata.push_back(_metadata_to_libcudf(meta)) + c_metadata.push_back(_metadata_to_libcudf(meta)) + + cdef ArrowSchema* raw_schema_ptr with nogil: - c_table_result = move( - cpp_to_arrow((
cudf_object).view(), c_table_metadata) - ) + raw_schema_ptr = to_arrow_schema_raw(tbl.view(), c_metadata) - return pa.pyarrow_wrap_table(c_table_result) + return PyCapsule_New(raw_schema_ptr, 'arrow_schema', _release_schema) -@to_arrow.register(Scalar) -def _to_arrow_scalar(cudf_object, metadata=None): - # Note that metadata for scalars is primarily important for preserving - # information on nested types since names are otherwise irrelevant. - if metadata is None: - metadata = ColumnMetadata() - metadata = ColumnMetadata(metadata) if isinstance(metadata, str) else metadata - cdef column_metadata c_scalar_metadata = _metadata_to_libcudf(metadata) - cdef shared_ptr[pa.CScalar] c_scalar_result +def _table_to_host_array(Table tbl): + cdef ArrowArray* raw_host_array_ptr with nogil: - c_scalar_result = move( - cpp_to_arrow( - dereference(( cudf_object).c_obj), c_scalar_metadata - ) - ) + raw_host_array_ptr = to_arrow_host_raw(tbl.view()) + + return PyCapsule_New(raw_host_array_ptr, "arrow_array", _release_array) + + +class _TableWithArrowMetadata: + def __init__(self, tbl, metadata=None): + self.tbl = tbl + self.metadata = metadata - return pa.pyarrow_wrap_scalar(c_scalar_result) + def __arrow_c_array__(self, requested_schema=None): + return _table_to_schema(self.tbl, self.metadata), _table_to_host_array(self.tbl) + + +# TODO: In the long run we should get rid of the `to_arrow` functions in favor of using +# the protocols directly via `pa.table(cudf_object, schema=...)` directly. We can do the +# same for columns. We cannot do this for scalars since there is no corresponding +# protocol. Since this will require broader changes throughout the codebase, the current +# approach is to leverage the protocol internally but to continue exposing `to_arrow`. +@to_arrow.register(Table) +def _to_arrow_table(cudf_object, metadata=None): + test_table = _TableWithArrowMetadata(cudf_object, metadata) + return pa.table(test_table) @to_arrow.register(Column) def _to_arrow_array(cudf_object, metadata=None): """Create a PyArrow array from a pylibcudf column.""" - if metadata is None: - metadata = ColumnMetadata() - metadata = ColumnMetadata(metadata) if isinstance(metadata, str) else metadata - return to_arrow(Table([cudf_object]), [metadata])[0] + if metadata is not None: + metadata = [metadata] + return to_arrow(Table([cudf_object]), metadata)[0] + + +@to_arrow.register(Scalar) +def _to_arrow_scalar(cudf_object, metadata=None): + # Note that metadata for scalars is primarily important for preserving + # information on nested types since names are otherwise irrelevant. + return to_arrow(Column.from_scalar(cudf_object, 1), metadata=metadata)[0] diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/interop.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/interop.pxd index 2151da28d4b..24d96b602dc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/interop.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/interop.pxd @@ -3,11 +3,11 @@ from libcpp.memory cimport shared_ptr, unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector -from pyarrow.lib cimport CScalar, CTable from cudf._lib.types import cudf_to_np_types, np_to_cudf_types from cudf._lib.pylibcudf.libcudf.column.column cimport column +from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar from cudf._lib.pylibcudf.libcudf.table.table cimport table from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view @@ -29,6 +29,9 @@ cdef extern from "cudf/interop.hpp" nogil: cdef struct ArrowArrayStream: void (*release)(ArrowArrayStream*) noexcept nogil + cdef struct ArrowDeviceArray: + ArrowArray array + cdef extern from "cudf/interop.hpp" namespace "cudf" \ nogil: @@ -38,27 +41,49 @@ cdef extern from "cudf/interop.hpp" namespace "cudf" \ DLManagedTensor* to_dlpack(table_view input_table ) except + - cdef unique_ptr[table] from_arrow(CTable input) except + - cdef unique_ptr[scalar] from_arrow(CScalar input) except + - cdef cppclass column_metadata: column_metadata() except + column_metadata(string name_) except + string name vector[column_metadata] children_meta - cdef shared_ptr[CTable] to_arrow( - table_view input, - vector[column_metadata] metadata, - ) except + - - cdef shared_ptr[CScalar] to_arrow( - const scalar& input, - column_metadata metadata, - ) except + - cdef unique_ptr[table] from_arrow_stream(ArrowArrayStream* input) except + cdef unique_ptr[column] from_arrow_column( const ArrowSchema* schema, const ArrowArray* input ) except + + + +cdef extern from *: + # Rather than exporting the underlying functions directly to Cython, we expose + # these wrappers that handle the release to avoid needing to teach Cython how + # to handle unique_ptrs with custom deleters that aren't default constructible. + # This will go away once we introduce cudf::arrow_column (need a + # cudf::arrow_schema as well), see + # https://github.com/rapidsai/cudf/issues/16104. + """ + #include + #include + + ArrowSchema* to_arrow_schema_raw( + cudf::table_view const& input, + cudf::host_span metadata) { + return to_arrow_schema(input, metadata).release(); + } + + ArrowArray* to_arrow_host_raw( + cudf::table_view const& tbl, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) { + // Assumes the sync event is null and the data is already on the host. + ArrowArray *arr = new ArrowArray(); + auto device_arr = cudf::to_arrow_host(tbl, stream, mr); + ArrowArrayMove(&device_arr->array, arr); + return arr; + } + """ + cdef ArrowSchema *to_arrow_schema_raw( + const table_view& tbl, + const vector[column_metadata]& metadata, + ) except + nogil + cdef ArrowArray* to_arrow_host_raw(const table_view& tbl) except + nogil diff --git a/python/cudf/cudf/pylibcudf_tests/common/utils.py b/python/cudf/cudf/pylibcudf_tests/common/utils.py index e19ff58927f..acb2b5be85c 100644 --- a/python/cudf/cudf/pylibcudf_tests/common/utils.py +++ b/python/cudf/cudf/pylibcudf_tests/common/utils.py @@ -44,7 +44,7 @@ def metadata_from_arrow_type( def assert_column_eq( lhs: pa.Array | plc.Column, rhs: pa.Array | plc.Column, - check_field_nullability=True, + check_field_nullability=False, ) -> None: """Verify that a pylibcudf array and PyArrow array are equal. @@ -59,7 +59,9 @@ def assert_column_eq( on child fields are equal. Useful for checking roundtripping of lossy formats like JSON that may not - preserve this information. + preserve this information. Also, our Arrow interop functions make different + choices by default than pyarrow field constructors since the interop functions + may make data-dependent choices. """ # Nested types require children metadata to be passed to the conversion function. if isinstance(lhs, (pa.Array, pa.ChunkedArray)) and isinstance( From 1c63e1ee31a07fb4999d7356919280ba3d528741 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Thu, 15 Aug 2024 21:51:47 -0400 Subject: [PATCH 071/270] Initial investigation into NumPy proxying in `cudf.pandas` (#16286) Apart of #15397. Closes #14537. Creates `ProxyNDarray` which inherits from `np.ndarray`. Authors: - Matthew Murray (https://github.com/Matt711) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16286 --- python/cudf/cudf/pandas/_wrappers/numpy.py | 3 +++ python/cudf/cudf/pandas/fast_slow_proxy.py | 20 +++++++++++++++- python/cudf/cudf/pandas/proxy_base.py | 23 +++++++++++++++++++ .../cudf_pandas_tests/test_cudf_pandas.py | 8 +++++++ 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 python/cudf/cudf/pandas/proxy_base.py diff --git a/python/cudf/cudf/pandas/_wrappers/numpy.py b/python/cudf/cudf/pandas/_wrappers/numpy.py index 3b012169676..eabea9713f1 100644 --- a/python/cudf/cudf/pandas/_wrappers/numpy.py +++ b/python/cudf/cudf/pandas/_wrappers/numpy.py @@ -14,6 +14,7 @@ make_final_proxy_type, make_intermediate_proxy_type, ) +from ..proxy_base import ProxyNDarrayBase from .common import ( array_interface, array_method, @@ -111,12 +112,14 @@ def wrap_ndarray(cls, arr: cupy.ndarray | numpy.ndarray, constructor): numpy.ndarray, fast_to_slow=cupy.ndarray.get, slow_to_fast=cupy.asarray, + bases=(ProxyNDarrayBase,), additional_attributes={ "__array__": array_method, # So that pa.array(wrapped-numpy-array) works "__arrow_array__": arrow_array_method, "__cuda_array_interface__": cuda_array_interface, "__array_interface__": array_interface, + "__array_ufunc__": _FastSlowAttribute("__array_ufunc__"), # ndarrays are unhashable "__hash__": None, # iter(cupy-array) produces an iterable of zero-dim device diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index bb678fd1efe..61aa6310082 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -19,6 +19,7 @@ from ..options import _env_get_bool from ..testing import assert_eq from .annotation import nvtx +from .proxy_base import ProxyNDarrayBase def call_operator(fn, args, kwargs): @@ -564,7 +565,11 @@ def _fsproxy_wrap(cls, value, func): _FinalProxy subclasses can override this classmethod if they need particular behaviour when wrapped up. """ - proxy = object.__new__(cls) + base_class = _get_proxy_base_class(cls) + if base_class is object: + proxy = base_class.__new__(cls) + else: + proxy = base_class.__new__(cls, value) proxy._fsproxy_wrapped = value return proxy @@ -1193,6 +1198,19 @@ def is_proxy_object(obj: Any) -> bool: return False +def _get_proxy_base_class(cls): + """Returns the proxy base class if one exists""" + for proxy_class in PROXY_BASE_CLASSES: + if proxy_class in cls.__mro__: + return proxy_class + return object + + +PROXY_BASE_CLASSES: set[type] = { + ProxyNDarrayBase, +} + + NUMPY_TYPES: set[str] = set(np.sctypeDict.values()) diff --git a/python/cudf/cudf/pandas/proxy_base.py b/python/cudf/cudf/pandas/proxy_base.py new file mode 100644 index 00000000000..61d9cde127c --- /dev/null +++ b/python/cudf/cudf/pandas/proxy_base.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import cupy as cp +import numpy as np + + +class ProxyNDarrayBase(np.ndarray): + def __new__(cls, arr): + if isinstance(arr, cp.ndarray): + obj = np.asarray(arr.get()).view(cls) + return obj + elif isinstance(arr, np.ndarray): + obj = np.asarray(arr).view(cls) + return obj + else: + raise TypeError( + "Unsupported array type. Must be numpy.ndarray or cupy.ndarray" + ) + + def __array_finalize__(self, obj): + self._fsproxy_wrapped = getattr(obj, "_fsproxy_wrapped", None) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 6292022d8e4..e5483fff913 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1632,3 +1632,11 @@ def test_change_index_name(index): assert s.index.name == name assert df.index.name == name + + +def test_numpy_ndarray_isinstancecheck(series): + s1, s2 = series + arr1 = s1.values + arr2 = s2.values + assert isinstance(arr1, np.ndarray) + assert isinstance(arr2, np.ndarray) From e690d9d25b4fadbd553f7ef14ac4918e95d98b0e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:48:49 -1000 Subject: [PATCH 072/270] Ensure size is always passed to NumericalColumn (#16576) https://github.com/rapidsai/cudf/pull/16457 requires `NumericalColumn` to be constructed with `size`. It appears another PR got in after this PR was created so there are currently a few usages where `size` isn't passed in. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16576 --- python/cudf/cudf/core/_internals/where.py | 4 +-- python/cudf/cudf/core/column/categorical.py | 37 +++++---------------- python/cudf/cudf/core/column/column.py | 1 + python/cudf/cudf/core/column/numerical.py | 1 + python/cudf/cudf/core/dataframe.py | 5 +-- python/cudf/cudf/core/index.py | 1 + 6 files changed, 13 insertions(+), 36 deletions(-) diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 9f36499586b..0c754317185 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -110,9 +110,7 @@ def _make_categorical_like(result, column): if isinstance(column, cudf.core.column.CategoricalColumn): result = cudf.core.column.build_categorical_column( categories=column.categories, - codes=cudf.core.column.NumericalColumn( - result.base_data, dtype=result.dtype - ), + codes=result, mask=result.base_mask, size=result.size, offset=result.offset, diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index d25983842f9..66aed38bffd 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -659,10 +659,7 @@ def slice(self, start: int, stop: int, stride: int | None = None) -> Self: Self, cudf.core.column.build_categorical_column( categories=self.categories, - codes=cudf.core.column.NumericalColumn( - codes.base_data, # type: ignore[arg-type] - dtype=codes.dtype, - ), + codes=codes, mask=codes.base_mask, ordered=self.ordered, size=codes.size, @@ -734,10 +731,7 @@ def sort_values( codes = self.codes.sort_values(ascending, na_position) col = column.build_categorical_column( categories=self.dtype.categories._values, - codes=cudf.core.column.NumericalColumn( - codes.base_data, # type: ignore[arg-type] - dtype=codes.dtype, - ), + codes=codes, mask=codes.base_mask, size=codes.size, ordered=self.dtype.ordered, @@ -845,10 +839,7 @@ def unique(self) -> CategoricalColumn: codes = self.codes.unique() return column.build_categorical_column( categories=self.categories, - codes=cudf.core.column.NumericalColumn( - codes.base_data, # type: ignore[arg-type] - dtype=codes.dtype, - ), + codes=codes, mask=codes.base_mask, offset=codes.offset, size=codes.size, @@ -986,9 +977,7 @@ def find_and_replace( result = column.build_categorical_column( categories=new_cats["cats"], - codes=cudf.core.column.NumericalColumn( - output.base_data, dtype=output.dtype - ), + codes=output, mask=output.base_mask, offset=output.offset, size=output.size, @@ -1184,10 +1173,7 @@ def _concat( return column.build_categorical_column( categories=column.as_column(cats), - codes=cudf.core.column.NumericalColumn( - codes_col.base_data, # type: ignore[arg-type] - dtype=codes_col.dtype, - ), + codes=codes_col, mask=codes_col.base_mask, size=codes_col.size, offset=codes_col.offset, @@ -1199,10 +1185,7 @@ def _with_type_metadata( if isinstance(dtype, CategoricalDtype): return column.build_categorical_column( categories=dtype.categories._values, - codes=cudf.core.column.NumericalColumn( - self.codes.base_data, # type: ignore[arg-type] - dtype=self.codes.dtype, - ), + codes=self.codes, mask=self.codes.base_mask, ordered=dtype.ordered, size=self.codes.size, @@ -1345,9 +1328,7 @@ def _set_categories( Self, column.build_categorical_column( categories=new_cats, - codes=cudf.core.column.NumericalColumn( - new_codes.base_data, dtype=new_codes.dtype - ), + codes=new_codes, mask=new_codes.base_mask, size=new_codes.size, offset=new_codes.offset, @@ -1478,9 +1459,7 @@ def pandas_categorical_as_column( return column.build_categorical_column( categories=categorical.categories, - codes=cudf.core.column.NumericalColumn( - codes.base_data, dtype=codes.dtype - ), + codes=codes, size=codes.size, mask=mask, ordered=categorical.ordered, diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index b0e33e8b9ce..090c02da990 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1513,6 +1513,7 @@ def column_empty( * cudf.dtype(libcudf.types.size_type_dtype).itemsize ) ), + size=None, dtype=libcudf.types.size_type_dtype, ), ) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 16e78ef35ef..ac36813202a 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -654,6 +654,7 @@ def _with_type_metadata(self: ColumnBase, dtype: Dtype) -> ColumnBase: categories=dtype.categories._values, codes=cudf.core.column.NumericalColumn( self.base_data, # type: ignore[arg-type] + self.size, dtype=self.dtype, ), mask=self.base_mask, diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 3033abd53f5..f935217f4f9 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -46,7 +46,6 @@ from cudf.core.column import ( CategoricalColumn, ColumnBase, - NumericalColumn, StructColumn, as_column, build_categorical_column, @@ -8541,9 +8540,7 @@ def _reassign_categories(categories, cols, col_idxs): if idx in categories: cols[name] = build_categorical_column( categories=categories[idx], - codes=NumericalColumn( - cols[name].base_data, dtype=cols[name].dtype - ), + codes=cols[name], mask=cols[name].base_mask, offset=cols[name].offset, size=cols[name].size, diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index d02633a97fa..ee2f0317f8d 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2501,6 +2501,7 @@ def _get_dt_field(self, field: str) -> Index: out_column = self._column.get_dt_field(field) out_column = NumericalColumn( data=out_column.base_data, + size=out_column.size, dtype=out_column.dtype, mask=out_column.base_mask, offset=out_column.offset, From e197d72f2daafb2f4804f823019b1ca7810ed560 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Fri, 16 Aug 2024 09:45:30 -0700 Subject: [PATCH 073/270] Replace `NativeFile` dependency in dask-cudf Parquet reader (#16569) Replaces `read_parquet` logic that currently depends on `NativeFile` for remote-storage access. **NOTE**: ~It is possible to remove `NativeFile` usage without adding the new `_prefetch_remote_buffers` logic.~ ~However, I'd like to replace the cudf data-transfer logic soon anyway.~ Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/cudf/pull/16569 --- python/dask_cudf/dask_cudf/backends.py | 21 ++++ python/dask_cudf/dask_cudf/io/parquet.py | 102 +++++++----------- .../dask_cudf/dask_cudf/io/tests/test_s3.py | 64 +++++++---- 3 files changed, 101 insertions(+), 86 deletions(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 82ea2ac033a..a65ae819b44 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -498,6 +498,25 @@ def _unsupported_kwargs(old, new, kwargs): ) +def _raise_unsupported_parquet_kwargs( + open_file_options=None, filesystem=None, **kwargs +): + import fsspec + + if open_file_options is not None: + raise ValueError( + "The open_file_options argument is no longer supported " + "by the 'cudf' backend." + ) + + if filesystem not in ("fsspec", None) and not isinstance( + filesystem, fsspec.AbstractFileSystem + ): + raise ValueError( + f"filesystem={filesystem} is not supported by the 'cudf' backend." + ) + + # Register cudf->pandas to_pandas_dispatch = PandasBackendEntrypoint.to_backend_dispatch() @@ -573,6 +592,7 @@ def from_dict( def read_parquet(*args, engine=None, **kwargs): from dask_cudf.io.parquet import CudfEngine + _raise_unsupported_parquet_kwargs(**kwargs) return _default_backend( dd.read_parquet, *args, @@ -665,6 +685,7 @@ def read_parquet(*args, engine=None, **kwargs): from dask_cudf.io.parquet import CudfEngine + _raise_unsupported_parquet_kwargs(**kwargs) return _default_backend( dx.read_parquet, *args, engine=CudfEngine, **kwargs ) diff --git a/python/dask_cudf/dask_cudf/io/parquet.py b/python/dask_cudf/dask_cudf/io/parquet.py index f0cab953458..8f52fce7818 100644 --- a/python/dask_cudf/dask_cudf/io/parquet.py +++ b/python/dask_cudf/dask_cudf/io/parquet.py @@ -1,7 +1,6 @@ # Copyright (c) 2019-2024, NVIDIA CORPORATION. import itertools import warnings -from contextlib import ExitStack from functools import partial from io import BufferedWriter, BytesIO, IOBase @@ -22,18 +21,13 @@ import cudf from cudf.core.column import as_column, build_categorical_column from cudf.io import write_to_dataset -from cudf.io.parquet import ( - _apply_post_filters, - _default_open_file_options, - _normalize_filters, -) +from cudf.io.parquet import _apply_post_filters, _normalize_filters from cudf.utils.dtypes import cudf_dtype_from_pa_type from cudf.utils.ioutils import ( _ROW_GROUP_SIZE_BYTES_DEFAULT, + _fsspec_data_transfer, _is_local_filesystem, - _open_remote_files, ) -from cudf.utils.utils import maybe_filter_deprecation class CudfEngine(ArrowDatasetEngine): @@ -98,63 +92,45 @@ def _read_paths( dataset_kwargs = dataset_kwargs or {} dataset_kwargs["partitioning"] = partitioning or "hive" - with ExitStack() as stack: - # Non-local filesystem handling - paths_or_fobs = paths - if not _is_local_filesystem(fs): - paths_or_fobs = _open_remote_files( - paths_or_fobs, - fs, - context_stack=stack, - **_default_open_file_options( - open_file_options, columns, row_groups - ), - ) - # Filter out deprecation warning unless the user - # specifies open_file_options and/or use_python_file_object. - # Otherwise, the FutureWarning is out of their control. - with maybe_filter_deprecation( - ( - not open_file_options - and "use_python_file_object" not in kwargs - ), - message="Support for reading pyarrow's NativeFile is deprecated", - category=FutureWarning, - ): - # Use cudf to read in data - try: - df = cudf.read_parquet( - paths_or_fobs, - engine="cudf", - columns=columns, - row_groups=row_groups if row_groups else None, - dataset_kwargs=dataset_kwargs, - categorical_partitions=False, - **kwargs, - ) - except RuntimeError as err: - # TODO: Remove try/except after null-schema issue is resolved - # (See: https://github.com/rapidsai/cudf/issues/12702) - if len(paths_or_fobs) > 1: - df = cudf.concat( - [ - cudf.read_parquet( - pof, - engine="cudf", - columns=columns, - row_groups=row_groups[i] - if row_groups - else None, - dataset_kwargs=dataset_kwargs, - categorical_partitions=False, - **kwargs, - ) - for i, pof in enumerate(paths_or_fobs) - ] + # Non-local filesystem handling + paths_or_fobs = paths + if not _is_local_filesystem(fs): + paths_or_fobs = [ + _fsspec_data_transfer(fpath, fs=fs) for fpath in paths + ] + + # Use cudf to read in data + try: + df = cudf.read_parquet( + paths_or_fobs, + engine="cudf", + columns=columns, + row_groups=row_groups if row_groups else None, + dataset_kwargs=dataset_kwargs, + categorical_partitions=False, + **kwargs, + ) + except RuntimeError as err: + # TODO: Remove try/except after null-schema issue is resolved + # (See: https://github.com/rapidsai/cudf/issues/12702) + if len(paths_or_fobs) > 1: + df = cudf.concat( + [ + cudf.read_parquet( + pof, + engine="cudf", + columns=columns, + row_groups=row_groups[i] if row_groups else None, + dataset_kwargs=dataset_kwargs, + categorical_partitions=False, + **kwargs, ) - else: - raise err + for i, pof in enumerate(paths_or_fobs) + ] + ) + else: + raise err # Apply filters (if any are defined) df = _apply_post_filters(df, filters) diff --git a/python/dask_cudf/dask_cudf/io/tests/test_s3.py b/python/dask_cudf/dask_cudf/io/tests/test_s3.py index ac3245b3748..99f19917424 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_s3.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_s3.py @@ -5,8 +5,8 @@ from contextlib import contextmanager from io import BytesIO +import fsspec import pandas as pd -import pyarrow.fs as pa_fs import pytest from dask.dataframe import assert_eq @@ -135,35 +135,53 @@ def test_read_csv_warns(s3_base, s3so): assert df.a.sum().compute() == 4 -@pytest.mark.parametrize( - "open_file_options", - [ - {"precache_options": {"method": None}}, - {"precache_options": {"method": "parquet"}}, - {"open_file_func": None}, - ], -) -def test_read_parquet_open_file_options(s3_base, s3so, open_file_options, pdf): +def test_read_parquet_open_file_options_raises(): + with pytest.raises(ValueError): + dask_cudf.read_parquet( + "s3://my/path", + open_file_options={"precache_options": {"method": "parquet"}}, + ) + + +def test_read_parquet_filesystem(s3_base, s3so, pdf): + fname = "test_parquet_filesystem.parquet" + bucket = "parquet" buffer = BytesIO() pdf.to_parquet(path=buffer) buffer.seek(0) - with s3_context( - s3_base=s3_base, bucket="daskparquet", files={"file.parq": buffer} - ): - if "open_file_func" in open_file_options: - fs = pa_fs.S3FileSystem( - endpoint_override=s3so["client_kwargs"]["endpoint_url"], + with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): + path = f"s3://{bucket}/{fname}" + + # Cannot pass filesystem="arrow" + with pytest.raises(ValueError): + dask_cudf.read_parquet( + path, + storage_options=s3so, + filesystem="arrow", ) - open_file_options["open_file_func"] = fs.open_input_file + + # Can pass filesystem="fsspec" df = dask_cudf.read_parquet( - "s3://daskparquet/*.parq", + path, storage_options=s3so, - open_file_options=open_file_options, + filesystem="fsspec", ) - with pytest.warns(FutureWarning): - assert df.a.sum().compute() == 10 - with pytest.warns(FutureWarning): - assert df.b.sum().compute() == 9 + assert df.b.sum().compute() == 9 + + +def test_read_parquet_filesystem_explicit(s3_base, s3so, pdf): + fname = "test_parquet_filesystem_explicit.parquet" + bucket = "parquet" + buffer = BytesIO() + pdf.to_parquet(path=buffer) + buffer.seek(0) + with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): + path = f"s3://{bucket}/{fname}" + fs = fsspec.core.get_fs_token_paths( + path, mode="rb", storage_options=s3so + )[0] + df = dask_cudf.read_parquet(path, filesystem=fs) + assert df.b.sum().compute() == 9 def test_read_parquet(s3_base, s3so, pdf): From 623dfceb42eb3e73b352b295898ff3e6cfe7c865 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Fri, 16 Aug 2024 12:50:23 -0400 Subject: [PATCH 074/270] [FEA] Add support for `cudf.unique` (#16554) closes #16460 Authors: - Matthew Murray (https://github.com/Matt711) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16554 --- python/cudf/cudf/__init__.py | 2 +- python/cudf/cudf/core/algorithms.py | 122 ++++++++++++++++++++++++++ python/cudf/cudf/tests/test_unique.py | 117 ++++++++++++++++++++++++ 3 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 python/cudf/cudf/tests/test_unique.py diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index 77ae0791b81..ccc45413de4 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -24,7 +24,7 @@ register_series_accessor, ) from cudf.api.types import dtype -from cudf.core.algorithms import factorize +from cudf.core.algorithms import factorize, unique from cudf.core.cut import cut from cudf.core.dataframe import DataFrame, from_dataframe, from_pandas, merge from cudf.core.dtypes import ( diff --git a/python/cudf/cudf/core/algorithms.py b/python/cudf/cudf/core/algorithms.py index e27d6ec8d3e..b28fce6d343 100644 --- a/python/cudf/cudf/core/algorithms.py +++ b/python/cudf/cudf/core/algorithms.py @@ -7,6 +7,7 @@ import cupy as cp import numpy as np +import cudf from cudf.core.column import as_column from cudf.core.index import Index, RangeIndex from cudf.core.scalar import Scalar @@ -145,3 +146,124 @@ def _interpolation(column: ColumnBase, index: BaseIndex) -> ColumnBase: first_nan_idx = valid_locs.values.argmax().item() result[:first_nan_idx] = np.nan return as_column(result) + + +def unique(values): + """ + Return unique values from array-like + + Parameters + ---------- + values : 1d array-like + + Returns + ------- + cudf.Series, + + The return can be: + + * Index : when the input is an Index + * cudf.Series : when the input is a Series + * cupy.ndarray : when the input is a cupy.ndarray + + Return cudf.Series, cudf.Index, or cupy.ndarray. + + See Also + -------- + Index.unique : Return unique values from an Index. + Series.unique : Return unique values of Series object. + + Examples + -------- + >>> cudf.unique(cudf.Series([2, 1, 3, 3])) + 0 2 + 1 1 + 2 3 + dtype: int64 + + >>> cudf.unique(cudf.Series([2] + [1] * 5)) + 0 2 + 1 1 + dtype: int64 + + >>> cudf.unique(cudf.Series([pd.Timestamp("20160101"), pd.Timestamp("20160101")])) + 0 2016-01-01 + dtype: datetime64[ns] + + >>> cudf.unique( + ... cudf.Series( + ... [ + ... pd.Timestamp("20160101", tz="US/Eastern"), + ... pd.Timestamp("20160101", tz="US/Eastern"), + ... pd.Timestamp("20160103", tz="US/Eastern"), + ... ] + ... ) + ... ) + 0 2016-01-01 00:00:00-05:00 + 1 2016-01-03 00:00:00-05:00 + dtype: datetime64[ns, US/Eastern] + + >>> cudf.unique( + ... cudf.Index( + ... [ + ... pd.Timestamp("20160101", tz="US/Eastern"), + ... pd.Timestamp("20160101", tz="US/Eastern"), + ... pd.Timestamp("20160103", tz="US/Eastern"), + ... ] + ... ) + ... ) + DatetimeIndex(['2016-01-01 00:00:00-05:00', '2016-01-03 00:00:00-05:00'],dtype='datetime64[ns, US/Eastern]') + + An unordered Categorical will return categories in the + order of appearance. + + >>> cudf.unique(cudf.Series(pd.Categorical(list("baabc")))) + 0 b + 1 a + 2 c + dtype: category + Categories (3, object): ['a', 'b', 'c'] + + >>> cudf.unique(cudf.Series(pd.Categorical(list("baabc"), categories=list("abc")))) + 0 b + 1 a + 2 c + dtype: category + Categories (3, object): ['a', 'b', 'c'] + + An ordered Categorical preserves the category ordering. + + >>> pd.unique( + ... pd.Series( + ... pd.Categorical(list("baabc"), categories=list("abc"), ordered=True) + ... ) + ... ) + 0 b + 1 a + 2 c + dtype: category + Categories (3, object): ['a' < 'b' < 'c'] + + An array of tuples + + >>> pd.unique(pd.Series([("a", "b"), ("b", "a"), ("a", "c"), ("b", "a")]).values) + array([('a', 'b'), ('b', 'a'), ('a', 'c')], dtype=object) + """ + if not isinstance(values, (cudf.Series, cudf.Index, cp.ndarray)): + raise ValueError( + "Must pass cudf.Series, cudf.Index, or cupy.ndarray object" + ) + if isinstance(values, cp.ndarray): + # pandas.unique will not sort the values in the result + # while cupy.unique documents it will, so we pass cupy.ndarray + # through cudf.Index to maintain the original order. + return cp.asarray(cudf.Index(values).unique()) + if isinstance(values, cudf.Series): + if get_option("mode.pandas_compatible"): + if isinstance(values.dtype, cudf.CategoricalDtype): + raise NotImplementedError( + "cudf.Categorical is not implemented" + ) + else: + return cp.asarray(values.unique()) + return values.unique() diff --git a/python/cudf/cudf/tests/test_unique.py b/python/cudf/cudf/tests/test_unique.py new file mode 100644 index 00000000000..699b3340521 --- /dev/null +++ b/python/cudf/cudf/tests/test_unique.py @@ -0,0 +1,117 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import cupy as cp +import numpy as np +import pandas as pd +import pytest + +import cudf +from cudf.testing import assert_eq + + +@pytest.fixture +def df(): + df = cudf.DataFrame() + np.random.seed(0) + + arr = np.random.randint(2, size=10, dtype=np.int64) + df["foo"] = arr + df["bar"] = cudf.Series([pd.Timestamp(x) for x in arr]) + + return df + + +@pytest.fixture(params=["foo", "bar"]) +def series_test_vals(request, df): + actual = cudf.unique(df[request.param]) + expected = pd.unique(df[request.param].to_pandas()) + return actual, expected + + +def test_unique_series_obj(series_test_vals): + actual, expected = series_test_vals + + assert isinstance(expected, np.ndarray) + assert isinstance(actual, cudf.Series) + assert_eq(actual, pd.Series(expected, name=actual.name)) + + +@pytest.mark.parametrize( + "index", + [ + (cudf.Index, pd.Index), + (cudf.MultiIndex, pd.MultiIndex), + (cudf.DatetimeIndex, pd.DatetimeIndex), + (cudf.CategoricalIndex, pd.CategoricalIndex), + ], +) +@pytest.mark.parametrize("col", ["foo", "bar"]) +def test_unique_index_obj(index, col, df): + if index[0] == cudf.MultiIndex: + df.index = cudf.MultiIndex.from_arrays([df[col], df[col]]) + else: + df.index = index[0](df[col]) + actual = cudf.unique(df.index) + expected = pd.unique(df.index.to_pandas()) + + isinstance(expected, np.ndarray) + assert isinstance(actual, index[0]) + + if index[0] == cudf.MultiIndex: + expect = index[1].from_arrays( + [ + [x[0] for x in expected], + [x[1] for x in expected], + ], + names=actual.names, + ) + assert_eq(actual, expect) + else: + assert_eq(actual, index[1](expected, name=actual.name)) + + +def test_unique_cupy_ndarray(df): + arr = np.asarray(df["foo"].to_pandas()) + garr = cp.asarray(df["foo"]) + + expected = pd.unique(arr) + actual = cudf.unique(garr) + + isinstance(expected, np.ndarray) + isinstance(actual, cp.ndarray) + assert_eq(actual, expected) + + +@pytest.mark.parametrize( + "data", + [ + ["abc", "def", "abc", "a", "def", None], + [10, 20, 100, -10, 0, 1, None, 10, 100], + ], +) +def test_category_dtype_unique(data): + gs = cudf.Series(data, dtype="category") + ps = gs.to_pandas() + + actual = cudf.unique(gs) + expected = pd.unique(ps) + + assert isinstance(expected, pd.Categorical) + assert isinstance(actual, cudf.Series) + assert_eq(actual, pd.Series(expected)) + + +def test_unique_fails_value_error(df): + with pytest.raises( + ValueError, + match="Must pass cudf.Series, cudf.Index, or cupy.ndarray object", + ): + cudf.unique(df) + + +def test_unique_fails_not_implemented_error(df): + with cudf.option_context("mode.pandas_compatible", True): + with pytest.raises( + NotImplementedError, match="cudf.Categorical is not implemented" + ): + cudf.unique(cudf.Series(["foo", "foo"], dtype="category")) From e16c2f2493d316259dc2472b448e61b6e717b7dd Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 16 Aug 2024 07:17:40 -1000 Subject: [PATCH 075/270] Make (Indexed)Frame.__init__ require data (and index) (#16430) This PR makes `data` and `Index` required arguments of `Frame` and `IndexedFrame` where relevant so we can gradually move towards ensuring `data` and `index` are valid mapping of columns and a cudf Index respectively Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16430 --- python/cudf/cudf/core/dataframe.py | 2 +- python/cudf/cudf/core/frame.py | 8 ++------ python/cudf/cudf/core/indexed_frame.py | 16 +++++++++------- python/cudf/cudf/core/reshape.py | 2 +- python/cudf/cudf/core/series.py | 2 +- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index f935217f4f9..3d805881c5a 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -697,7 +697,7 @@ def __init__( ): if copy is not None: raise NotImplementedError("copy is not currently implemented.") - super().__init__() + super().__init__({}, index=cudf.Index([])) if nan_as_null is no_default: nan_as_null = not cudf.get_option("mode.pandas_compatible") diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 32c313e42d3..ce23d671a6c 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -53,14 +53,10 @@ class Frame(BinaryOperand, Scannable): A Frame representing the (optional) index columns. """ - _data: "ColumnAccessor" - _VALID_BINARY_OPERATIONS = BinaryOperand._SUPPORTED_BINARY_OPERATIONS - def __init__(self, data=None): - if data is None: - data = {} - self._data = cudf.core.column_accessor.ColumnAccessor(data) + def __init__(self, data: ColumnAccessor | MutableMapping[Any, ColumnBase]): + self._data = ColumnAccessor(data) @property def _num_columns(self) -> int: diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index ae7369c80d1..8eb6de79bce 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -265,7 +265,6 @@ class IndexedFrame(Frame): # mypy can't handle bound type variables as class members _loc_indexer_type: type[_LocIndexerClass] # type: ignore _iloc_indexer_type: type[_IlocIndexerClass] # type: ignore - _index: cudf.core.index.BaseIndex _groupby = GroupBy _resampler = _Resampler @@ -284,18 +283,21 @@ class IndexedFrame(Frame): "cummax": {"op_name": "cumulative max"}, } - def __init__(self, data=None, index=None): + def __init__( + self, + data: ColumnAccessor | MutableMapping[Any, ColumnBase], + index: BaseIndex, + ): super().__init__(data=data) - # TODO: Right now it is possible to initialize an IndexedFrame without - # an index. The code's correctness relies on the subclass constructors - # assigning the attribute after the fact. We should restructure those - # to ensure that this constructor is always invoked with an index. + if not isinstance(index, cudf.core._base_index.BaseIndex): + raise ValueError( + f"index must be a cudf index not {type(index).__name__}" + ) self._index = index @property def _num_rows(self) -> int: # Important to use the index because the data may be empty. - # TODO: Remove once DataFrame.__init__ is cleaned up return len(self.index) @property diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index df471692702..703a239bea2 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -490,7 +490,7 @@ def concat( elif len(objs) == 1: obj = objs[0] result = cudf.DataFrame._from_data( - data=None if join == "inner" else obj._data.copy(deep=True), + data={} if join == "inner" else obj._data.copy(deep=True), index=cudf.RangeIndex(len(obj)) if ignore_index else obj.index.copy(deep=True), diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 2fb4fde6552..4be10752651 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -518,7 +518,7 @@ def from_categorical(cls, categorical, codes=None): @classmethod @_performance_tracking - def from_arrow(cls, array: pa.Array): + def from_arrow(cls, array: pa.Array) -> Self: """Create from PyArrow Array/ChunkedArray. Parameters From 30011c58ed2444f0a6ba9f80c17766e591a610a1 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 16 Aug 2024 07:19:54 -1000 Subject: [PATCH 076/270] Clean up reshaping ops (#16553) Uses some more "idiomatic" cudf patterns such as * Checking `isinstance(column.dtype, ...)` instead of `isinstance(column, ...)` (to avoid importing the column objects) * Using `DataFrame._from_data(dict)` instead of creating an empty `DataFrame` and adding columns one by one Also avoids some column materialization in `DataFrame.columns = `: * For `RangeIndex`, avoid materializing to a column to get a distinct count * For `MultiIndex`, avoid creating a `cudf.MultiIndex` with columns as it's converted to a CPU object to get column labels for the `ColumnAccessor` Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16553 --- python/cudf/cudf/core/dataframe.py | 8 +- python/cudf/cudf/core/reshape.py | 141 ++++++++++++++++------------- 2 files changed, 82 insertions(+), 67 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 3d805881c5a..6ee3d69441f 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -2654,8 +2654,12 @@ def columns(self, columns): elif isinstance(columns, (cudf.BaseIndex, ColumnBase, Series)): level_names = (getattr(columns, "name", None),) rangeindex = isinstance(columns, cudf.RangeIndex) - columns = as_column(columns) - if columns.distinct_count(dropna=False) != len(columns): + if rangeindex: + unique_count = len(columns) + else: + columns = as_column(columns) + unique_count = columns.distinct_count(dropna=False) + if unique_count != len(columns): raise ValueError("Duplicate column names are not allowed") pd_columns = pd.Index(columns.to_pandas()) label_dtype = pd_columns.dtype diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 703a239bea2..3d205957126 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -3,7 +3,7 @@ import itertools import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import numpy as np import pandas as pd @@ -14,7 +14,7 @@ from cudf.api.extensions import no_default from cudf.core._compat import PANDAS_LT_300 from cudf.core.column import ColumnBase, as_column, column_empty_like -from cudf.core.column.categorical import CategoricalColumn +from cudf.core.column_accessor import ColumnAccessor from cudf.utils.dtypes import min_unsigned_type if TYPE_CHECKING: @@ -101,7 +101,9 @@ def _get_combined_index(indexes, intersect: bool = False, sort=None): return index -def _normalize_series_and_dataframe(objs, axis): +def _normalize_series_and_dataframe( + objs: list[cudf.Series | cudf.DataFrame], axis: Literal[0, 1] +) -> None: """Convert any cudf.Series objects in objs to DataFrames in place.""" # Default to naming series by a numerical id if they are not named. sr_name = 0 @@ -335,7 +337,7 @@ def concat( result = obj.to_frame() else: result = obj.copy(deep=True) - result.columns = pd.RangeIndex(len(result._data)) + result.columns = cudf.RangeIndex(len(result._data)) else: result = type(obj)._from_data( data=obj._data.copy(deep=True), @@ -350,7 +352,7 @@ def concat( result = obj.copy(deep=True) if keys_objs is not None and isinstance(result, cudf.DataFrame): k = keys_objs[0] - result.columns = cudf.MultiIndex.from_tuples( + result.columns = pd.MultiIndex.from_tuples( [ (k, *c) if isinstance(c, tuple) else (k, c) for c in result._column_names @@ -369,7 +371,6 @@ def concat( raise TypeError( "Can only concatenate Series and DataFrame objects when axis=1" ) - df = cudf.DataFrame() _normalize_series_and_dataframe(objs, axis=axis) any_empty = any(obj.empty for obj in objs) @@ -393,18 +394,23 @@ def concat( objs = [obj for obj in objs if obj.shape != (0, 0)] if len(objs) == 0: - return df + # TODO: https://github.com/rapidsai/cudf/issues/16550 + return cudf.DataFrame() # Don't need to align indices of all `objs` since we # would anyway return an empty dataframe below if not empty_inner: objs = _align_objs(objs, how=join, sort=sort) - df.index = objs[0].index + result_index = objs[0].index + else: + result_index = None + result_data = {} + result_columns = None if keys_objs is None: for o in objs: for name, col in o._data.items(): - if name in df._data: + if name in result_data: raise NotImplementedError( f"A Column with duplicate name found: {name}, cuDF " f"doesn't support having multiple columns with " @@ -414,11 +420,11 @@ def concat( # if join is inner and it contains an empty df # we return an empty df, hence creating an empty # column with dtype metadata retained. - df[name] = cudf.core.column.column_empty_like( + result_data[name] = cudf.core.column.column_empty_like( col, newsize=0 ) else: - df[name] = col + result_data[name] = col result_columns = ( objs[0] @@ -451,21 +457,21 @@ def concat( else: col_label = (k, name) if empty_inner: - df[col_label] = cudf.core.column.column_empty_like( - col, newsize=0 + result_data[col_label] = ( + cudf.core.column.column_empty_like(col, newsize=0) ) else: - df[col_label] = col + result_data[col_label] = col - if keys_objs is None: - df.columns = result_columns.unique() - if ignore_index: - df.columns = cudf.RangeIndex(len(result_columns.unique())) - elif ignore_index: - # with ignore_index the column names change to numbers - df.columns = cudf.RangeIndex(len(result_columns)) + df = cudf.DataFrame._from_data( + ColumnAccessor(result_data, verify=False), index=result_index + ) + if ignore_index: + df.columns = cudf.RangeIndex(df._num_columns) + elif result_columns is not None: + df.columns = result_columns elif not only_series: - df.columns = cudf.MultiIndex.from_tuples(df._column_names) + df.columns = pd.MultiIndex.from_tuples(df._column_names) if empty_inner: # if join is inner and it contains an empty df @@ -486,6 +492,7 @@ def concat( if len(objs) == 0: # If objs is empty, that indicates all of # objs are empty dataframes. + # TODO: https://github.com/rapidsai/cudf/issues/16550 return cudf.DataFrame() elif len(objs) == 1: obj = objs[0] @@ -519,7 +526,7 @@ def concat( elif typ is cudf.MultiIndex: return cudf.MultiIndex._concat(objs) elif issubclass(typ, cudf.Index): - return cudf.core.index.Index._concat(objs) + return cudf.Index._concat(objs) else: raise TypeError(f"cannot concatenate object of type {typ}") @@ -632,18 +639,19 @@ def melt( value_vars = [c for c in frame._column_names if c not in unique_id] # Error for unimplemented support for datatype - dtypes = [frame[col].dtype for col in id_vars + value_vars] - if any(isinstance(typ, cudf.CategoricalDtype) for typ in dtypes): + if any( + isinstance(frame[col].dtype, cudf.CategoricalDtype) + for col in id_vars + value_vars + ): raise NotImplementedError( "Categorical columns are not yet supported for function" ) # Check dtype homogeneity in value_var # Because heterogeneous concat is unimplemented - dtypes = [frame[col].dtype for col in value_vars] - if len(dtypes) > 0: - dtype = dtypes[0] - if any(t != dtype for t in dtypes): + if len(value_vars) > 1: + dtype = frame[value_vars[0]].dtype + if any(frame[col].dtype != dtype for col in value_vars): raise ValueError("all cols in value_vars must have the same dtype") # overlap @@ -969,37 +977,39 @@ def _pivot(df, index, columns): index_labels, index_idx = index._encode() column_labels = columns_labels.to_pandas().to_flat_index() - def as_tuple(x): - return x if isinstance(x, tuple) else (x,) - result = {} - for v in df: - names = [as_tuple(v) + as_tuple(name) for name in column_labels] + if len(index_labels) != 0 and len(columns_labels) != 0: + + def as_tuple(x): + return x if isinstance(x, tuple) else (x,) + nrows = len(index_labels) - ncols = len(names) - num_elements = nrows * ncols - if num_elements > 0: - col = df._data[v] + for col_label, col in df._data.items(): + names = [ + as_tuple(col_label) + as_tuple(name) for name in column_labels + ] + new_size = nrows * len(names) scatter_map = (columns_idx * np.int32(nrows)) + index_idx - target = cudf.DataFrame._from_data( - { - None: cudf.core.column.column_empty_like( - col, masked=True, newsize=nrows * ncols - ) - } + target_col = cudf.core.column.column_empty_like( + col, masked=True, newsize=new_size ) - target._data[None][scatter_map] = col - result_frames = target._split(range(nrows, nrows * ncols, nrows)) + target_col[scatter_map] = col + target = cudf.Index._from_column(target_col) result.update( { - name: next(iter(f._columns)) - for name, f in zip(names, result_frames) + name: idx._column + for name, idx in zip( + names, target._split(range(nrows, new_size, nrows)) + ) } ) # the result of pivot always has a multicolumn - ca = cudf.core.column_accessor.ColumnAccessor( - result, multiindex=True, level_names=(None,) + columns._data.names + ca = ColumnAccessor( + result, + multiindex=True, + level_names=(None,) + columns._data.names, + verify=False, ) return cudf.DataFrame._from_data( ca, index=cudf.Index(index_labels, name=index.name) @@ -1070,19 +1080,20 @@ def pivot(data, columns=None, index=no_default, values=no_default): if index is no_default: index = df.index else: - index = cudf.core.index.Index(df.loc[:, index]) + index = cudf.Index(df.loc[:, index]) columns = cudf.Index(df.loc[:, columns]) # Create a DataFrame composed of columns from both # columns and index - columns_index = {} - columns_index = { - i: col - for i, col in enumerate( - itertools.chain(index._data.columns, columns._data.columns) - ) - } - columns_index = cudf.DataFrame(columns_index) + ca = ColumnAccessor( + dict( + enumerate( + itertools.chain(index._data.columns, columns._data.columns) + ) + ), + verify=False, + ) + columns_index = cudf.DataFrame._from_data(ca) # Check that each row is unique: if len(columns_index) != len(columns_index.drop_duplicates()): @@ -1225,13 +1236,13 @@ def unstack(df, level, fill_value=None, sort: bool = True): return result -def _get_unique(column, dummy_na): +def _get_unique(column: ColumnBase, dummy_na: bool) -> ColumnBase: """ Returns unique values in a column, if dummy_na is False, nan's are also dropped. """ - if isinstance(column, cudf.core.column.CategoricalColumn): - unique = column.categories + if isinstance(column.dtype, cudf.CategoricalDtype): + unique = column.categories # type: ignore[attr-defined] else: unique = column.unique().sort_values() if not dummy_na: @@ -1251,11 +1262,11 @@ def _one_hot_encode_column( `prefix`, separated with category name with `prefix_sep`. The encoding columns maybe coerced into `dtype`. """ - if isinstance(column, CategoricalColumn): + if isinstance(column.dtype, cudf.CategoricalDtype): if column.size == column.null_count: column = column_empty_like(categories, newsize=column.size) else: - column = column._get_decategorized_column() + column = column._get_decategorized_column() # type: ignore[attr-defined] if column.size * categories.size >= np.iinfo(size_type_dtype).max: raise ValueError( @@ -1536,7 +1547,7 @@ def pivot_table( table_columns = tuple( map(lambda column: column[1:], table._data.names) ) - table.columns = cudf.MultiIndex.from_tuples( + table.columns = pd.MultiIndex.from_tuples( tuples=table_columns, names=column_names ) From bc8ca9befdd77d3f4a270a64dae178b2ef355181 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 16 Aug 2024 12:02:21 -0700 Subject: [PATCH 077/270] Setup pylibcudf package (#16299) Migrates cudf._lib.pylibcudf to a new pylibcudf package Authors: - Thomas Li (https://github.com/lithomas1) - Matthew Murray (https://github.com/Matt711) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16299 --- .github/labeler.yml | 2 +- .github/workflows/pr.yaml | 12 +- build.sh | 15 +- ci/build_docs.sh | 2 +- ci/build_python.sh | 7 + ci/build_wheel_cudf.sh | 8 +- ci/build_wheel_pylibcudf.sh | 16 ++ ci/cudf_pandas_scripts/pandas-tests/run.sh | 2 + ci/cudf_pandas_scripts/run_tests.sh | 2 + ci/test_python_cudf.sh | 2 +- ci/test_wheel_cudf.sh | 10 +- ci/test_wheel_cudf_polars.sh | 6 +- ci/test_wheel_dask_cudf.sh | 5 +- .../all_cuda-118_arch-x86_64.yaml | 2 + .../all_cuda-125_arch-x86_64.yaml | 2 + conda/recipes/cudf/meta.yaml | 2 + conda/recipes/cudf_kafka/meta.yaml | 4 +- conda/recipes/pylibcudf/build.sh | 4 + .../recipes/pylibcudf/conda_build_config.yaml | 20 +++ conda/recipes/pylibcudf/meta.yaml | 108 ++++++++++++ dependencies.yaml | 165 +++++++++++++++++- .../api_docs/pylibcudf/aggregation.rst | 2 +- .../api_docs/pylibcudf/binaryop.rst | 2 +- .../user_guide/api_docs/pylibcudf/column.rst | 2 +- .../api_docs/pylibcudf/column_factories.rst | 2 +- .../api_docs/pylibcudf/concatenate.rst | 2 +- .../user_guide/api_docs/pylibcudf/copying.rst | 2 +- .../api_docs/pylibcudf/datetime.rst | 2 +- .../api_docs/pylibcudf/expressions.rst | 2 +- .../user_guide/api_docs/pylibcudf/filling.rst | 2 +- .../api_docs/pylibcudf/gpumemoryview.rst | 2 +- .../user_guide/api_docs/pylibcudf/groupby.rst | 2 +- .../user_guide/api_docs/pylibcudf/interop.rst | 2 +- .../user_guide/api_docs/pylibcudf/io/avro.rst | 2 +- .../user_guide/api_docs/pylibcudf/io/csv.rst | 2 +- .../api_docs/pylibcudf/io/index.rst | 2 +- .../user_guide/api_docs/pylibcudf/io/json.rst | 2 +- .../api_docs/pylibcudf/io/parquet.rst | 2 +- .../user_guide/api_docs/pylibcudf/join.rst | 2 +- .../user_guide/api_docs/pylibcudf/lists.rst | 2 +- .../user_guide/api_docs/pylibcudf/merge.rst | 2 +- .../api_docs/pylibcudf/quantiles.rst | 2 +- .../user_guide/api_docs/pylibcudf/reduce.rst | 2 +- .../user_guide/api_docs/pylibcudf/replace.rst | 2 +- .../user_guide/api_docs/pylibcudf/reshape.rst | 2 +- .../user_guide/api_docs/pylibcudf/rolling.rst | 2 +- .../user_guide/api_docs/pylibcudf/round.rst | 2 +- .../user_guide/api_docs/pylibcudf/scalar.rst | 2 +- .../user_guide/api_docs/pylibcudf/search.rst | 2 +- .../user_guide/api_docs/pylibcudf/sorting.rst | 2 +- .../api_docs/pylibcudf/stream_compaction.rst | 2 +- .../api_docs/pylibcudf/strings/capitalize.rst | 2 +- .../api_docs/pylibcudf/strings/char_types.rst | 2 +- .../api_docs/pylibcudf/strings/contains.rst | 2 +- .../api_docs/pylibcudf/strings/find.rst | 2 +- .../pylibcudf/strings/regex_flags.rst | 2 +- .../pylibcudf/strings/regex_program.rst | 2 +- .../api_docs/pylibcudf/strings/replace.rst | 2 +- .../api_docs/pylibcudf/strings/slice.rst | 2 +- .../user_guide/api_docs/pylibcudf/table.rst | 2 +- .../user_guide/api_docs/pylibcudf/traits.rst | 2 +- .../api_docs/pylibcudf/transform.rst | 2 +- .../user_guide/api_docs/pylibcudf/types.rst | 2 +- .../user_guide/api_docs/pylibcudf/unary.rst | 2 +- python/cudf/CMakeLists.txt | 4 +- python/cudf/cudf/_lib/CMakeLists.txt | 1 - python/cudf/cudf/_lib/__init__.py | 1 - python/cudf/cudf/_lib/aggregation.pyx | 3 +- python/cudf/cudf/_lib/avro.pyx | 4 +- python/cudf/cudf/_lib/binaryop.pyx | 3 +- python/cudf/cudf/_lib/column.pxd | 9 +- python/cudf/cudf/_lib/column.pyx | 20 +-- python/cudf/cudf/_lib/concat.pyx | 3 +- python/cudf/cudf/_lib/copying.pxd | 2 +- python/cudf/cudf/_lib/copying.pyx | 20 +-- python/cudf/cudf/_lib/csv.pyx | 21 ++- python/cudf/cudf/_lib/datetime.pyx | 13 +- python/cudf/cudf/_lib/filling.pyx | 4 +- python/cudf/cudf/_lib/groupby.pyx | 7 +- python/cudf/cudf/_lib/hash.pyx | 15 +- python/cudf/cudf/_lib/interop.pyx | 9 +- python/cudf/cudf/_lib/io/utils.pxd | 7 +- python/cudf/cudf/_lib/io/utils.pyx | 11 +- python/cudf/cudf/_lib/join.pyx | 2 +- python/cudf/cudf/_lib/json.pyx | 15 +- python/cudf/cudf/_lib/labeling.pyx | 10 +- python/cudf/cudf/_lib/lists.pyx | 7 +- python/cudf/cudf/_lib/merge.pyx | 2 +- python/cudf/cudf/_lib/null_mask.pyx | 11 +- .../cudf/_lib/nvtext/byte_pair_encode.pyx | 11 +- .../cudf/cudf/_lib/nvtext/edit_distance.pyx | 9 +- .../cudf/cudf/_lib/nvtext/generate_ngrams.pyx | 13 +- python/cudf/cudf/_lib/nvtext/jaccard.pyx | 11 +- python/cudf/cudf/_lib/nvtext/minhash.pyx | 11 +- .../cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx | 13 +- python/cudf/cudf/_lib/nvtext/normalize.pyx | 9 +- python/cudf/cudf/_lib/nvtext/replace.pyx | 13 +- python/cudf/cudf/_lib/nvtext/stemmer.pyx | 11 +- .../cudf/_lib/nvtext/subword_tokenize.pyx | 7 +- python/cudf/cudf/_lib/nvtext/tokenize.pyx | 13 +- python/cudf/cudf/_lib/orc.pyx | 33 ++-- python/cudf/cudf/_lib/parquet.pyx | 39 +++-- python/cudf/cudf/_lib/partitioning.pyx | 13 +- python/cudf/cudf/_lib/pylibcudf/io/avro.pxd | 12 -- .../pylibcudf/libcudf/strings/extract.pxd | 15 -- .../_lib/pylibcudf/strings/char_types.pxd | 5 - .../cudf/_lib/pylibcudf/strings/contains.pxd | 7 - .../_lib/pylibcudf/strings/regex_flags.pxd | 2 - python/cudf/cudf/_lib/quantiles.pyx | 5 +- python/cudf/cudf/_lib/reduce.pyx | 3 +- python/cudf/cudf/_lib/replace.pyx | 3 +- python/cudf/cudf/_lib/reshape.pyx | 5 +- python/cudf/cudf/_lib/rolling.pyx | 3 +- python/cudf/cudf/_lib/round.pyx | 4 +- python/cudf/cudf/_lib/scalar.pxd | 3 +- python/cudf/cudf/_lib/scalar.pyx | 14 +- python/cudf/cudf/_lib/search.pyx | 2 +- python/cudf/cudf/_lib/sort.pyx | 13 +- python/cudf/cudf/_lib/stream_compaction.pyx | 2 +- python/cudf/cudf/_lib/string_casting.pyx | 21 +-- python/cudf/cudf/_lib/strings/attributes.pyx | 9 +- python/cudf/cudf/_lib/strings/capitalize.pyx | 2 +- python/cudf/cudf/_lib/strings/case.pyx | 2 +- python/cudf/cudf/_lib/strings/char_types.pyx | 11 +- python/cudf/cudf/_lib/strings/combine.pyx | 13 +- python/cudf/cudf/_lib/strings/contains.pyx | 19 +- .../strings/convert/convert_fixed_point.pyx | 11 +- .../_lib/strings/convert/convert_floats.pyx | 9 +- .../_lib/strings/convert/convert_integers.pyx | 9 +- .../_lib/strings/convert/convert_lists.pyx | 11 +- .../_lib/strings/convert/convert_urls.pyx | 9 +- python/cudf/cudf/_lib/strings/extract.pyx | 11 +- python/cudf/cudf/_lib/strings/find.pyx | 6 +- .../cudf/cudf/_lib/strings/find_multiple.pyx | 9 +- python/cudf/cudf/_lib/strings/findall.pyx | 11 +- python/cudf/cudf/_lib/strings/json.pyx | 11 +- python/cudf/cudf/_lib/strings/padding.pyx | 11 +- python/cudf/cudf/_lib/strings/repeat.pyx | 9 +- python/cudf/cudf/_lib/strings/replace.pyx | 5 +- python/cudf/cudf/_lib/strings/replace_re.pyx | 17 +- .../cudf/_lib/strings/split/partition.pyx | 11 +- python/cudf/cudf/_lib/strings/split/split.pyx | 19 +- python/cudf/cudf/_lib/strings/strip.pyx | 11 +- python/cudf/cudf/_lib/strings/substring.pyx | 2 +- python/cudf/cudf/_lib/strings/translate.pyx | 13 +- python/cudf/cudf/_lib/strings/wrap.pyx | 9 +- python/cudf/cudf/_lib/strings_udf.pyx | 12 +- python/cudf/cudf/_lib/text.pyx | 7 +- python/cudf/cudf/_lib/timezone.pyx | 5 +- python/cudf/cudf/_lib/transform.pyx | 24 +-- python/cudf/cudf/_lib/transpose.pyx | 7 +- python/cudf/cudf/_lib/types.pxd | 8 +- python/cudf/cudf/_lib/types.pyx | 12 +- python/cudf/cudf/_lib/unary.pyx | 3 +- python/cudf/cudf/_lib/utils.pxd | 4 +- python/cudf/cudf/_lib/utils.pyx | 9 +- .../cudf/cudf/core/_internals/expressions.py | 4 +- python/cudf/cudf/core/buffer/buffer.py | 3 +- python/cudf/cudf/core/column/numerical.py | 3 +- python/cudf/cudf/core/indexed_frame.py | 4 +- python/cudf/cudf/pandas/__init__.py | 3 +- python/cudf/pyproject.toml | 3 + .../cudf_kafka/cudf_kafka/_lib/CMakeLists.txt | 4 +- python/cudf_kafka/cudf_kafka/_lib/kafka.pxd | 5 +- python/cudf_kafka/cudf_kafka/_lib/kafka.pyx | 3 +- .../cudf_polars/containers/column.py | 2 +- .../cudf_polars/containers/dataframe.py | 3 +- python/cudf_polars/cudf_polars/dsl/expr.py | 3 +- python/cudf_polars/cudf_polars/dsl/ir.py | 3 +- .../cudf_polars/cudf_polars/dsl/translate.py | 3 +- .../cudf_polars/typing/__init__.py | 4 +- .../cudf_polars/cudf_polars/utils/dtypes.py | 3 +- .../cudf_polars/cudf_polars/utils/sorting.py | 2 +- python/cudf_polars/pyproject.toml | 2 +- .../tests/containers/test_column.py | 3 +- .../tests/containers/test_dataframe.py | 3 +- python/cudf_polars/tests/dsl/test_expr.py | 3 +- .../tests/expressions/test_literal.py | 3 +- .../tests/expressions/test_sort.py | 3 +- .../cudf_polars/tests/utils/test_broadcast.py | 3 +- python/pylibcudf/CMakeLists.txt | 100 +++++++++++ python/pylibcudf/README.md | 1 + .../cmake/Modules/LinkPyarrowHeaders.cmake | 0 .../cmake/Modules/WheelHelpers.cmake | 0 .../pylibcudf/CMakeLists.txt | 2 +- python/pylibcudf/pylibcudf/VERSION | 1 + .../_lib => pylibcudf}/pylibcudf/__init__.pxd | 0 .../_lib => pylibcudf}/pylibcudf/__init__.py | 1 + python/pylibcudf/pylibcudf/_version.py | 24 +++ .../pylibcudf/aggregation.pxd | 5 +- .../pylibcudf/aggregation.pyx | 21 +-- .../_lib => pylibcudf}/pylibcudf/binaryop.pxd | 3 +- .../_lib => pylibcudf}/pylibcudf/binaryop.pyx | 15 +- .../_lib => pylibcudf}/pylibcudf/column.pxd | 11 +- .../_lib => pylibcudf}/pylibcudf/column.pyx | 11 +- .../pylibcudf/column_factories.pxd | 3 +- .../pylibcudf/column_factories.pyx | 7 +- .../pylibcudf/concatenate.pxd | 0 .../pylibcudf/concatenate.pyx | 11 +- .../_lib => pylibcudf}/pylibcudf/copying.pxd | 5 +- .../_lib => pylibcudf}/pylibcudf/copying.pyx | 21 ++- .../_lib => pylibcudf}/pylibcudf/datetime.pxd | 0 .../_lib => pylibcudf}/pylibcudf/datetime.pyx | 7 +- .../pylibcudf/exception_handler.pxd | 0 .../pylibcudf/experimental.pxd | 0 .../pylibcudf/experimental.pyx | 3 +- .../pylibcudf/expressions.pxd | 3 +- .../pylibcudf/expressions.pyx | 15 +- .../_lib => pylibcudf}/pylibcudf/filling.pxd | 2 +- .../_lib => pylibcudf}/pylibcudf/filling.pyx | 9 +- .../pylibcudf/gpumemoryview.pxd | 0 .../pylibcudf/gpumemoryview.pyx | 0 .../_lib => pylibcudf}/pylibcudf/groupby.pxd | 9 +- .../_lib => pylibcudf}/pylibcudf/groupby.pyx | 15 +- .../_lib => pylibcudf}/pylibcudf/interop.pyx | 6 +- .../pylibcudf/io/CMakeLists.txt | 0 .../pylibcudf/io/__init__.pxd | 0 .../pylibcudf/io/__init__.py | 0 python/pylibcudf/pylibcudf/io/avro.pxd | 12 ++ .../_lib => pylibcudf}/pylibcudf/io/avro.pyx | 7 +- .../_lib => pylibcudf}/pylibcudf/io/csv.pyx | 11 +- .../pylibcudf/io/datasource.pxd | 5 +- .../pylibcudf/io/datasource.pyx | 5 +- .../_lib => pylibcudf}/pylibcudf/io/json.pxd | 7 +- .../_lib => pylibcudf}/pylibcudf/io/json.pyx | 19 +- .../pylibcudf/io/parquet.pxd | 11 +- .../pylibcudf/io/parquet.pyx | 15 +- .../_lib => pylibcudf}/pylibcudf/io/types.pxd | 7 +- .../_lib => pylibcudf}/pylibcudf/io/types.pyx | 13 +- .../_lib => pylibcudf}/pylibcudf/join.pxd | 2 +- .../_lib => pylibcudf}/pylibcudf/join.pyx | 9 +- .../pylibcudf/libcudf/CMakeLists.txt | 0 .../pylibcudf/libcudf/__init__.pxd | 0 .../pylibcudf/libcudf/__init__.py | 0 .../pylibcudf/libcudf/aggregation.pxd | 3 +- .../pylibcudf/libcudf/aggregation.pyx | 0 .../pylibcudf/libcudf/binaryop.pxd | 11 +- .../pylibcudf/libcudf/binaryop.pyx | 0 .../pylibcudf/libcudf/column/__init__.pxd | 0 .../pylibcudf/libcudf/column/__init__.py | 0 .../pylibcudf/libcudf/column/column.pxd | 9 +- .../libcudf/column/column_factories.pxd | 11 +- .../pylibcudf/libcudf/column/column_view.pxd | 7 +- .../pylibcudf/libcudf/concatenate.pxd | 7 +- .../pylibcudf/libcudf/contiguous_split.pxd | 5 +- .../pylibcudf/libcudf/copying.pxd | 19 +- .../pylibcudf/libcudf/copying.pyx | 0 .../pylibcudf/libcudf/datetime.pxd | 7 +- .../pylibcudf/libcudf/experimental.pxd | 0 .../pylibcudf/libcudf/expressions.pxd | 9 +- .../pylibcudf/libcudf/expressions.pyx | 0 .../pylibcudf/libcudf/filling.pxd | 13 +- .../pylibcudf/libcudf/groupby.pxd | 19 +- .../pylibcudf/libcudf/hash.pxd | 7 +- .../pylibcudf/libcudf/interop.pxd | 13 +- .../pylibcudf/libcudf/io/CMakeLists.txt | 0 .../pylibcudf/libcudf/io/__init__.pxd | 0 .../pylibcudf/libcudf/io/__init__.py | 0 .../pylibcudf/libcudf/io/arrow_io_source.pxd | 3 +- .../pylibcudf/libcudf/io/avro.pxd | 5 +- .../pylibcudf/libcudf/io/csv.pxd | 7 +- .../pylibcudf/libcudf/io/data_sink.pxd | 0 .../pylibcudf/libcudf/io/datasource.pxd | 0 .../pylibcudf/libcudf/io/json.pxd | 7 +- .../pylibcudf/libcudf/io/json.pyx | 0 .../pylibcudf/libcudf/io/orc.pxd | 7 +- .../pylibcudf/libcudf/io/orc_metadata.pxd | 5 +- .../pylibcudf/libcudf/io/parquet.pxd | 76 ++++---- .../pylibcudf/libcudf/io/parquet_metadata.pxd | 5 +- .../pylibcudf/libcudf/io/text.pxd | 3 +- .../pylibcudf/libcudf/io/timezone.pxd | 3 +- .../pylibcudf/libcudf/io/types.pxd | 11 +- .../pylibcudf/libcudf/io/types.pyx | 0 .../pylibcudf/libcudf/join.pxd | 9 +- .../pylibcudf/libcudf/labeling.pxd | 5 +- .../pylibcudf/libcudf/lists/__init__.pxd | 0 .../pylibcudf/libcudf/lists/__init__.py | 0 .../pylibcudf/libcudf/lists/combine.pxd | 7 +- .../pylibcudf/libcudf/lists/contains.pxd | 13 +- .../libcudf/lists/count_elements.pxd | 7 +- .../pylibcudf/libcudf/lists/explode.pxd | 7 +- .../pylibcudf/libcudf/lists/extract.pxd | 9 +- .../pylibcudf/libcudf/lists/filling.pxd | 5 +- .../pylibcudf/libcudf/lists/gather.pxd | 7 +- .../libcudf/lists/lists_column_view.pxd | 4 +- .../pylibcudf/libcudf/lists/reverse.pxd | 7 +- .../libcudf/lists/set_operations.pxd | 9 +- .../pylibcudf/libcudf/lists/sorting.pxd | 9 +- .../libcudf/lists/stream_compaction.pxd | 9 +- .../pylibcudf/libcudf/merge.pxd | 7 +- .../pylibcudf/libcudf/null_mask.pxd | 11 +- .../pylibcudf/libcudf/nvtext/__init__.pxd | 0 .../pylibcudf/libcudf/nvtext/__init__.py | 0 .../libcudf/nvtext/byte_pair_encode.pxd | 7 +- .../libcudf/nvtext/edit_distance.pxd | 5 +- .../libcudf/nvtext/generate_ngrams.pxd | 9 +- .../pylibcudf/libcudf/nvtext/jaccard.pxd | 7 +- .../pylibcudf/libcudf/nvtext/minhash.pxd | 7 +- .../libcudf/nvtext/ngrams_tokenize.pxd | 9 +- .../pylibcudf/libcudf/nvtext/normalize.pxd | 5 +- .../pylibcudf/libcudf/nvtext/replace.pxd | 9 +- .../pylibcudf/libcudf/nvtext/stemmer.pxd | 7 +- .../libcudf/nvtext/subword_tokenize.pxd | 5 +- .../pylibcudf/libcudf/nvtext/tokenize.pxd | 9 +- .../pylibcudf/libcudf/partitioning.pxd | 11 +- .../pylibcudf/libcudf/quantiles.pxd | 11 +- .../pylibcudf/libcudf/reduce.pxd | 14 +- .../pylibcudf/libcudf/reduce.pyx | 0 .../pylibcudf/libcudf/replace.pxd | 9 +- .../pylibcudf/libcudf/replace.pyx | 0 .../pylibcudf/libcudf/reshape.pxd | 9 +- .../pylibcudf/libcudf/rolling.pxd | 11 +- .../pylibcudf/libcudf/round.pxd | 5 +- .../pylibcudf/libcudf/round.pyx | 0 .../pylibcudf/libcudf/scalar/__init__.pxd | 0 .../pylibcudf/libcudf/scalar/__init__.py | 0 .../pylibcudf/libcudf/scalar/scalar.pxd | 9 +- .../libcudf/scalar/scalar_factories.pxd | 5 +- .../pylibcudf/libcudf/search.pxd | 9 +- .../pylibcudf/libcudf/sorting.pxd | 15 +- .../pylibcudf/libcudf/stream_compaction.pxd | 13 +- .../pylibcudf/libcudf/stream_compaction.pyx | 0 .../pylibcudf/libcudf/strings/CMakeLists.txt | 0 .../pylibcudf/libcudf/strings/__init__.pxd | 0 .../pylibcudf/libcudf/strings/__init__.py | 0 .../pylibcudf/libcudf/strings/attributes.pxd | 5 +- .../pylibcudf/libcudf/strings/capitalize.pxd | 11 +- .../pylibcudf/libcudf/strings/case.pxd | 5 +- .../pylibcudf/libcudf/strings/char_types.pxd | 7 +- .../pylibcudf/libcudf/strings/char_types.pyx | 0 .../pylibcudf/libcudf/strings/combine.pxd | 9 +- .../pylibcudf/libcudf/strings/contains.pxd | 9 +- .../libcudf/strings/convert/__init__.pxd | 0 .../libcudf/strings/convert/__init__.py | 0 .../strings/convert/convert_booleans.pxd | 7 +- .../strings/convert/convert_datetime.pxd | 7 +- .../strings/convert/convert_durations.pxd | 7 +- .../strings/convert/convert_fixed_point.pxd | 7 +- .../strings/convert/convert_floats.pxd | 7 +- .../strings/convert/convert_integers.pxd | 7 +- .../libcudf/strings/convert/convert_ipv4.pxd | 5 +- .../libcudf/strings/convert/convert_lists.pxd | 7 +- .../libcudf/strings/convert/convert_urls.pxd | 5 +- .../pylibcudf/libcudf/strings/extract.pxd | 14 ++ .../pylibcudf/libcudf/strings/find.pxd | 9 +- .../libcudf/strings/find_multiple.pxd | 5 +- .../pylibcudf/libcudf/strings/findall.pxd | 7 +- .../pylibcudf/libcudf/strings/json.pxd | 7 +- .../pylibcudf/libcudf/strings/padding.pxd | 11 +- .../pylibcudf/libcudf/strings/regex_flags.pxd | 0 .../pylibcudf/libcudf/strings/regex_flags.pyx | 0 .../libcudf/strings/regex_program.pxd | 3 +- .../pylibcudf/libcudf/strings/repeat.pxd | 7 +- .../pylibcudf/libcudf/strings/replace.pxd | 9 +- .../pylibcudf/libcudf/strings/replace_re.pxd | 13 +- .../pylibcudf/libcudf/strings/side_type.pxd | 0 .../libcudf/strings/split/__init__.pxd | 0 .../libcudf/strings/split/__init__.py | 0 .../libcudf/strings/split/partition.pxd | 9 +- .../pylibcudf/libcudf/strings/split/split.pxd | 13 +- .../pylibcudf/libcudf/strings/strip.pxd | 9 +- .../pylibcudf/libcudf/strings/substring.pxd | 9 +- .../pylibcudf/libcudf/strings/translate.pxd | 9 +- .../pylibcudf/libcudf/strings/wrap.pxd | 7 +- .../pylibcudf/libcudf/strings_udf.pxd | 7 +- .../pylibcudf/libcudf/table/__init__.pxd | 0 .../pylibcudf/libcudf/table/__init__.py | 0 .../pylibcudf/libcudf/table/table.pxd | 10 +- .../pylibcudf/libcudf/table/table_view.pxd | 5 +- .../pylibcudf/libcudf/transform.pxd | 17 +- .../pylibcudf/libcudf/transpose.pxd | 5 +- .../pylibcudf/libcudf/types.pxd | 0 .../pylibcudf/libcudf/types.pyx | 0 .../pylibcudf/libcudf/unary.pxd | 7 +- .../pylibcudf/libcudf/unary.pyx | 0 .../pylibcudf/libcudf/utilities/__init__.pxd | 0 .../pylibcudf/libcudf/utilities/__init__.py | 0 .../pylibcudf/libcudf/utilities/host_span.pxd | 0 .../pylibcudf/libcudf/utilities/traits.pxd | 3 +- .../libcudf/utilities/type_dispatcher.pxd | 2 +- .../pylibcudf/libcudf/wrappers/__init__.pxd | 0 .../pylibcudf/libcudf/wrappers/__init__.py | 0 .../pylibcudf/libcudf/wrappers/decimals.pxd | 3 +- .../pylibcudf/libcudf/wrappers/durations.pxd | 0 .../pylibcudf/libcudf/wrappers/timestamps.pxd | 0 .../_lib => pylibcudf}/pylibcudf/lists.pxd | 3 +- .../_lib => pylibcudf}/pylibcudf/lists.pyx | 29 ++- .../_lib => pylibcudf}/pylibcudf/merge.pxd | 0 .../_lib => pylibcudf}/pylibcudf/merge.pyx | 9 +- .../pylibcudf/quantiles.pxd | 3 +- .../pylibcudf/quantiles.pyx | 11 +- .../_lib => pylibcudf}/pylibcudf/reduce.pxd | 2 +- .../_lib => pylibcudf}/pylibcudf/reduce.pyx | 17 +- .../_lib => pylibcudf}/pylibcudf/replace.pxd | 3 +- .../_lib => pylibcudf}/pylibcudf/replace.pyx | 7 +- .../_lib => pylibcudf}/pylibcudf/reshape.pxd | 2 +- .../_lib => pylibcudf}/pylibcudf/reshape.pyx | 9 +- .../_lib => pylibcudf}/pylibcudf/rolling.pxd | 2 +- .../_lib => pylibcudf}/pylibcudf/rolling.pyx | 9 +- .../_lib => pylibcudf}/pylibcudf/round.pxd | 3 +- .../_lib => pylibcudf}/pylibcudf/round.pyx | 10 +- .../_lib => pylibcudf}/pylibcudf/scalar.pxd | 3 +- .../_lib => pylibcudf}/pylibcudf/scalar.pyx | 7 +- .../_lib => pylibcudf}/pylibcudf/search.pxd | 0 .../_lib => pylibcudf}/pylibcudf/search.pyx | 7 +- .../_lib => pylibcudf}/pylibcudf/sorting.pxd | 10 +- .../_lib => pylibcudf}/pylibcudf/sorting.pyx | 11 +- .../pylibcudf/stream_compaction.pxd | 6 +- .../pylibcudf/stream_compaction.pyx | 17 +- .../pylibcudf/strings/CMakeLists.txt | 0 .../pylibcudf/strings/__init__.pxd | 0 .../pylibcudf/strings/__init__.py | 0 .../pylibcudf/strings/capitalize.pxd | 4 +- .../pylibcudf/strings/capitalize.pyx | 15 +- .../pylibcudf/strings/case.pxd | 2 +- .../pylibcudf/strings/case.pyx | 7 +- .../pylibcudf/strings/char_types.pxd | 3 + .../pylibcudf/strings/char_types.pyx | 2 +- .../pylibcudf/pylibcudf/strings/contains.pxd | 7 + .../pylibcudf/strings/contains.pyx | 9 +- .../pylibcudf/strings/find.pxd | 6 +- .../pylibcudf/strings/find.pyx | 27 ++- .../pylibcudf/strings/regex_flags.pxd | 2 + .../pylibcudf/strings/regex_flags.pyx | 2 +- .../pylibcudf/strings/regex_program.pxd | 3 +- .../pylibcudf/strings/regex_program.pyx | 8 +- .../pylibcudf/strings/replace.pxd | 6 +- .../pylibcudf/strings/replace.pyx | 15 +- .../pylibcudf/strings/slice.pxd | 4 +- .../pylibcudf/strings/slice.pyx | 21 ++- .../_lib => pylibcudf}/pylibcudf/table.pxd | 5 +- .../_lib => pylibcudf}/pylibcudf/table.pyx | 7 +- .../pylibcudf/tests}/common/utils.py | 14 +- .../pylibcudf/tests}/conftest.py | 5 +- .../pylibcudf/tests}/io/test_avro.py | 3 +- .../pylibcudf/tests}/io/test_csv.py | 5 +- .../pylibcudf/tests}/io/test_json.py | 5 +- .../pylibcudf/tests}/io/test_parquet.py | 7 +- .../tests}/io/test_source_sink_info.py | 3 +- .../pylibcudf/tests}/pytest.ini | 0 .../pylibcudf/tests}/test_binaryops.py | 3 +- .../pylibcudf/tests}/test_column_factories.py | 3 +- .../tests}/test_column_from_device.py | 3 +- .../pylibcudf/tests}/test_copying.py | 3 +- .../pylibcudf/tests}/test_datetime.py | 6 +- .../pylibcudf/tests}/test_expressions.py | 3 +- .../pylibcudf/tests}/test_interop.py | 3 +- .../pylibcudf/tests}/test_join.py | 3 +- .../pylibcudf/tests}/test_lists.py | 6 +- .../pylibcudf/tests}/test_quantiles.py | 3 +- .../pylibcudf/tests}/test_regex_program.py | 3 +- .../pylibcudf/tests}/test_reshape.py | 3 +- .../pylibcudf/tests}/test_round.py | 8 +- .../tests}/test_string_capitalize.py | 10 +- .../pylibcudf/tests}/test_string_case.py | 10 +- .../pylibcudf/tests}/test_string_contains.py | 6 +- .../pylibcudf/tests}/test_string_find.py | 8 +- .../pylibcudf/tests}/test_string_replace.py | 8 +- .../pylibcudf/tests}/test_string_slice.py | 3 +- .../pylibcudf/tests}/test_table.py | 3 +- .../pylibcudf/tests}/test_traits.py | 2 +- .../pylibcudf/tests}/test_transform.py | 3 +- .../pylibcudf/tests}/test_unary.py | 2 +- .../_lib => pylibcudf}/pylibcudf/traits.pxd | 0 .../_lib => pylibcudf}/pylibcudf/traits.pyx | 3 +- .../pylibcudf/transform.pxd | 0 .../pylibcudf/transform.pyx | 5 +- .../_lib => pylibcudf}/pylibcudf/types.pxd | 3 +- .../_lib => pylibcudf}/pylibcudf/types.pyx | 27 ++- .../_lib => pylibcudf}/pylibcudf/unary.pxd | 3 +- .../_lib => pylibcudf}/pylibcudf/unary.pyx | 9 +- .../_lib => pylibcudf}/pylibcudf/utils.pxd | 5 +- .../_lib => pylibcudf}/pylibcudf/utils.pyx | 5 +- .../_lib => pylibcudf/pylibcudf}/variant.pxd | 0 python/pylibcudf/pyproject.toml | 123 +++++++++++++ 475 files changed, 1916 insertions(+), 1522 deletions(-) create mode 100755 ci/build_wheel_pylibcudf.sh create mode 100644 conda/recipes/pylibcudf/build.sh create mode 100644 conda/recipes/pylibcudf/conda_build_config.yaml create mode 100644 conda/recipes/pylibcudf/meta.yaml delete mode 100644 python/cudf/cudf/_lib/pylibcudf/io/avro.pxd delete mode 100644 python/cudf/cudf/_lib/pylibcudf/libcudf/strings/extract.pxd delete mode 100644 python/cudf/cudf/_lib/pylibcudf/strings/char_types.pxd delete mode 100644 python/cudf/cudf/_lib/pylibcudf/strings/contains.pxd delete mode 100644 python/cudf/cudf/_lib/pylibcudf/strings/regex_flags.pxd create mode 100644 python/pylibcudf/CMakeLists.txt create mode 120000 python/pylibcudf/README.md rename python/{cudf => pylibcudf}/cmake/Modules/LinkPyarrowHeaders.cmake (100%) rename python/{cudf => pylibcudf}/cmake/Modules/WheelHelpers.cmake (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/CMakeLists.txt (96%) create mode 120000 python/pylibcudf/pylibcudf/VERSION rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/__init__.py (99%) create mode 100644 python/pylibcudf/pylibcudf/_version.py rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/aggregation.pxd (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/aggregation.pyx (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/binaryop.pxd (90%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/binaryop.pyx (86%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/column.pxd (84%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/column.pyx (98%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/column_factories.pxd (92%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/column_factories.pyx (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/concatenate.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/concatenate.pyx (80%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/copying.pxd (94%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/copying.pyx (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/datetime.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/datetime.pyx (78%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/exception_handler.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/experimental.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/experimental.pyx (92%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/expressions.pxd (91%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/expressions.pyx (94%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/filling.pxd (90%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/filling.pyx (94%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/gpumemoryview.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/gpumemoryview.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/groupby.pxd (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/groupby.pyx (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/interop.pyx (98%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/CMakeLists.txt (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/__init__.py (100%) create mode 100644 python/pylibcudf/pylibcudf/io/avro.pxd rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/avro.pyx (89%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/csv.pyx (97%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/datasource.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/datasource.pyx (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/json.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/json.pyx (95%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/parquet.pxd (72%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/parquet.pyx (93%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/types.pxd (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/io/types.pyx (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/join.pxd (91%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/join.pyx (95%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/CMakeLists.txt (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/aggregation.pxd (98%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/aggregation.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/binaryop.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/binaryop.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/column/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/column/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/column/column.pxd (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/column/column_factories.pxd (93%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/column/column_view.pxd (97%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/concatenate.pxd (77%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/contiguous_split.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/copying.pxd (90%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/copying.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/datetime.pxd (92%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/experimental.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/expressions.pxd (90%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/expressions.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/filling.pxd (74%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/groupby.pxd (83%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/hash.pxd (86%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/interop.pxd (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/CMakeLists.txt (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/arrow_io_source.pxd (86%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/avro.pxd (91%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/csv.pxd (98%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/data_sink.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/datasource.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/json.pxd (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/json.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/orc.pxd (97%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/orc_metadata.pxd (94%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/parquet.pxd (80%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/parquet_metadata.pxd (89%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/text.pxd (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/timezone.pxd (86%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/types.pxd (92%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/io/types.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/join.pxd (88%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/labeling.pxd (78%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/combine.pxd (78%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/contains.pxd (75%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/count_elements.pxd (61%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/explode.pxd (59%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/extract.pxd (64%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/filling.pxd (76%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/gather.pxd (67%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/lists_column_view.pxd (86%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/reverse.pxd (62%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/set_operations.pxd (81%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/sorting.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/lists/stream_compaction.pxd (68%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/merge.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/null_mask.pxd (80%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/byte_pair_encode.pxd (73%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/edit_distance.pxd (75%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/generate_ngrams.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/jaccard.pxd (61%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/minhash.pxd (70%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/ngrams_tokenize.pxd (58%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/normalize.pxd (75%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/replace.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/stemmer.pxd (79%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/subword_tokenize.pxd (92%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/nvtext/tokenize.pxd (84%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/partitioning.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/quantiles.pxd (70%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/reduce.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/reduce.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/replace.pxd (83%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/replace.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/reshape.pxd (57%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/rolling.pxd (64%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/round.pxd (75%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/round.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/scalar/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/scalar/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/scalar/scalar.pxd (91%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/scalar/scalar_factories.pxd (76%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/search.pxd (73%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/sorting.pxd (84%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/stream_compaction.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/stream_compaction.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/CMakeLists.txt (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/attributes.pxd (76%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/capitalize.pxd (63%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/case.pxd (81%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/char_types.pxd (82%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/char_types.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/combine.pxd (83%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/contains.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_booleans.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_datetime.pxd (76%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_durations.pxd (72%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_fixed_point.pxd (73%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_floats.pxd (71%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_integers.pxd (80%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_ipv4.pxd (76%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_lists.pxd (62%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/convert/convert_urls.pxd (72%) create mode 100644 python/pylibcudf/pylibcudf/libcudf/strings/extract.pxd rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/find.pxd (83%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/find_multiple.pxd (68%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/findall.pxd (56%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/json.pxd (79%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/padding.pxd (59%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/regex_flags.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/regex_flags.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/regex_program.pxd (84%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/repeat.pxd (67%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/replace.pxd (73%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/replace_re.pxd (63%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/side_type.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/split/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/split/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/split/partition.pxd (63%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/split/split.pxd (78%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/strip.pxd (52%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/substring.pxd (66%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/translate.pxd (73%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings/wrap.pxd (58%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/strings_udf.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/table/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/table/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/table/table.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/table/table_view.pxd (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/transform.pxd (73%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/transpose.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/types.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/types.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/unary.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/unary.pyx (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/utilities/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/utilities/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/utilities/host_span.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/utilities/traits.pxd (93%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/utilities/type_dispatcher.pxd (73%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/wrappers/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/wrappers/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/wrappers/decimals.pxd (90%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/wrappers/durations.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/libcudf/wrappers/timestamps.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/lists.pxd (94%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/lists.pyx (95%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/merge.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/merge.pyx (83%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/quantiles.pxd (86%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/quantiles.pyx (93%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/reduce.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/reduce.pyx (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/replace.pxd (92%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/replace.pyx (97%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/reshape.pxd (80%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/reshape.pyx (86%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/rolling.pxd (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/rolling.pyx (89%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/round.pxd (77%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/round.pyx (85%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/scalar.pxd (92%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/scalar.pyx (94%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/search.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/search.pyx (93%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/sorting.pxd (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/sorting.pyx (96%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/stream_compaction.pxd (89%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/stream_compaction.pyx (95%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/CMakeLists.txt (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/__init__.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/__init__.py (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/capitalize.pxd (64%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/capitalize.pyx (84%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/case.pxd (76%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/case.pyx (79%) create mode 100644 python/pylibcudf/pylibcudf/strings/char_types.pxd rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/char_types.pyx (64%) create mode 100644 python/pylibcudf/pylibcudf/strings/contains.pxd rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/contains.pyx (75%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/find.pxd (77%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/find.pyx (90%) create mode 100644 python/pylibcudf/pylibcudf/strings/regex_flags.pxd rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/regex_flags.pyx (59%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/regex_program.pxd (70%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/regex_program.pyx (84%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/replace.pxd (71%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/replace.pyx (90%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/slice.pxd (69%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/strings/slice.pyx (81%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/table.pxd (78%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/table.pyx (93%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/common/utils.py (97%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/conftest.py (98%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/io/test_avro.py (98%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/io/test_csv.py (98%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/io/test_json.py (99%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/io/test_parquet.py (97%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/io/test_source_sink_info.py (98%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/pytest.ini (100%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_binaryops.py (99%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_column_factories.py (99%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_column_from_device.py (97%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_copying.py (99%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_datetime.py (83%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_expressions.py (97%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_interop.py (98%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_join.py (94%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_lists.py (99%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_quantiles.py (99%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_regex_program.py (89%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_reshape.py (96%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_round.py (86%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_string_capitalize.py (86%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_string_case.py (80%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_string_contains.py (92%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_string_find.py (97%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_string_replace.py (95%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_string_slice.py (98%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_table.py (93%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_traits.py (98%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_transform.py (95%) rename python/{cudf/cudf/pylibcudf_tests => pylibcudf/pylibcudf/tests}/test_unary.py (93%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/traits.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/traits.pyx (98%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/transform.pxd (100%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/transform.pyx (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/types.pxd (91%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/types.pyx (66%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/unary.pxd (87%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/unary.pyx (94%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/utils.pxd (71%) rename python/{cudf/cudf/_lib => pylibcudf}/pylibcudf/utils.pyx (93%) rename python/{cudf/cudf/_lib => pylibcudf/pylibcudf}/variant.pxd (100%) create mode 100644 python/pylibcudf/pyproject.toml diff --git a/.github/labeler.yml b/.github/labeler.yml index 48967417af3..90cdda4d3ca 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -12,7 +12,7 @@ cudf.polars: - 'python/cudf_polars/**' pylibcudf: - - 'python/cudf/cudf/_lib/pylibcudf/**' + - 'python/cudf/pylibcudf/**' libcudf: - 'cpp/**' diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ea8a1762b2c..74bdc666c68 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,6 +23,7 @@ jobs: - static-configure - conda-notebook-tests - docs-build + - wheel-build-pylibcudf - wheel-build-cudf - wheel-tests-cudf - wheel-build-cudf-polars @@ -120,10 +121,17 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cudf: + wheel-build-pylibcudf: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/build_wheel_pylibcudf.sh" + wheel-build-cudf: + needs: wheel-build-pylibcudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_cudf.sh" @@ -135,7 +143,7 @@ jobs: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: - needs: wheel-build-cudf + needs: wheel-build-pylibcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/build.sh b/build.sh index 52bb1e64d16..957f41aedac 100755 --- a/build.sh +++ b/build.sh @@ -17,11 +17,12 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings" -HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] +VALIDARGS="clean libcudf pylibcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings" +HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] clean - remove all existing build artifacts and configuration (start over) libcudf - build the cudf C++ code only + pylibcudf - build the pylibcudf Python package cudf - build the cudf Python package cudfjar - build cudf JAR with static libcudf using devtoolset toolchain dask_cudf - build the dask_cudf Python package @@ -268,7 +269,7 @@ fi ################################################################################ # Configure, build, and install libcudf -if buildAll || hasArg libcudf || hasArg cudf || hasArg cudfjar; then +if buildAll || hasArg libcudf || hasArg pylibcudf || hasArg cudf || hasArg cudfjar; then if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then CUDF_CMAKE_CUDA_ARCHITECTURES="${CUDF_CMAKE_CUDA_ARCHITECTURES:-NATIVE}" if [[ "$CUDF_CMAKE_CUDA_ARCHITECTURES" == "NATIVE" ]]; then @@ -340,6 +341,14 @@ if buildAll || hasArg libcudf; then fi fi +# Build and install the pylibcudf Python package +if buildAll || hasArg pylibcudf; then + + cd ${REPODIR}/python/pylibcudf + SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES};${EXTRA_CMAKE_ARGS}" \ + python ${PYTHON_ARGS_FOR_INSTALL} . +fi + # Build and install the cudf Python package if buildAll || hasArg cudf; then diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 14dc7a59048..c67d127e635 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -29,7 +29,7 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libcudf cudf dask-cudf + libcudf pylibcudf cudf dask-cudf export RAPIDS_DOCS_DIR="$(mktemp -d)" diff --git a/ci/build_python.sh b/ci/build_python.sh index 79e09432779..2e3f70ba767 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -22,9 +22,16 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) # TODO: Remove `--no-test` flag once importing on a CPU # node works correctly # With boa installed conda build forwards to the boa builder + +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + conda/recipes/pylibcudf + RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cudf RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 1b563bc499c..7c0fb1efebe 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -7,10 +7,14 @@ package_dir="python/cudf" export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" +# Download the pylibcudf built in the previous step +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcudf_dist + +echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/pylibcudf_dist/pylibcudf_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" ./ci/build_wheel.sh ${package_dir} python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh new file mode 100755 index 00000000000..b25d118ff81 --- /dev/null +++ b/ci/build_wheel_pylibcudf.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir="python/pylibcudf" + +export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" + +./ci/build_wheel.sh ${package_dir} + +python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* + + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 48ee4a05628..8deaeab78a3 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -11,7 +11,9 @@ rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep +python -m pip install $(ls ./local-pylibcudf-dep/pylibcudf*.whl) python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 1c3b99953fb..bfb655db3ca 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -36,7 +36,9 @@ if [ "$no_cudf" = true ]; then echo "Skipping cudf install" else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep + python -m pip install $(ls ./local-pylibcudf-dep/pylibcudf*.whl) python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests] fi diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index 217dd2fd9a8..ae34047e87f 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -15,7 +15,7 @@ trap "EXITCODE=1" ERR set +e rapids-logger "pytest pylibcudf" -pushd python/cudf/cudf/pylibcudf_tests +pushd python/pylibcudf/pylibcudf/tests python -m pytest \ --cache-clear \ --dist=worksteal \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index fdb61278d36..5a2c3ccac8f 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -3,11 +3,15 @@ set -eou pipefail +# Download the pylibcudf built in the previous step RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cudf*.whl)[test] +# Install both pylibcudf and cudf +python -m pip install \ + "$(echo ./local-pylibcudf-dep/pylibcudf*.whl)[test]" \ + "$(echo ./dist/cudf*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ @@ -15,7 +19,7 @@ mkdir -p "${RAPIDS_TESTS_DIR}" rapids-logger "pytest pylibcudf" -pushd python/cudf/cudf/pylibcudf_tests +pushd python/pylibcudf/pylibcudf/tests python -m pytest \ --cache-clear \ --dist=worksteal \ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index cc9f5788685..357d4170d47 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -10,7 +10,7 @@ set -eou pipefail # files in cudf_polars/pylibcudf", rather than "are there changes # between upstream and this branch which touch cudf_polars/pylibcudf" # TODO: is the target branch exposed anywhere in an environment variable? -if [ -n "$(git diff --name-only origin/branch-24.10...HEAD -- python/cudf_polars/ python/cudf/cudf/_lib/pylibcudf/)" ]; +if [ -n "$(git diff --name-only origin/branch-24.10...HEAD -- python/cudf_polars/ python/pylibcudf/)" ]; then HAS_CHANGES=1 else @@ -21,8 +21,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install ./local-cudf-dep/cudf*.whl +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep +python -m pip install ./local-pylibcudf-dep/pylibcudf*.whl rapids-logger "Install cudf_polars" python -m pip install $(echo ./dist/cudf_polars*.whl)[test] diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index c3800d3cc25..4d045472604 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -7,8 +7,11 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install ./local-cudf-dep/cudf*.whl +python -m pip install \ + "$(echo ./local-pylibcudf-dep/pylibcudf*.whl)" \ + "$(echo ./local-cudf-dep/cudf*.whl)" # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cudf*.whl)[test] diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8d5fc2e31d9..d0d18e57abc 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -56,12 +56,14 @@ dependencies: - ninja - notebook - numba>=0.57 +- numpy - numpy>=1.23,<2.0a0 - numpydoc - nvcc_linux-64=11.8 - nvcomp==3.0.6 - nvtx>=0.2.1 - packaging +- pandas - pandas>=2.0,<2.2.3dev0 - pandoc - pip diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 7b0485d7f29..caf39a32d79 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -55,11 +55,13 @@ dependencies: - ninja - notebook - numba>=0.57 +- numpy - numpy>=1.23,<2.0a0 - numpydoc - nvcomp==3.0.6 - nvtx>=0.2.1 - packaging +- pandas - pandas>=2.0,<2.2.3dev0 - pandoc - pip diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 8d7ef63715b..7e86147732e 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -68,6 +68,7 @@ requirements: - numpy 1.23 - pyarrow ==16.1.0.* - libcudf ={{ version }} + - pylibcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} - cudatoolkit @@ -87,6 +88,7 @@ requirements: - numpy >=1.23,<2.0a0 - {{ pin_compatible('pyarrow', max_pin='x.x') }} - libcudf ={{ version }} + - pylibcudf ={{ version }} - {{ pin_compatible('rmm', max_pin='x.x') }} - fsspec >=0.6.0 {% if cuda_major == "11" %} diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index 748a32e5518..d04d9b21a46 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -58,7 +58,7 @@ requirements: - python - cython >=3.0.3 - cuda-version ={{ cuda_version }} - - cudf ={{ version }} + - pylibcudf ={{ version }} - libcudf_kafka ={{ version }} - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 @@ -69,7 +69,7 @@ requirements: - python - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - libcudf_kafka ={{ version }} - - cudf ={{ version }} + - pylibcudf ={{ version }} {% if cuda_major != "11" %} - cuda-cudart {% endif %} diff --git a/conda/recipes/pylibcudf/build.sh b/conda/recipes/pylibcudf/build.sh new file mode 100644 index 00000000000..483346504db --- /dev/null +++ b/conda/recipes/pylibcudf/build.sh @@ -0,0 +1,4 @@ +# Copyright (c) 2018-2024, NVIDIA CORPORATION. + +# This assumes the script is executed from the root of the repo directory +./build.sh pylibcudf diff --git a/conda/recipes/pylibcudf/conda_build_config.yaml b/conda/recipes/pylibcudf/conda_build_config.yaml new file mode 100644 index 00000000000..af894cccda0 --- /dev/null +++ b/conda/recipes/pylibcudf/conda_build_config.yaml @@ -0,0 +1,20 @@ +c_compiler_version: + - 11 + +cxx_compiler_version: + - 11 + +c_stdlib: + - sysroot + +c_stdlib_version: + - "2.17" + +cmake_version: + - ">=3.26.4,!=3.30.0" + +cuda_compiler: + - cuda-nvcc + +cuda11_compiler: + - nvcc diff --git a/conda/recipes/pylibcudf/meta.yaml b/conda/recipes/pylibcudf/meta.yaml new file mode 100644 index 00000000000..f405fd10f5d --- /dev/null +++ b/conda/recipes/pylibcudf/meta.yaml @@ -0,0 +1,108 @@ +# Copyright (c) 2018-2024, NVIDIA CORPORATION. + +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set py_version = environ['CONDA_PY'] %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} +{% set date_string = environ['RAPIDS_DATE_STRING'] %} + +package: + name: pylibcudf + version: {{ version }} + +source: + path: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + - CMAKE_C_COMPILER_LAUNCHER + - CMAKE_CUDA_COMPILER_LAUNCHER + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL + - SCCACHE_BUCKET + - SCCACHE_IDLE_TIMEOUT + - SCCACHE_REGION + - SCCACHE_S3_KEY_PREFIX=pylibcudf-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=pylibcudf-linux64 # [linux64] + - SCCACHE_S3_USE_SSL + - SCCACHE_S3_NO_CREDENTIALS + ignore_run_exports: + # libcudf's run_exports pinning is looser than we would like + - libcudf + ignore_run_exports_from: + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcufile-dev # [linux64] + {% endif %} + +requirements: + build: + - cmake {{ cmake_version }} + - ninja + - {{ compiler('c') }} + - {{ compiler('cxx') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} ={{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} + {% endif %} + - cuda-version ={{ cuda_version }} + - {{ stdlib("c") }} + host: + - python + - cython >=3.0.3 + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - scikit-build-core >=0.10.0 + - dlpack >=0.8,<1.0 + # TODO: Change to `2.0` for NumPy 2 + - numpy 1.23 + - pyarrow ==16.1.0.* + - libcudf ={{ version }} + - rmm ={{ minor_version }} + {% if cuda_major == "11" %} + - cudatoolkit + {% else %} + - cuda-cudart-dev + - cuda-nvrtc + - libcufile-dev # [linux64] + {% endif %} + - cuda-version ={{ cuda_version }} + run: + - python + - typing_extensions >=4.0.0 + - pandas >=2.0,<2.2.3dev0 + # TODO: Update `numpy` in `host` when dropping `<2.0a0` + - numpy >=1.23,<2.0a0 + - {{ pin_compatible('pyarrow', max_pin='x.x') }} + - {{ pin_compatible('rmm', max_pin='x.x') }} + - fsspec >=0.6.0 + {% if cuda_major == "11" %} + - cuda-python >=11.7.1,<12.0a0 + {% else %} + - cuda-python >=12.0,<13.0a0 + {% endif %} + - nvtx >=0.2.1 + - packaging + +test: + requires: + - cuda-version ={{ cuda_version }} + imports: + - pylibcudf + +about: + home: https://rapids.ai/ + license: Apache-2.0 + license_family: APACHE + license_file: LICENSE + summary: pylibcudf library diff --git a/dependencies.yaml b/dependencies.yaml index b0d62a9fb0d..ca615905a15 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -10,6 +10,7 @@ files: - build_all - build_cpp - build_python_common + - build_python_pylibcudf - build_python_cudf - cuda - cuda_version @@ -22,12 +23,14 @@ files: - rapids_build_setuptools - run_common - run_cudf + - run_pylibcudf - run_dask_cudf - run_custreamz - test_cpp - test_python_common - test_python_cudf - test_python_dask_cudf + - test_python_pylibcudf - depends_on_cupy test_static_build: output: none @@ -76,14 +79,14 @@ files: - docs - libarrow_run - py_version - py_rapids_build_cudf: + py_build_cudf: output: pyproject pyproject_dir: python/cudf extras: table: build-system includes: - rapids_build_skbuild - py_build_cudf: + py_rapids_build_cudf: output: pyproject pyproject_dir: python/cudf extras: @@ -93,6 +96,7 @@ files: - build_base - build_python_common - build_python_cudf + - pylibcudf_build_dep py_run_cudf: output: pyproject pyproject_dir: python/cudf @@ -103,6 +107,7 @@ files: - run_cudf - pyarrow_run - depends_on_cupy + - depends_on_pylibcudf py_test_cudf: output: pyproject pyproject_dir: python/cudf @@ -112,6 +117,40 @@ files: includes: - test_python_common - test_python_cudf + py_rapids_build_pylibcudf: + output: pyproject + pyproject_dir: python/pylibcudf + extras: + table: build-system + includes: + - rapids_build_skbuild + py_build_pylibcudf: + output: pyproject + pyproject_dir: python/pylibcudf + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_base + - build_python_common + - build_python_pylibcudf + py_run_pylibcudf: + output: pyproject + pyproject_dir: python/pylibcudf + extras: + table: project + includes: + - run_pylibcudf + - pyarrow_run + py_test_pylibcudf: + output: pyproject + pyproject_dir: python/pylibcudf + extras: + table: project.optional-dependencies + key: test + includes: + - test_python_common + - test_python_pylibcudf py_test_pandas_cudf: output: pyproject pyproject_dir: python/cudf @@ -142,7 +181,7 @@ files: table: project includes: - run_cudf_polars - - depends_on_cudf + - depends_on_pylibcudf py_test_cudf_polars: output: pyproject pyproject_dir: python/cudf_polars @@ -326,11 +365,36 @@ dependencies: # Sync with conda build constraint & wheel run constraint. # TODO: Change to `2.0.*` for NumPy 2 - numpy==1.23.* - build_python_cudf: + build_python_pylibcudf: common: - output_types: conda packages: - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - rmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - rmm-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*rmm_unsuffixed]} + build_python_cudf: + common: + - output_types: conda + packages: + - *rmm_unsuffixed - pip - pip: - git+https://github.com/python-streamz/streamz.git@master @@ -349,12 +413,33 @@ dependencies: cuda_suffixed: "true" packages: - rmm-cu12==24.10.*,>=0.0.0a0 + - pylibcudf-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - rmm-cu11==24.10.*,>=0.0.0a0 + - pylibcudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_unsuffixed]} + pylibcudf_build_dep: + common: + - output_types: conda + packages: + - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0 + specific: + - output_types: [pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - pylibcudf-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - pylibcudf-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*pylibcudf_unsuffixed]} libarrow_build: common: - output_types: conda @@ -560,6 +645,45 @@ dependencies: # TODO: Update `numpy` in `build_python_common` when dropping `<2.0a0` - numpy>=1.23,<2.0a0 - pandas>=2.0,<2.2.3dev0 + run_pylibcudf: + common: + - output_types: [conda, requirements, pyproject] + packages: + - nvtx>=0.2.1 + - packaging + - typing_extensions>=4.0.0 + - output_types: conda + packages: + - *rmm_unsuffixed + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [conda, requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - cuda-python>=12.0,<13.0a0 + - matrix: {cuda: "11.*"} + packages: &run_pylibcudf_packages_all_cu11 + - cuda-python>=11.7.1,<12.0a0 + - {matrix: null, packages: *run_pylibcudf_packages_all_cu11} + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - rmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - rmm-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*rmm_unsuffixed]} run_cudf: common: - output_types: [conda, requirements, pyproject] @@ -680,6 +804,14 @@ dependencies: - pytest<8 - pytest-cov - pytest-xdist + test_python_pylibcudf: + common: + - output_types: [conda, requirements, pyproject] + packages: + - fastavro>=0.22.9 + - hypothesis + - numpy + - pandas test_python_cudf: common: - output_types: [conda, requirements, pyproject] @@ -724,6 +856,31 @@ dependencies: packages: - dask-cuda==24.10.*,>=0.0.0a0 - *numba + depends_on_pylibcudf: + common: + - output_types: conda + packages: + - *pylibcudf_unsuffixed + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm, cubinlinker, ptxcompiler. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - pylibcudf-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - pylibcudf-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*pylibcudf_unsuffixed]} depends_on_cudf: common: - output_types: conda diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst index 739305af5d4..4b2b213b6c3 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst @@ -2,5 +2,5 @@ aggregation =========== -.. automodule:: cudf._lib.pylibcudf.aggregation +.. automodule:: pylibcudf.aggregation :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst index e5bc6aa7cda..8bbbfbf88c1 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst @@ -2,5 +2,5 @@ binaryop ======== -.. automodule:: cudf._lib.pylibcudf.binaryop +.. automodule:: pylibcudf.binaryop :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/column.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/column.rst index d1105d356b4..d26c8737cf4 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/column.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/column.rst @@ -2,5 +2,5 @@ Column ====== -.. automodule:: cudf._lib.pylibcudf.column +.. automodule:: pylibcudf.column :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/column_factories.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/column_factories.rst index c858135b6ce..8dfaa4bae03 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/column_factories.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/column_factories.rst @@ -2,5 +2,5 @@ column_factories ================ -.. automodule:: cudf._lib.pylibcudf.column_factories +.. automodule:: pylibcudf.column_factories :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst index e83739056f4..7912cb83767 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst @@ -2,5 +2,5 @@ concatenate =========== -.. automodule:: cudf._lib.pylibcudf.concatenate +.. automodule:: pylibcudf.concatenate :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/copying.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/copying.rst index fddd3ea440f..25e3ef50e6a 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/copying.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/copying.rst @@ -2,5 +2,5 @@ copying ======= -.. automodule:: cudf._lib.pylibcudf.copying +.. automodule:: pylibcudf.copying :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst index 558268ea495..71f7874cfbe 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst @@ -2,5 +2,5 @@ datetime ======== -.. automodule:: cudf._lib.pylibcudf.datetime +.. automodule:: pylibcudf.datetime :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst index 03f769ee861..5493d4662a9 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst @@ -2,5 +2,5 @@ expressions =========== -.. automodule:: cudf._lib.pylibcudf.expressions +.. automodule:: pylibcudf.expressions :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst index 542a5e12bc4..0d328a0b0e9 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst @@ -2,5 +2,5 @@ filling ======== -.. automodule:: cudf._lib.pylibcudf.filling +.. automodule:: pylibcudf.filling :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/gpumemoryview.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/gpumemoryview.rst index dffc7c24e02..5515a74adcc 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/gpumemoryview.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/gpumemoryview.rst @@ -2,5 +2,5 @@ gpumemoryview ============= -.. automodule:: cudf._lib.pylibcudf.gpumemoryview +.. automodule:: pylibcudf.gpumemoryview :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst index d6e994f7dbc..27cda383818 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst @@ -2,5 +2,5 @@ groupby ======= -.. automodule:: cudf._lib.pylibcudf.groupby +.. automodule:: pylibcudf.groupby :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/interop.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/interop.rst index 881ab8d7be4..0d2cb55212e 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/interop.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/interop.rst @@ -2,5 +2,5 @@ interop ======= -.. automodule:: cudf._lib.pylibcudf.interop +.. automodule:: pylibcudf.interop :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/avro.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/avro.rst index 495bd505fdc..1c57a6157f5 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/avro.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/avro.rst @@ -2,5 +2,5 @@ Avro ==== -.. automodule:: cudf._lib.pylibcudf.io.avro +.. automodule:: pylibcudf.io.avro :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst index 5a2276f8b2d..59f7d8fe54c 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst @@ -2,5 +2,5 @@ CSV === -.. automodule:: cudf._lib.pylibcudf.io.csv +.. automodule:: pylibcudf.io.csv :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst index e2d342ffe47..c8933981736 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst @@ -5,7 +5,7 @@ I/O I/O Utility Classes =================== -.. automodule:: cudf._lib.pylibcudf.io.types +.. automodule:: pylibcudf.io.types :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/json.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/json.rst index 6aeae1f322a..a4626f43cc3 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/json.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/json.rst @@ -2,5 +2,5 @@ JSON ==== -.. automodule:: cudf._lib.pylibcudf.io.json +.. automodule:: pylibcudf.io.json :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst index 9dfbadfa216..07c2503ab28 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst @@ -2,5 +2,5 @@ Parquet ======= -.. automodule:: cudf._lib.pylibcudf.io.parquet +.. automodule:: pylibcudf.io.parquet :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst index 05b9709d116..de065e4fc40 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst @@ -2,5 +2,5 @@ join ==== -.. automodule:: cudf._lib.pylibcudf.join +.. automodule:: pylibcudf.join :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst index a127dd6006a..0fe1a876073 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst @@ -2,5 +2,5 @@ lists ===== -.. automodule:: cudf._lib.pylibcudf.lists +.. automodule:: pylibcudf.lists :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst index ef1189a064a..3f634ffcfd7 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst @@ -2,5 +2,5 @@ merge ===== -.. automodule:: cudf._lib.pylibcudf.merge +.. automodule:: pylibcudf.merge :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/quantiles.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/quantiles.rst index 3417c1ff59d..0f0f701b5dc 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/quantiles.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/quantiles.rst @@ -2,5 +2,5 @@ quantiles ========= -.. automodule:: cudf._lib.pylibcudf.quantiles +.. automodule:: pylibcudf.quantiles :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst index e6f1b02331d..047f217c276 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst @@ -2,5 +2,5 @@ reduce ====== -.. automodule:: cudf._lib.pylibcudf.reduce +.. automodule:: pylibcudf.reduce :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst index 7f846872fca..7410b20e1b0 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst @@ -2,5 +2,5 @@ replace ======= -.. automodule:: cudf._lib.pylibcudf.replace +.. automodule:: pylibcudf.replace :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/reshape.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/reshape.rst index 964cef04923..09ec0501bb9 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/reshape.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/reshape.rst @@ -2,5 +2,5 @@ reshape ======= -.. automodule:: cudf._lib.pylibcudf.reshape +.. automodule:: pylibcudf.reshape :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst index 0817d117a94..1f8da467e84 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst @@ -2,5 +2,5 @@ rolling ======= -.. automodule:: cudf._lib.pylibcudf.rolling +.. automodule:: pylibcudf.rolling :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/round.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/round.rst index c97fda12301..e064357cbd1 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/round.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/round.rst @@ -2,5 +2,5 @@ round ===== -.. automodule:: cudf._lib.pylibcudf.round +.. automodule:: pylibcudf.round :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/scalar.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/scalar.rst index b12f47618fb..a9100c6bb2d 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/scalar.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/scalar.rst @@ -2,5 +2,5 @@ Scalar ====== -.. automodule:: cudf._lib.pylibcudf.scalar +.. automodule:: pylibcudf.scalar :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst index aa57bcd9d92..02307037994 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst @@ -2,5 +2,5 @@ search ====== -.. automodule:: cudf._lib.pylibcudf.search +.. automodule:: pylibcudf.search :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst index e9441366eeb..b8fd8fda9bd 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst @@ -2,5 +2,5 @@ sorting ======= -.. automodule:: cudf._lib.pylibcudf.sorting +.. automodule:: pylibcudf.sorting :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst index 00b479446d8..0252d0684d9 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst @@ -2,5 +2,5 @@ stream_compaction ================= -.. automodule:: cudf._lib.pylibcudf.stream_compaction +.. automodule:: pylibcudf.stream_compaction :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst index 578b2b75e37..6b9ed8d47e7 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/capitalize.rst @@ -2,5 +2,5 @@ capitalize ========== -.. automodule:: cudf._lib.pylibcudf.strings.capitalize +.. automodule:: pylibcudf.strings.capitalize :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst index 577ec34915b..896fa6086db 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/char_types.rst @@ -2,5 +2,5 @@ char_types ========== -.. automodule:: cudf._lib.pylibcudf.strings.char_types +.. automodule:: pylibcudf.strings.char_types :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/contains.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/contains.rst index e5745331bc7..d2d164be638 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/contains.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/contains.rst @@ -2,5 +2,5 @@ contains ======== -.. automodule:: cudf._lib.pylibcudf.strings.contains +.. automodule:: pylibcudf.strings.contains :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst index 61d4079e9a3..7c540e99929 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/find.rst @@ -2,5 +2,5 @@ find ==== -.. automodule:: cudf._lib.pylibcudf.strings.find +.. automodule:: pylibcudf.strings.find :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst index 0126b6a3706..53fd712d864 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_flags.rst @@ -2,5 +2,5 @@ regex_flags =========== -.. automodule:: cudf._lib.pylibcudf.strings.regex_flags +.. automodule:: pylibcudf.strings.regex_flags :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst index 2f398186d51..6f3d2f6681c 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/regex_program.rst @@ -2,5 +2,5 @@ regex_program ============= -.. automodule:: cudf._lib.pylibcudf.strings.regex_program +.. automodule:: pylibcudf.strings.regex_program :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/replace.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/replace.rst index 9575ec226a7..d5417adac43 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/replace.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/replace.rst @@ -2,5 +2,5 @@ replace ======= -.. automodule:: cudf._lib.pylibcudf.strings.replace +.. automodule:: pylibcudf.strings.replace :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/slice.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/slice.rst index 0ee5af71c03..e9908904512 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/slice.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/slice.rst @@ -2,5 +2,5 @@ slice ===== -.. automodule:: cudf._lib.pylibcudf.strings.slice +.. automodule:: pylibcudf.strings.slice :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/table.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/table.rst index d8337b6596d..e39ca18a12b 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/table.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/table.rst @@ -2,5 +2,5 @@ Table ===== -.. automodule:: cudf._lib.pylibcudf.table +.. automodule:: pylibcudf.table :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/traits.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/traits.rst index 294ca8dc78c..2cce7b9d7d7 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/traits.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/traits.rst @@ -2,5 +2,5 @@ traits ====== -.. automodule:: cudf._lib.pylibcudf.traits +.. automodule:: pylibcudf.traits :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/transform.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/transform.rst index ef04bbad7e6..839163f83fc 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/transform.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/transform.rst @@ -2,5 +2,5 @@ transform ========= -.. automodule:: cudf._lib.pylibcudf.transform +.. automodule:: pylibcudf.transform :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/types.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/types.rst index 8d5409bbd97..75521ac2f4d 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/types.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/types.rst @@ -2,5 +2,5 @@ types ===== -.. automodule:: cudf._lib.pylibcudf.types +.. automodule:: pylibcudf.types :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst index add4baa0a54..34077242b90 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst @@ -2,5 +2,5 @@ unary ===== -.. automodule:: cudf._lib.pylibcudf.unary +.. automodule:: pylibcudf.unary :members: diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index ecadbf5cbbc..e11d62b3bd5 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -79,7 +79,7 @@ if(NOT cudf_FOUND) # require access to libcudf, we place the library and all its dependent artifacts in the cudf # directory as a single source of truth and modify the other rpaths appropriately. set(cython_lib_dir cudf) - include(cmake/Modules/WheelHelpers.cmake) + include(../pylibcudf/cmake/Modules/WheelHelpers.cmake) # TODO: This install is currently overzealous. We should only install the libraries that are # downloaded by CPM during the build, not libraries that were found on the system. However, in # practice right this would only be a problem is if libcudf was not found but some of the @@ -92,7 +92,7 @@ endif() rapids_cython_init() -include(cmake/Modules/LinkPyarrowHeaders.cmake) +include(../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) add_subdirectory(cudf/_lib) add_subdirectory(udf_cpp) diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index d32a2d8e3f8..d6182673308 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -73,5 +73,4 @@ target_link_libraries(interop PUBLIC nanoarrow) add_subdirectory(io) add_subdirectory(nvtext) -add_subdirectory(pylibcudf) add_subdirectory(strings) diff --git a/python/cudf/cudf/_lib/__init__.py b/python/cudf/cudf/_lib/__init__.py index 34c0e29d0b1..918edb6d3f1 100644 --- a/python/cudf/cudf/_lib/__init__.py +++ b/python/cudf/cudf/_lib/__init__.py @@ -21,7 +21,6 @@ orc, parquet, partitioning, - pylibcudf, quantiles, reduce, replace, diff --git a/python/cudf/cudf/_lib/aggregation.pyx b/python/cudf/cudf/_lib/aggregation.pyx index 1616c24eec2..7c91533cf93 100644 --- a/python/cudf/cudf/_lib/aggregation.pyx +++ b/python/cudf/cudf/_lib/aggregation.pyx @@ -3,8 +3,9 @@ import pandas as pd from numba.np import numpy_support +import pylibcudf + import cudf -from cudf._lib import pylibcudf from cudf._lib.types import SUPPORTED_NUMPY_TO_PYLIBCUDF_TYPES from cudf.utils import cudautils diff --git a/python/cudf/cudf/_lib/avro.pyx b/python/cudf/cudf/_lib/avro.pyx index 3c132b22880..b1759635a36 100644 --- a/python/cudf/cudf/_lib/avro.pyx +++ b/python/cudf/cudf/_lib/avro.pyx @@ -2,8 +2,8 @@ from cudf._lib.utils cimport data_from_pylibcudf_io -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.io.types import SourceInfo +import pylibcudf as plc +from pylibcudf.io.types import SourceInfo cpdef read_avro(datasource, columns=None, skip_rows=0, num_rows=-1): diff --git a/python/cudf/cudf/_lib/binaryop.pyx b/python/cudf/cudf/_lib/binaryop.pyx index 2e352dd7904..e2547476849 100644 --- a/python/cudf/cudf/_lib/binaryop.pyx +++ b/python/cudf/cudf/_lib/binaryop.pyx @@ -4,7 +4,8 @@ from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar from cudf._lib.types cimport dtype_to_pylibcudf_type -from cudf._lib import pylibcudf +import pylibcudf + from cudf._lib.scalar import as_device_scalar from cudf.core.buffer import acquire_spill_lock diff --git a/python/cudf/cudf/_lib/column.pxd b/python/cudf/cudf/_lib/column.pxd index 437f44af9f0..8ceea4920e2 100644 --- a/python/cudf/cudf/_lib/column.pxd +++ b/python/cudf/cudf/_lib/column.pxd @@ -5,14 +5,13 @@ from typing import Literal from libcpp cimport bool from libcpp.memory cimport unique_ptr -from rmm._lib.device_buffer cimport device_buffer - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type +from rmm._lib.device_buffer cimport device_buffer cdef class Column: diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index f0c07dfbc1b..2e400f775d3 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -7,11 +7,11 @@ import cupy as cp import numpy as np import pandas as pd +import pylibcudf import rmm import cudf import cudf._lib as libcudf -from cudf._lib import pylibcudf from cudf.core.buffer import ( Buffer, ExposureTrackedBuffer, @@ -39,18 +39,18 @@ from cudf._lib.types cimport ( from cudf._lib.null_mask import bitmask_allocation_size_bytes from cudf._lib.types import dtype_from_pylibcudf_column - -cimport cudf._lib.pylibcudf.libcudf.copying as cpp_copying -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -cimport cudf._lib.pylibcudf.libcudf.unary as libcudf_unary -from cudf._lib.pylibcudf.libcudf.column.column cimport column, column_contents -from cudf._lib.pylibcudf.libcudf.column.column_factories cimport ( +cimport pylibcudf.libcudf.copying as cpp_copying +cimport pylibcudf.libcudf.types as libcudf_types +cimport pylibcudf.libcudf.unary as libcudf_unary +from pylibcudf.libcudf.column.column cimport column, column_contents +from pylibcudf.libcudf.column.column_factories cimport ( make_column_from_scalar as cpp_make_column_from_scalar, make_numeric_column, ) -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.null_mask cimport null_count as cpp_null_count -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.null_mask cimport null_count as cpp_null_count +from pylibcudf.libcudf.scalar.scalar cimport scalar + from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/concat.pyx b/python/cudf/cudf/_lib/concat.pyx index 89ddcfee99e..e661059faa3 100644 --- a/python/cudf/cudf/_lib/concat.pyx +++ b/python/cudf/cudf/_lib/concat.pyx @@ -5,7 +5,8 @@ from libcpp cimport bool from cudf._lib.column cimport Column from cudf._lib.utils cimport data_from_pylibcudf_table -from cudf._lib import pylibcudf +import pylibcudf + from cudf.core.buffer import acquire_spill_lock diff --git a/python/cudf/cudf/_lib/copying.pxd b/python/cudf/cudf/_lib/copying.pxd index 8fc7f4e1da0..14c7d2066d8 100644 --- a/python/cudf/cudf/_lib/copying.pxd +++ b/python/cudf/cudf/_lib/copying.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.contiguous_split cimport packed_columns +from pylibcudf.libcudf.contiguous_split cimport packed_columns cdef class _CPackedColumns: diff --git a/python/cudf/cudf/_lib/copying.pyx b/python/cudf/cudf/_lib/copying.pyx index 796c70e615c..16182e31c08 100644 --- a/python/cudf/cudf/_lib/copying.pyx +++ b/python/cudf/cudf/_lib/copying.pyx @@ -10,8 +10,9 @@ from libcpp.vector cimport vector from rmm._lib.device_buffer cimport DeviceBuffer +import pylibcudf + import cudf -from cudf._lib import pylibcudf from cudf.core.buffer import Buffer, acquire_spill_lock, as_buffer from cudf._lib.column cimport Column @@ -26,17 +27,16 @@ from cudf.core.abc import Serializable from libcpp.memory cimport make_unique -cimport cudf._lib.pylibcudf.libcudf.contiguous_split as cpp_contiguous_split -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.lists.gather cimport ( +cimport pylibcudf.libcudf.contiguous_split as cpp_contiguous_split +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.lists.gather cimport ( segmented_gather as cpp_segmented_gather, ) -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.utils cimport columns_from_pylibcudf_table, data_from_table_view # workaround for https://github.com/cython/cython/issues/3885 diff --git a/python/cudf/cudf/_lib/csv.pyx b/python/cudf/cudf/_lib/csv.pyx index 099b61d62ae..e9aa97ecbc9 100644 --- a/python/cudf/cudf/_lib/csv.pyx +++ b/python/cudf/cudf/_lib/csv.pyx @@ -6,8 +6,9 @@ from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.pylibcudf.io.datasource cimport Datasource, NativeFileDatasource +cimport pylibcudf.libcudf.types as libcudf_types +from pylibcudf.io.datasource cimport Datasource, NativeFileDatasource + from cudf._lib.types cimport dtype_to_pylibcudf_type import errno @@ -23,22 +24,24 @@ from cudf.core.buffer import acquire_spill_lock from libcpp cimport bool -from cudf._lib.io.utils cimport make_sink_info -from cudf._lib.pylibcudf.libcudf.io.csv cimport ( +from pylibcudf.libcudf.io.csv cimport ( csv_writer_options, write_csv as cpp_write_csv, ) -from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink -from cudf._lib.pylibcudf.libcudf.io.types cimport compression_type, sink_info -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.io.data_sink cimport data_sink +from pylibcudf.libcudf.io.types cimport compression_type, sink_info +from pylibcudf.libcudf.table.table_view cimport table_view + +from cudf._lib.io.utils cimport make_sink_info from cudf._lib.utils cimport data_from_pylibcudf_io, table_view_from_table from pyarrow.lib import NativeFile -import cudf._lib.pylibcudf as plc +import pylibcudf as plc + from cudf.api.types import is_hashable -from cudf._lib.pylibcudf.types cimport DataType +from pylibcudf.types cimport DataType CSV_HEX_TYPE_MAP = { "hex": np.dtype("int64"), diff --git a/python/cudf/cudf/_lib/datetime.pyx b/python/cudf/cudf/_lib/datetime.pyx index b30ef875a7b..483250dd36f 100644 --- a/python/cudf/cudf/_lib/datetime.pyx +++ b/python/cudf/cudf/_lib/datetime.pyx @@ -7,13 +7,14 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -cimport cudf._lib.pylibcudf.libcudf.datetime as libcudf_datetime +cimport pylibcudf.libcudf.datetime as libcudf_datetime +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.filling cimport calendrical_month_sequence +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.filling cimport calendrical_month_sequence -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/filling.pyx b/python/cudf/cudf/_lib/filling.pyx index b7302f3d07a..b2f4c620144 100644 --- a/python/cudf/cudf/_lib/filling.pyx +++ b/python/cudf/cudf/_lib/filling.pyx @@ -2,12 +2,12 @@ from cudf.core.buffer import acquire_spill_lock - from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar from cudf._lib.utils cimport columns_from_pylibcudf_table -from cudf._lib import pylibcudf +import pylibcudf + from cudf._lib.scalar import as_device_scalar diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index 9d18e023fe8..c199ed96d4f 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -18,10 +18,11 @@ from cudf._lib.utils cimport columns_from_pylibcudf_table from cudf._lib.scalar import as_device_scalar -from cudf._lib.pylibcudf.libcudf.replace cimport replace_policy -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.replace cimport replace_policy +from pylibcudf.libcudf.scalar.scalar cimport scalar + +import pylibcudf -from cudf._lib import pylibcudf from cudf._lib.aggregation import make_aggregation # The sets below define the possible aggregations that can be performed on diff --git a/python/cudf/cudf/_lib/hash.pyx b/python/cudf/cudf/_lib/hash.pyx index b8331d5a226..48f75b12a73 100644 --- a/python/cudf/cudf/_lib/hash.pyx +++ b/python/cudf/cudf/_lib/hash.pyx @@ -7,10 +7,9 @@ from libcpp.pair cimport pair from libcpp.utility cimport move from libcpp.vector cimport vector -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.hash cimport ( +cimport pylibcudf.libcudf.types as libcudf_types +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.hash cimport ( md5, murmurhash3_x86_32, sha1, @@ -20,11 +19,13 @@ from cudf._lib.pylibcudf.libcudf.hash cimport ( sha512, xxhash_64, ) -from cudf._lib.pylibcudf.libcudf.partitioning cimport ( +from pylibcudf.libcudf.partitioning cimport ( hash_partition as cpp_hash_partition, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view + +from cudf._lib.column cimport Column from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns diff --git a/python/cudf/cudf/_lib/interop.pyx b/python/cudf/cudf/_lib/interop.pyx index 37595b65e65..1dc586bb257 100644 --- a/python/cudf/cudf/_lib/interop.pyx +++ b/python/cudf/cudf/_lib/interop.pyx @@ -4,15 +4,16 @@ from cpython cimport pycapsule from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib import pylibcudf +import pylibcudf -from cudf._lib.pylibcudf.libcudf.interop cimport ( +from pylibcudf.libcudf.interop cimport ( DLManagedTensor, from_dlpack as cpp_from_dlpack, to_dlpack as cpp_to_dlpack, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view + from cudf._lib.utils cimport ( columns_from_pylibcudf_table, columns_from_unique_ptr, diff --git a/python/cudf/cudf/_lib/io/utils.pxd b/python/cudf/cudf/_lib/io/utils.pxd index 680a87c789e..1938f00c179 100644 --- a/python/cudf/cudf/_lib/io/utils.pxd +++ b/python/cudf/cudf/_lib/io/utils.pxd @@ -3,14 +3,15 @@ from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.libcudf.io.data_sink cimport data_sink +from pylibcudf.libcudf.io.types cimport ( column_name_info, sink_info, source_info, ) +from cudf._lib.column cimport Column + cdef source_info make_source_info(list src) except* cdef sink_info make_sinks_info( diff --git a/python/cudf/cudf/_lib/io/utils.pyx b/python/cudf/cudf/_lib/io/utils.pyx index 58956b9e9b7..b1900138d94 100644 --- a/python/cudf/cudf/_lib/io/utils.pyx +++ b/python/cudf/cudf/_lib/io/utils.pyx @@ -7,17 +7,18 @@ from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.io.datasource cimport Datasource -from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink -from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.io.datasource cimport Datasource +from pylibcudf.libcudf.io.data_sink cimport data_sink +from pylibcudf.libcudf.io.datasource cimport datasource +from pylibcudf.libcudf.io.types cimport ( column_name_info, host_buffer, sink_info, source_info, ) +from cudf._lib.column cimport Column + import codecs import errno import io diff --git a/python/cudf/cudf/_lib/join.pyx b/python/cudf/cudf/_lib/join.pyx index 0a54f0d67a0..2559358c21f 100644 --- a/python/cudf/cudf/_lib/join.pyx +++ b/python/cudf/cudf/_lib/join.pyx @@ -4,7 +4,7 @@ from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column -from cudf._lib import pylibcudf +import pylibcudf # The functions below return the *gathermaps* that represent # the join result when joining on the keys `lhs` and `rhs`. diff --git a/python/cudf/cudf/_lib/json.pyx b/python/cudf/cudf/_lib/json.pyx index 03bf9ed8b75..9bbbcf60dcf 100644 --- a/python/cudf/cudf/_lib/json.pyx +++ b/python/cudf/cudf/_lib/json.pyx @@ -9,18 +9,19 @@ from cudf.core.buffer import acquire_spill_lock from libcpp cimport bool -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types +cimport pylibcudf.libcudf.io.types as cudf_io_types +from pylibcudf.io.types cimport compression_type +from pylibcudf.libcudf.io.json cimport json_recovery_mode_t +from pylibcudf.libcudf.io.types cimport compression_type +from pylibcudf.libcudf.types cimport data_type, type_id +from pylibcudf.types cimport DataType + from cudf._lib.column cimport Column from cudf._lib.io.utils cimport add_df_col_struct_names -from cudf._lib.pylibcudf.io.types cimport compression_type -from cudf._lib.pylibcudf.libcudf.io.json cimport json_recovery_mode_t -from cudf._lib.pylibcudf.libcudf.io.types cimport compression_type -from cudf._lib.pylibcudf.libcudf.types cimport data_type, type_id -from cudf._lib.pylibcudf.types cimport DataType from cudf._lib.types cimport dtype_to_data_type from cudf._lib.utils cimport _data_from_columns, data_from_pylibcudf_io -import cudf._lib.pylibcudf as plc +import pylibcudf as plc cdef json_recovery_mode_t _get_json_recovery_mode(object on_bad_lines): diff --git a/python/cudf/cudf/_lib/labeling.pyx b/python/cudf/cudf/_lib/labeling.pyx index 439a727a9ca..2e1959a348d 100644 --- a/python/cudf/cudf/_lib/labeling.pyx +++ b/python/cudf/cudf/_lib/labeling.pyx @@ -6,13 +6,11 @@ from libcpp cimport bool as cbool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.labeling cimport inclusive, label_bins as cpp_label_bins + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.labeling cimport ( - inclusive, - label_bins as cpp_label_bins, -) # Note that the parameter input shadows a Python built-in in the local scope, diff --git a/python/cudf/cudf/_lib/lists.pyx b/python/cudf/cudf/_lib/lists.pyx index f6d9c8c404c..7e8710bedb6 100644 --- a/python/cudf/cudf/_lib/lists.pyx +++ b/python/cudf/cudf/_lib/lists.pyx @@ -4,13 +4,14 @@ from cudf.core.buffer import acquire_spill_lock from libcpp cimport bool +from pylibcudf.libcudf.types cimport null_order, size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.types cimport null_order, size_type from cudf._lib.utils cimport columns_from_pylibcudf_table -from cudf._lib import pylibcudf +import pylibcudf -from cudf._lib.pylibcudf cimport Scalar +from pylibcudf cimport Scalar @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/merge.pyx b/python/cudf/cudf/_lib/merge.pyx index fe7f7ad2918..9372acdab44 100644 --- a/python/cudf/cudf/_lib/merge.pyx +++ b/python/cudf/cudf/_lib/merge.pyx @@ -4,7 +4,7 @@ from libcpp cimport bool from cudf._lib.utils cimport columns_from_pylibcudf_table -from cudf._lib import pylibcudf +import pylibcudf def merge_sorted( diff --git a/python/cudf/cudf/_lib/null_mask.pyx b/python/cudf/cudf/_lib/null_mask.pyx index b00deae2270..3a7b6a59bf3 100644 --- a/python/cudf/cudf/_lib/null_mask.pyx +++ b/python/cudf/cudf/_lib/null_mask.pyx @@ -10,9 +10,8 @@ from libcpp.memory cimport make_unique, unique_ptr from libcpp.pair cimport pair from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.null_mask cimport ( +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.null_mask cimport ( bitmask_allocation_size_bytes as cpp_bitmask_allocation_size_bytes, bitmask_and as cpp_bitmask_and, bitmask_or as cpp_bitmask_or, @@ -20,8 +19,10 @@ from cudf._lib.pylibcudf.libcudf.null_mask cimport ( create_null_mask as cpp_create_null_mask, underlying_type_t_mask_state, ) -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport mask_state, size_type +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport mask_state, size_type + +from cudf._lib.column cimport Column from cudf._lib.utils cimport table_view_from_columns diff --git a/python/cudf/cudf/_lib/nvtext/byte_pair_encode.pyx b/python/cudf/cudf/_lib/nvtext/byte_pair_encode.pyx index d60162d0656..0d768e24f39 100644 --- a/python/cudf/cudf/_lib/nvtext/byte_pair_encode.pyx +++ b/python/cudf/cudf/_lib/nvtext/byte_pair_encode.pyx @@ -6,15 +6,16 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.byte_pair_encode cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.byte_pair_encode cimport ( bpe_merge_pairs as cpp_bpe_merge_pairs, byte_pair_encoding as cpp_byte_pair_encoding, load_merge_pairs as cpp_load_merge_pairs, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.scalar.scalar cimport string_scalar + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/nvtext/edit_distance.pyx b/python/cudf/cudf/_lib/nvtext/edit_distance.pyx index 514b6610575..e3c2273345a 100644 --- a/python/cudf/cudf/_lib/nvtext/edit_distance.pyx +++ b/python/cudf/cudf/_lib/nvtext/edit_distance.pyx @@ -5,14 +5,15 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.edit_distance cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.edit_distance cimport ( edit_distance as cpp_edit_distance, edit_distance_matrix as cpp_edit_distance_matrix, ) +from cudf._lib.column cimport Column + @acquire_spill_lock() def edit_distance(Column strings, Column targets): diff --git a/python/cudf/cudf/_lib/nvtext/generate_ngrams.pyx b/python/cudf/cudf/_lib/nvtext/generate_ngrams.pyx index a6b9a1e4f7a..6591b527eec 100644 --- a/python/cudf/cudf/_lib/nvtext/generate_ngrams.pyx +++ b/python/cudf/cudf/_lib/nvtext/generate_ngrams.pyx @@ -5,16 +5,17 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.generate_ngrams cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.generate_ngrams cimport ( generate_character_ngrams as cpp_generate_character_ngrams, generate_ngrams as cpp_generate_ngrams, hash_character_ngrams as cpp_hash_character_ngrams, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/nvtext/jaccard.pyx b/python/cudf/cudf/_lib/nvtext/jaccard.pyx index 42fe15d6869..0ebf7c281e3 100644 --- a/python/cudf/cudf/_lib/nvtext/jaccard.pyx +++ b/python/cudf/cudf/_lib/nvtext/jaccard.pyx @@ -5,13 +5,14 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.jaccard cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.jaccard cimport ( jaccard_index as cpp_jaccard_index, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/nvtext/minhash.pyx b/python/cudf/cudf/_lib/nvtext/minhash.pyx index 4c92999e190..5ee15d0e409 100644 --- a/python/cudf/cudf/_lib/nvtext/minhash.pyx +++ b/python/cudf/cudf/_lib/nvtext/minhash.pyx @@ -5,14 +5,15 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.minhash cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.minhash cimport ( minhash as cpp_minhash, minhash64 as cpp_minhash64, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx b/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx index ccd8de8c96f..dec4f037d98 100644 --- a/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx +++ b/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx @@ -5,14 +5,15 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.ngrams_tokenize cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.ngrams_tokenize cimport ( ngrams_tokenize as cpp_ngrams_tokenize, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/nvtext/normalize.pyx b/python/cudf/cudf/_lib/nvtext/normalize.pyx index 9f81f865bb7..5e86a9ce959 100644 --- a/python/cudf/cudf/_lib/nvtext/normalize.pyx +++ b/python/cudf/cudf/_lib/nvtext/normalize.pyx @@ -6,14 +6,15 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.normalize cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.normalize cimport ( normalize_characters as cpp_normalize_characters, normalize_spaces as cpp_normalize_spaces, ) +from cudf._lib.column cimport Column + @acquire_spill_lock() def normalize_spaces(Column strings): diff --git a/python/cudf/cudf/_lib/nvtext/replace.pyx b/python/cudf/cudf/_lib/nvtext/replace.pyx index ce2edc58d19..61ae3da5782 100644 --- a/python/cudf/cudf/_lib/nvtext/replace.pyx +++ b/python/cudf/cudf/_lib/nvtext/replace.pyx @@ -5,15 +5,16 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.replace cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.replace cimport ( filter_tokens as cpp_filter_tokens, replace_tokens as cpp_replace_tokens, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/nvtext/stemmer.pyx b/python/cudf/cudf/_lib/nvtext/stemmer.pyx index 8f75953ae99..5bf25562fed 100644 --- a/python/cudf/cudf/_lib/nvtext/stemmer.pyx +++ b/python/cudf/cudf/_lib/nvtext/stemmer.pyx @@ -7,16 +7,17 @@ from libcpp.utility cimport move from enum import IntEnum -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.stemmer cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.stemmer cimport ( is_letter as cpp_is_letter, letter_type, porter_stemmer_measure as cpp_porter_stemmer_measure, underlying_type_t_letter_type, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column class LetterType(IntEnum): diff --git a/python/cudf/cudf/_lib/nvtext/subword_tokenize.pyx b/python/cudf/cudf/_lib/nvtext/subword_tokenize.pyx index 1112667a087..ee442ece5c6 100644 --- a/python/cudf/cudf/_lib/nvtext/subword_tokenize.pyx +++ b/python/cudf/cudf/_lib/nvtext/subword_tokenize.pyx @@ -9,9 +9,8 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.subword_tokenize cimport ( +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.subword_tokenize cimport ( hashed_vocabulary as cpp_hashed_vocabulary, load_vocabulary_file as cpp_load_vocabulary_file, move as tr_move, @@ -19,6 +18,8 @@ from cudf._lib.pylibcudf.libcudf.nvtext.subword_tokenize cimport ( tokenizer_result as cpp_tokenizer_result, ) +from cudf._lib.column cimport Column + cdef class Hashed_Vocabulary: cdef unique_ptr[cpp_hashed_vocabulary] c_obj diff --git a/python/cudf/cudf/_lib/nvtext/tokenize.pyx b/python/cudf/cudf/_lib/nvtext/tokenize.pyx index 98afd94ab1c..a7e63f1e9ae 100644 --- a/python/cudf/cudf/_lib/nvtext/tokenize.pyx +++ b/python/cudf/cudf/_lib/nvtext/tokenize.pyx @@ -5,10 +5,9 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.nvtext.tokenize cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.nvtext.tokenize cimport ( character_tokenize as cpp_character_tokenize, count_tokens as cpp_count_tokens, detokenize as cpp_detokenize, @@ -17,8 +16,10 @@ from cudf._lib.pylibcudf.libcudf.nvtext.tokenize cimport ( tokenize_vocabulary as cpp_tokenize_vocabulary, tokenize_with_vocabulary as cpp_tokenize_with_vocabulary, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/orc.pyx b/python/cudf/cudf/_lib/orc.pyx index 9609e3131b4..d506dcd4346 100644 --- a/python/cudf/cudf/_lib/orc.pyx +++ b/python/cudf/cudf/_lib/orc.pyx @@ -14,23 +14,17 @@ from libcpp.vector cimport vector import datetime from collections import OrderedDict -cimport cudf._lib.pylibcudf.libcudf.lists.lists_column_view as cpp_lists_column_view +cimport pylibcudf.libcudf.lists.lists_column_view as cpp_lists_column_view try: import ujson as json except ImportError: import json -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -from cudf._lib.column cimport Column -from cudf._lib.io.utils cimport ( - make_sink_info, - make_source_info, - update_column_struct_field_names, -) -from cudf._lib.pylibcudf.io.datasource cimport NativeFileDatasource -from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink -from cudf._lib.pylibcudf.libcudf.io.orc cimport ( +cimport pylibcudf.libcudf.io.types as cudf_io_types +from pylibcudf.io.datasource cimport NativeFileDatasource +from pylibcudf.libcudf.io.data_sink cimport data_sink +from pylibcudf.libcudf.io.orc cimport ( chunked_orc_writer_options, orc_chunked_writer, orc_reader_options, @@ -38,7 +32,7 @@ from cudf._lib.pylibcudf.libcudf.io.orc cimport ( read_orc as libcudf_read_orc, write_orc as libcudf_write_orc, ) -from cudf._lib.pylibcudf.libcudf.io.orc_metadata cimport ( +from pylibcudf.libcudf.io.orc_metadata cimport ( binary_statistics, bucket_statistics, column_statistics, @@ -53,7 +47,7 @@ from cudf._lib.pylibcudf.libcudf.io.orc_metadata cimport ( string_statistics, timestamp_statistics, ) -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.libcudf.io.types cimport ( column_in_metadata, compression_type, sink_info, @@ -61,9 +55,16 @@ from cudf._lib.pylibcudf.libcudf.io.types cimport ( table_input_metadata, table_with_metadata, ) -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type, type_id -from cudf._lib.variant cimport get_if as std_get_if, holds_alternative +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport data_type, size_type, type_id +from pylibcudf.variant cimport get_if as std_get_if, holds_alternative + +from cudf._lib.column cimport Column +from cudf._lib.io.utils cimport ( + make_sink_info, + make_source_info, + update_column_struct_field_names, +) from cudf._lib.types import SUPPORTED_NUMPY_TO_LIBCUDF_TYPES diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index 0fffb6ade58..4bfb79ff651 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -31,40 +31,43 @@ from libcpp.unordered_map cimport unordered_map from libcpp.utility cimport move from libcpp.vector cimport vector -cimport cudf._lib.pylibcudf.libcudf.io.data_sink as cudf_io_data_sink -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -from cudf._lib.column cimport Column -from cudf._lib.io.utils cimport ( - add_df_col_struct_names, - make_sinks_info, - make_source_info, -) -from cudf._lib.pylibcudf.expressions cimport Expression -from cudf._lib.pylibcudf.io.datasource cimport NativeFileDatasource -from cudf._lib.pylibcudf.io.parquet cimport ChunkedParquetReader -from cudf._lib.pylibcudf.libcudf.io.parquet cimport ( +cimport pylibcudf.libcudf.io.data_sink as cudf_io_data_sink +cimport pylibcudf.libcudf.io.types as cudf_io_types +from pylibcudf.expressions cimport Expression +from pylibcudf.io.datasource cimport NativeFileDatasource +from pylibcudf.io.parquet cimport ChunkedParquetReader +from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_writer_options, merge_row_group_metadata as parquet_merge_metadata, parquet_chunked_writer as cpp_parquet_chunked_writer, parquet_writer_options, write_parquet as parquet_writer, ) -from cudf._lib.pylibcudf.libcudf.io.parquet_metadata cimport ( +from pylibcudf.libcudf.io.parquet_metadata cimport ( parquet_metadata, read_parquet_metadata as parquet_metadata_reader, ) -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.libcudf.io.types cimport ( column_in_metadata, table_input_metadata, ) -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column +from cudf._lib.io.utils cimport ( + add_df_col_struct_names, + make_sinks_info, + make_source_info, +) from cudf._lib.utils cimport table_view_from_table from pyarrow.lib import NativeFile -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf cimport Table +import pylibcudf as plc + +from pylibcudf cimport Table + from cudf.utils.ioutils import _ROW_GROUP_SIZE_BYTES_DEFAULT diff --git a/python/cudf/cudf/_lib/partitioning.pyx b/python/cudf/cudf/_lib/partitioning.pyx index 708ec4174aa..d94f0e1b564 100644 --- a/python/cudf/cudf/_lib/partitioning.pyx +++ b/python/cudf/cudf/_lib/partitioning.pyx @@ -7,19 +7,18 @@ from libcpp.pair cimport pair from libcpp.utility cimport move from libcpp.vector cimport vector +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.partitioning cimport partition as cpp_partition +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.partitioning cimport ( - partition as cpp_partition, -) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns from cudf._lib.reduce import minmax from cudf._lib.stream_compaction import distinct_count as cpp_distinct_count -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types +cimport pylibcudf.libcudf.types as libcudf_types @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/pylibcudf/io/avro.pxd b/python/cudf/cudf/_lib/pylibcudf/io/avro.pxd deleted file mode 100644 index 3695f36a6e7..00000000000 --- a/python/cudf/cudf/_lib/pylibcudf/io/avro.pxd +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata -from cudf._lib.pylibcudf.libcudf.io.avro cimport avro_reader_options -from cudf._lib.pylibcudf.libcudf.types cimport size_type - - -cpdef TableWithMetadata read_avro( - SourceInfo source_info, - list columns = *, - size_type skip_rows = *, - size_type num_rows = * -) diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/extract.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/extract.pxd deleted file mode 100644 index 57903ca27de..00000000000 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/extract.pxd +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. - -from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program -from cudf._lib.pylibcudf.libcudf.table.table cimport table - - -cdef extern from "cudf/strings/extract.hpp" namespace "cudf::strings" nogil: - - cdef unique_ptr[table] extract( - column_view source_strings, - regex_program) except + diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/char_types.pxd b/python/cudf/cudf/_lib/pylibcudf/strings/char_types.pxd deleted file mode 100644 index a80e02f520c..00000000000 --- a/python/cudf/cudf/_lib/pylibcudf/strings/char_types.pxd +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. - -from cudf._lib.pylibcudf.libcudf.strings.char_types cimport ( - string_character_types, -) diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/contains.pxd b/python/cudf/cudf/_lib/pylibcudf/strings/contains.pxd deleted file mode 100644 index 275aa95d97e..00000000000 --- a/python/cudf/cudf/_lib/pylibcudf/strings/contains.pxd +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. - -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.strings.regex_program cimport RegexProgram - - -cpdef Column contains_re(Column input, RegexProgram prog) diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/regex_flags.pxd b/python/cudf/cudf/_lib/pylibcudf/strings/regex_flags.pxd deleted file mode 100644 index 79937bf574a..00000000000 --- a/python/cudf/cudf/_lib/pylibcudf/strings/regex_flags.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags diff --git a/python/cudf/cudf/_lib/quantiles.pyx b/python/cudf/cudf/_lib/quantiles.pyx index 7b50c00919a..7666b7ff8da 100644 --- a/python/cudf/cudf/_lib/quantiles.pyx +++ b/python/cudf/cudf/_lib/quantiles.pyx @@ -13,10 +13,11 @@ from cudf._lib.types cimport ( from cudf._lib.types import Interpolation -from cudf._lib.pylibcudf.libcudf.types cimport interpolation, sorted +from pylibcudf.libcudf.types cimport interpolation, sorted + from cudf._lib.utils cimport columns_from_pylibcudf_table -import cudf._lib.pylibcudf as plc +import pylibcudf as plc @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/reduce.pyx b/python/cudf/cudf/_lib/reduce.pyx index 511bba20ef5..944753d28b8 100644 --- a/python/cudf/cudf/_lib/reduce.pyx +++ b/python/cudf/cudf/_lib/reduce.pyx @@ -8,7 +8,8 @@ from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar from cudf._lib.types cimport dtype_to_pylibcudf_type, is_decimal_type_id -from cudf._lib import pylibcudf +import pylibcudf + from cudf._lib.aggregation import make_aggregation diff --git a/python/cudf/cudf/_lib/replace.pyx b/python/cudf/cudf/_lib/replace.pyx index 2b5f32c7675..b50c6dd25e3 100644 --- a/python/cudf/cudf/_lib/replace.pyx +++ b/python/cudf/cudf/_lib/replace.pyx @@ -6,7 +6,8 @@ from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar -from cudf._lib import pylibcudf +import pylibcudf + from cudf._lib.scalar import as_device_scalar diff --git a/python/cudf/cudf/_lib/reshape.pyx b/python/cudf/cudf/_lib/reshape.pyx index 6bba8f0df35..6cebeb2bc16 100644 --- a/python/cudf/cudf/_lib/reshape.pyx +++ b/python/cudf/cudf/_lib/reshape.pyx @@ -2,11 +2,12 @@ from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.types cimport size_type from cudf._lib.utils cimport columns_from_pylibcudf_table -import cudf._lib.pylibcudf as plc +import pylibcudf as plc @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/rolling.pyx b/python/cudf/cudf/_lib/rolling.pyx index 5439e70fdce..687b261c2c7 100644 --- a/python/cudf/cudf/_lib/rolling.pyx +++ b/python/cudf/cudf/_lib/rolling.pyx @@ -4,7 +4,8 @@ from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column -from cudf._lib import pylibcudf +import pylibcudf + from cudf._lib.aggregation import make_aggregation diff --git a/python/cudf/cudf/_lib/round.pyx b/python/cudf/cudf/_lib/round.pyx index f8ad57947c8..f961c09e6f6 100644 --- a/python/cudf/cudf/_lib/round.pyx +++ b/python/cudf/cudf/_lib/round.pyx @@ -4,8 +4,8 @@ from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.round import RoundingMethod +import pylibcudf as plc +from pylibcudf.round import RoundingMethod @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/scalar.pxd b/python/cudf/cudf/_lib/scalar.pxd index b57acbb37f1..27095ca02d4 100644 --- a/python/cudf/cudf/_lib/scalar.pxd +++ b/python/cudf/cudf/_lib/scalar.pxd @@ -3,10 +3,9 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr +from pylibcudf.libcudf.scalar.scalar cimport scalar from rmm._lib.memory_resource cimport DeviceMemoryResource -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar - cdef class DeviceScalar: cdef public object c_value diff --git a/python/cudf/cudf/_lib/scalar.pyx b/python/cudf/cudf/_lib/scalar.pyx index e68398498d1..0dde91316fb 100644 --- a/python/cudf/cudf/_lib/scalar.pyx +++ b/python/cudf/cudf/_lib/scalar.pyx @@ -11,38 +11,40 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move +import pylibcudf + import cudf -from cudf._lib import pylibcudf from cudf._lib.types import LIBCUDF_TO_SUPPORTED_NUMPY_TYPES from cudf.core.dtypes import ListDtype, StructDtype from cudf.core.missing import NA, NaT -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types +cimport pylibcudf.libcudf.types as libcudf_types # We currently need this cimport because some of the implementations here # access the c_obj of the scalar, and because we need to be able to call # pylibcudf.Scalar.from_libcudf. Both of those are temporarily acceptable until # DeviceScalar is phased out entirely from cuDF Cython (at which point # cudf.Scalar will be directly backed by pylibcudf.Scalar). -from cudf._lib.pylibcudf cimport Scalar as plc_Scalar -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport ( +from pylibcudf cimport Scalar as plc_Scalar +from pylibcudf.libcudf.scalar.scalar cimport ( duration_scalar, list_scalar, scalar, struct_scalar, timestamp_scalar, ) -from cudf._lib.pylibcudf.libcudf.wrappers.durations cimport ( +from pylibcudf.libcudf.wrappers.durations cimport ( duration_ms, duration_ns, duration_s, duration_us, ) -from cudf._lib.pylibcudf.libcudf.wrappers.timestamps cimport ( +from pylibcudf.libcudf.wrappers.timestamps cimport ( timestamp_ms, timestamp_ns, timestamp_s, timestamp_us, ) + from cudf._lib.types cimport dtype_from_column_view, underlying_type_t_type_id diff --git a/python/cudf/cudf/_lib/search.pyx b/python/cudf/cudf/_lib/search.pyx index 1ee73949fd3..8108361052b 100644 --- a/python/cudf/cudf/_lib/search.pyx +++ b/python/cudf/cudf/_lib/search.pyx @@ -4,7 +4,7 @@ from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column -from cudf._lib import pylibcudf +import pylibcudf @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/sort.pyx b/python/cudf/cudf/_lib/sort.pyx index ff9565b9a89..185552ede82 100644 --- a/python/cudf/cudf/_lib/sort.pyx +++ b/python/cudf/cudf/_lib/sort.pyx @@ -9,18 +9,19 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector +from pylibcudf.libcudf.aggregation cimport rank_method +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.search cimport lower_bound, upper_bound +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport null_order, order as cpp_order + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.aggregation cimport rank_method -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.search cimport lower_bound, upper_bound -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport null_order, order as cpp_order from cudf._lib.utils cimport ( columns_from_pylibcudf_table, table_view_from_columns, ) -from cudf._lib import pylibcudf +import pylibcudf @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/stream_compaction.pyx b/python/cudf/cudf/_lib/stream_compaction.pyx index 834f91f48d9..1b8831940e3 100644 --- a/python/cudf/cudf/_lib/stream_compaction.pyx +++ b/python/cudf/cudf/_lib/stream_compaction.pyx @@ -7,7 +7,7 @@ from libcpp cimport bool from cudf._lib.column cimport Column from cudf._lib.utils cimport columns_from_pylibcudf_table -from cudf._lib import pylibcudf +import pylibcudf @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/string_casting.pyx b/python/cudf/cudf/_lib/string_casting.pyx index dfad7fd101c..8d463829a19 100644 --- a/python/cudf/cudf/_lib/string_casting.pyx +++ b/python/cudf/cudf/_lib/string_casting.pyx @@ -12,39 +12,40 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_booleans cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.convert.convert_booleans cimport ( from_booleans as cpp_from_booleans, to_booleans as cpp_to_booleans, ) -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_datetime cimport ( +from pylibcudf.libcudf.strings.convert.convert_datetime cimport ( from_timestamps as cpp_from_timestamps, is_timestamp as cpp_is_timestamp, to_timestamps as cpp_to_timestamps, ) -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_durations cimport ( +from pylibcudf.libcudf.strings.convert.convert_durations cimport ( from_durations as cpp_from_durations, to_durations as cpp_to_durations, ) -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_floats cimport ( +from pylibcudf.libcudf.strings.convert.convert_floats cimport ( from_floats as cpp_from_floats, to_floats as cpp_to_floats, ) -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_integers cimport ( +from pylibcudf.libcudf.strings.convert.convert_integers cimport ( from_integers as cpp_from_integers, hex_to_integers as cpp_hex_to_integers, integers_to_hex as cpp_integers_to_hex, is_hex as cpp_is_hex, to_integers as cpp_to_integers, ) -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_ipv4 cimport ( +from pylibcudf.libcudf.strings.convert.convert_ipv4 cimport ( integers_to_ipv4 as cpp_integers_to_ipv4, ipv4_to_integers as cpp_ipv4_to_integers, is_ipv4 as cpp_is_ipv4, ) -from cudf._lib.pylibcudf.libcudf.types cimport data_type, type_id +from pylibcudf.libcudf.types cimport data_type, type_id + from cudf._lib.types cimport underlying_type_t_type_id import cudf diff --git a/python/cudf/cudf/_lib/strings/attributes.pyx b/python/cudf/cudf/_lib/strings/attributes.pyx index 1f3d7c4eb1b..fe8c17c9e31 100644 --- a/python/cudf/cudf/_lib/strings/attributes.pyx +++ b/python/cudf/cudf/_lib/strings/attributes.pyx @@ -5,15 +5,16 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.attributes cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.attributes cimport ( code_points as cpp_code_points, count_bytes as cpp_count_bytes, count_characters as cpp_count_characters, ) +from cudf._lib.column cimport Column + @acquire_spill_lock() def count_characters(Column source_strings): diff --git a/python/cudf/cudf/_lib/strings/capitalize.pyx b/python/cudf/cudf/_lib/strings/capitalize.pyx index b3ca6a5ac8f..42c40e2e753 100644 --- a/python/cudf/cudf/_lib/strings/capitalize.pyx +++ b/python/cudf/cudf/_lib/strings/capitalize.pyx @@ -4,7 +4,7 @@ from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column -import cudf._lib.pylibcudf as plc +import pylibcudf as plc @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/case.pyx b/python/cudf/cudf/_lib/strings/case.pyx index 38f242a67d6..ad4cbb6f088 100644 --- a/python/cudf/cudf/_lib/strings/case.pyx +++ b/python/cudf/cudf/_lib/strings/case.pyx @@ -4,7 +4,7 @@ from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.strings import case +from pylibcudf.strings import case @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/char_types.pyx b/python/cudf/cudf/_lib/strings/char_types.pyx index 5b7b6d19d9e..376a6f8af97 100644 --- a/python/cudf/cudf/_lib/strings/char_types.pyx +++ b/python/cudf/cudf/_lib/strings/char_types.pyx @@ -7,15 +7,16 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.char_types cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.char_types cimport ( all_characters_of_type as cpp_all_characters_of_type, filter_characters_of_type as cpp_filter_characters_of_type, string_character_types, ) + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/strings/combine.pyx b/python/cudf/cudf/_lib/strings/combine.pyx index 288f333d4d8..76cc13db0da 100644 --- a/python/cudf/cudf/_lib/strings/combine.pyx +++ b/python/cudf/cudf/_lib/strings/combine.pyx @@ -5,18 +5,19 @@ from cudf.core.buffer import acquire_spill_lock from libcpp.memory cimport unique_ptr from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.combine cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.combine cimport ( concatenate as cpp_concatenate, join_list_elements as cpp_join_list_elements, join_strings as cpp_join_strings, output_if_empty_list, separator_on_nulls, ) -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.table.table_view cimport table_view + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar from cudf._lib.utils cimport table_view_from_columns diff --git a/python/cudf/cudf/_lib/strings/contains.pyx b/python/cudf/cudf/_lib/strings/contains.pyx index 502a1d14696..82f5e06c547 100644 --- a/python/cudf/cudf/_lib/strings/contains.pyx +++ b/python/cudf/cudf/_lib/strings/contains.pyx @@ -9,21 +9,22 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.contains cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.contains cimport ( count_re as cpp_count_re, like as cpp_like, matches_re as cpp_matches_re, ) -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags +from pylibcudf.libcudf.strings.regex_program cimport regex_program + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar -from cudf._lib.pylibcudf.strings import contains -from cudf._lib.pylibcudf.strings.regex_program import RegexProgram +from pylibcudf.strings import contains +from pylibcudf.strings.regex_program import RegexProgram @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/convert/convert_fixed_point.pyx b/python/cudf/cudf/_lib/strings/convert/convert_fixed_point.pyx index 6faff606226..a8df8c9a92c 100644 --- a/python/cudf/cudf/_lib/strings/convert/convert_fixed_point.pyx +++ b/python/cudf/cudf/_lib/strings/convert/convert_fixed_point.pyx @@ -7,15 +7,16 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_fixed_point cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.convert.convert_fixed_point cimport ( from_fixed_point as cpp_from_fixed_point, is_fixed_point as cpp_is_fixed_point, to_fixed_point as cpp_to_fixed_point, ) -from cudf._lib.pylibcudf.libcudf.types cimport data_type, type_id +from pylibcudf.libcudf.types cimport data_type, type_id + +from cudf._lib.column cimport Column @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/convert/convert_floats.pyx b/python/cudf/cudf/_lib/strings/convert/convert_floats.pyx index 341cbc99dab..7965b588703 100644 --- a/python/cudf/cudf/_lib/strings/convert/convert_floats.pyx +++ b/python/cudf/cudf/_lib/strings/convert/convert_floats.pyx @@ -5,13 +5,14 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_floats cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.convert.convert_floats cimport ( is_float as cpp_is_float, ) +from cudf._lib.column cimport Column + @acquire_spill_lock() def is_float(Column source_strings): diff --git a/python/cudf/cudf/_lib/strings/convert/convert_integers.pyx b/python/cudf/cudf/_lib/strings/convert/convert_integers.pyx index 081b03cdc0d..8b6da2bfa1c 100644 --- a/python/cudf/cudf/_lib/strings/convert/convert_integers.pyx +++ b/python/cudf/cudf/_lib/strings/convert/convert_integers.pyx @@ -5,13 +5,14 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_integers cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.convert.convert_integers cimport ( is_integer as cpp_is_integer, ) +from cudf._lib.column cimport Column + @acquire_spill_lock() def is_integer(Column source_strings): diff --git a/python/cudf/cudf/_lib/strings/convert/convert_lists.pyx b/python/cudf/cudf/_lib/strings/convert/convert_lists.pyx index 4418bf2a72d..73aebf8ab35 100644 --- a/python/cudf/cudf/_lib/strings/convert/convert_lists.pyx +++ b/python/cudf/cudf/_lib/strings/convert/convert_lists.pyx @@ -5,14 +5,15 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_lists cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.convert.convert_lists cimport ( format_list_column as cpp_format_list_column, ) +from cudf._lib.column cimport Column + from cudf._lib.scalar import as_device_scalar from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/strings/convert/convert_urls.pyx b/python/cudf/cudf/_lib/strings/convert/convert_urls.pyx index 5f62efe5c00..e52116d6247 100644 --- a/python/cudf/cudf/_lib/strings/convert/convert_urls.pyx +++ b/python/cudf/cudf/_lib/strings/convert/convert_urls.pyx @@ -5,14 +5,15 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.convert.convert_urls cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.convert.convert_urls cimport ( url_decode as cpp_url_decode, url_encode as cpp_url_encode, ) +from cudf._lib.column cimport Column + @acquire_spill_lock() def url_decode(Column source_strings): diff --git a/python/cudf/cudf/_lib/strings/extract.pyx b/python/cudf/cudf/_lib/strings/extract.pyx index 3b80c4f6368..63f4d57e562 100644 --- a/python/cudf/cudf/_lib/strings/extract.pyx +++ b/python/cudf/cudf/_lib/strings/extract.pyx @@ -8,12 +8,13 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.extract cimport extract as cpp_extract +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags +from pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.table.table cimport table + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.extract cimport extract as cpp_extract -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program -from cudf._lib.pylibcudf.libcudf.table.table cimport table from cudf._lib.utils cimport data_from_unique_ptr diff --git a/python/cudf/cudf/_lib/strings/find.pyx b/python/cudf/cudf/_lib/strings/find.pyx index 3c0009ee569..2d284d1aa9d 100644 --- a/python/cudf/cudf/_lib/strings/find.pyx +++ b/python/cudf/cudf/_lib/strings/find.pyx @@ -1,10 +1,12 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. -import cudf._lib.pylibcudf as plc +import pylibcudf as plc + from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.types cimport size_type @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/find_multiple.pyx b/python/cudf/cudf/_lib/strings/find_multiple.pyx index c75f28db21b..1358f8e3c2c 100644 --- a/python/cudf/cudf/_lib/strings/find_multiple.pyx +++ b/python/cudf/cudf/_lib/strings/find_multiple.pyx @@ -5,13 +5,14 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.find_multiple cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.find_multiple cimport ( find_multiple as cpp_find_multiple, ) +from cudf._lib.column cimport Column + @acquire_spill_lock() def find_multiple(Column source_strings, Column target_strings): diff --git a/python/cudf/cudf/_lib/strings/findall.pyx b/python/cudf/cudf/_lib/strings/findall.pyx index 0d409889bc8..3cf2084e30a 100644 --- a/python/cudf/cudf/_lib/strings/findall.pyx +++ b/python/cudf/cudf/_lib/strings/findall.pyx @@ -8,12 +8,13 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.findall cimport findall as cpp_findall +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags +from pylibcudf.libcudf.strings.regex_program cimport regex_program + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.findall cimport findall as cpp_findall -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/json.pyx b/python/cudf/cudf/_lib/strings/json.pyx index 560f284b56c..c9b0bba088d 100644 --- a/python/cudf/cudf/_lib/strings/json.pyx +++ b/python/cudf/cudf/_lib/strings/json.pyx @@ -5,14 +5,15 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.json cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.json cimport ( get_json_object as cpp_get_json_object, get_json_object_options, ) + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/strings/padding.pyx b/python/cudf/cudf/_lib/strings/padding.pyx index 9226810951f..d0239e91ec3 100644 --- a/python/cudf/cudf/_lib/strings/padding.pyx +++ b/python/cudf/cudf/_lib/strings/padding.pyx @@ -6,18 +6,19 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type from enum import IntEnum -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.strings.padding cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.strings.padding cimport ( pad as cpp_pad, zfill as cpp_zfill, ) -from cudf._lib.pylibcudf.libcudf.strings.side_type cimport ( +from pylibcudf.libcudf.strings.side_type cimport ( side_type, underlying_type_t_side_type, ) diff --git a/python/cudf/cudf/_lib/strings/repeat.pyx b/python/cudf/cudf/_lib/strings/repeat.pyx index 2b8116848cf..42fcfa5d94e 100644 --- a/python/cudf/cudf/_lib/strings/repeat.pyx +++ b/python/cudf/cudf/_lib/strings/repeat.pyx @@ -5,11 +5,12 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings cimport repeat as cpp_repeat +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings cimport repeat as cpp_repeat -from cudf._lib.pylibcudf.libcudf.types cimport size_type @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/replace.pyx b/python/cudf/cudf/_lib/strings/replace.pyx index 374831f1833..a260c4e4f45 100644 --- a/python/cudf/cudf/_lib/strings/replace.pyx +++ b/python/cudf/cudf/_lib/strings/replace.pyx @@ -4,11 +4,12 @@ from libc.stdint cimport int32_t from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.types cimport size_type from cudf._lib.scalar cimport DeviceScalar -import cudf._lib.pylibcudf as plc +import pylibcudf as plc @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/replace_re.pyx b/python/cudf/cudf/_lib/strings/replace_re.pyx index e13880a6186..fffc8b7c3f6 100644 --- a/python/cudf/cudf/_lib/strings/replace_re.pyx +++ b/python/cudf/cudf/_lib/strings/replace_re.pyx @@ -8,17 +8,18 @@ from libcpp.vector cimport vector from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program -from cudf._lib.pylibcudf.libcudf.strings.replace_re cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags +from pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.strings.replace_re cimport ( replace_re as cpp_replace_re, replace_with_backrefs as cpp_replace_with_backrefs, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/strings/split/partition.pyx b/python/cudf/cudf/_lib/strings/split/partition.pyx index be377c0f86b..a81fb18e752 100644 --- a/python/cudf/cudf/_lib/strings/split/partition.pyx +++ b/python/cudf/cudf/_lib/strings/split/partition.pyx @@ -5,14 +5,15 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.split.partition cimport ( +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.split.partition cimport ( partition as cpp_partition, rpartition as cpp_rpartition, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table cimport table + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar from cudf._lib.utils cimport data_from_unique_ptr diff --git a/python/cudf/cudf/_lib/strings/split/split.pyx b/python/cudf/cudf/_lib/strings/split/split.pyx index 942235686d7..f481fea4c51 100644 --- a/python/cudf/cudf/_lib/strings/split/split.pyx +++ b/python/cudf/cudf/_lib/strings/split/split.pyx @@ -7,13 +7,12 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program -from cudf._lib.pylibcudf.libcudf.strings.split.split cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags +from pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.strings.split.split cimport ( rsplit as cpp_rsplit, rsplit_re as cpp_rsplit_re, rsplit_record as cpp_rsplit_record, @@ -23,8 +22,10 @@ from cudf._lib.pylibcudf.libcudf.strings.split.split cimport ( split_record as cpp_split_record, split_record_re as cpp_split_record_re, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport size_type + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar from cudf._lib.utils cimport data_from_unique_ptr diff --git a/python/cudf/cudf/_lib/strings/strip.pyx b/python/cudf/cudf/_lib/strings/strip.pyx index 199fa5fc3b6..acf52cb7b9f 100644 --- a/python/cudf/cudf/_lib/strings/strip.pyx +++ b/python/cudf/cudf/_lib/strings/strip.pyx @@ -5,12 +5,13 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.side_type cimport side_type +from pylibcudf.libcudf.strings.strip cimport strip as cpp_strip + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.side_type cimport side_type -from cudf._lib.pylibcudf.libcudf.strings.strip cimport strip as cpp_strip from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/strings/substring.pyx b/python/cudf/cudf/_lib/strings/substring.pyx index 706c21c0634..db96d99c7b6 100644 --- a/python/cudf/cudf/_lib/strings/substring.pyx +++ b/python/cudf/cudf/_lib/strings/substring.pyx @@ -10,7 +10,7 @@ from cudf._lib.scalar import as_device_scalar from cudf._lib.scalar cimport DeviceScalar -import cudf._lib.pylibcudf as plc +import pylibcudf as plc @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings/translate.pyx b/python/cudf/cudf/_lib/strings/translate.pyx index 8846e2e280d..3fad91bbfc0 100644 --- a/python/cudf/cudf/_lib/strings/translate.pyx +++ b/python/cudf/cudf/_lib/strings/translate.pyx @@ -8,16 +8,17 @@ from libcpp.vector cimport vector from cudf.core.buffer import acquire_spill_lock -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.translate cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.translate cimport ( filter_characters as cpp_filter_characters, filter_type, translate as cpp_translate, ) -from cudf._lib.pylibcudf.libcudf.types cimport char_utf8 +from pylibcudf.libcudf.types cimport char_utf8 + +from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar diff --git a/python/cudf/cudf/_lib/strings/wrap.pyx b/python/cudf/cudf/_lib/strings/wrap.pyx index 92750f21e4d..eed5cf33b10 100644 --- a/python/cudf/cudf/_lib/strings/wrap.pyx +++ b/python/cudf/cudf/_lib/strings/wrap.pyx @@ -5,11 +5,12 @@ from libcpp.utility cimport move from cudf.core.buffer import acquire_spill_lock +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.wrap cimport wrap as cpp_wrap +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.wrap cimport wrap as cpp_wrap -from cudf._lib.pylibcudf.libcudf.types cimport size_type @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/strings_udf.pyx b/python/cudf/cudf/_lib/strings_udf.pyx index 7610cad0b40..78fc9f08bd8 100644 --- a/python/cudf/cudf/_lib/strings_udf.pyx +++ b/python/cudf/cudf/_lib/strings_udf.pyx @@ -2,7 +2,7 @@ from libc.stdint cimport uint8_t, uint16_t, uintptr_t -from cudf._lib.pylibcudf.libcudf.strings_udf cimport ( +from pylibcudf.libcudf.strings_udf cimport ( get_character_cases_table as cpp_get_character_cases_table, get_character_flags_table as cpp_get_character_flags_table, get_special_case_mapping_table as cpp_get_special_case_mapping_table, @@ -15,17 +15,17 @@ from libcpp.utility cimport move from cudf.core.buffer import as_buffer -from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer - -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column, column_view -from cudf._lib.pylibcudf.libcudf.strings_udf cimport ( +from pylibcudf.libcudf.column.column cimport column, column_view +from pylibcudf.libcudf.strings_udf cimport ( column_from_udf_string_array as cpp_column_from_udf_string_array, free_udf_string_array as cpp_free_udf_string_array, get_cuda_build_version as cpp_get_cuda_build_version, to_string_view_array as cpp_to_string_view_array, udf_string, ) +from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer + +from cudf._lib.column cimport Column def get_cuda_build_version(): diff --git a/python/cudf/cudf/_lib/text.pyx b/python/cudf/cudf/_lib/text.pyx index 6e63b8758b8..ece69b424bb 100644 --- a/python/cudf/cudf/_lib/text.pyx +++ b/python/cudf/cudf/_lib/text.pyx @@ -8,9 +8,8 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.io.text cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.io.text cimport ( byte_range_info, data_chunk_source, make_source, @@ -20,6 +19,8 @@ from cudf._lib.pylibcudf.libcudf.io.text cimport ( parse_options, ) +from cudf._lib.column cimport Column + def read_text(object filepaths_or_buffers, object delimiter=None, diff --git a/python/cudf/cudf/_lib/timezone.pyx b/python/cudf/cudf/_lib/timezone.pyx index 53977e984c2..bff3b2c4ce4 100644 --- a/python/cudf/cudf/_lib/timezone.pyx +++ b/python/cudf/cudf/_lib/timezone.pyx @@ -5,10 +5,11 @@ from libcpp.optional cimport make_optional from libcpp.string cimport string from libcpp.utility cimport move -from cudf._lib.pylibcudf.libcudf.io.timezone cimport ( +from pylibcudf.libcudf.io.timezone cimport ( make_timezone_transition_table as cpp_make_timezone_transition_table, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table cimport table + from cudf._lib.utils cimport columns_from_unique_ptr diff --git a/python/cudf/cudf/_lib/transform.pyx b/python/cudf/cudf/_lib/transform.pyx index 622725e06a3..baa08a545ec 100644 --- a/python/cudf/cudf/_lib/transform.pyx +++ b/python/cudf/cudf/_lib/transform.pyx @@ -15,23 +15,23 @@ from libcpp.pair cimport pair from libcpp.string cimport string from libcpp.utility cimport move -from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer - -cimport cudf._lib.pylibcudf.libcudf.transform as libcudf_transform -from cudf._lib.column cimport Column -from cudf._lib.pylibcudf cimport transform as plc_transform -from cudf._lib.pylibcudf.expressions cimport Expression -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.expressions cimport expression -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport ( +cimport pylibcudf.libcudf.transform as libcudf_transform +from pylibcudf cimport transform as plc_transform +from pylibcudf.expressions cimport Expression +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.expressions cimport expression +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport ( bitmask_type, data_type, size_type, type_id, ) +from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer + +from cudf._lib.column cimport Column from cudf._lib.types cimport underlying_type_t_type_id from cudf._lib.utils cimport ( columns_from_unique_ptr, diff --git a/python/cudf/cudf/_lib/transpose.pyx b/python/cudf/cudf/_lib/transpose.pyx index 82b23439e6a..f78fbd4c844 100644 --- a/python/cudf/cudf/_lib/transpose.pyx +++ b/python/cudf/cudf/_lib/transpose.pyx @@ -4,10 +4,11 @@ from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair from libcpp.utility cimport move +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.transpose cimport transpose as cpp_transpose + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.transpose cimport transpose as cpp_transpose from cudf._lib.utils cimport columns_from_table_view, table_view_from_columns diff --git a/python/cudf/cudf/_lib/types.pxd b/python/cudf/cudf/_lib/types.pxd index 519d5ff8554..4fd3d31841e 100644 --- a/python/cudf/cudf/_lib/types.pxd +++ b/python/cudf/cudf/_lib/types.pxd @@ -3,11 +3,9 @@ from libc.stdint cimport int32_t from libcpp cimport bool -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) +cimport pylibcudf.libcudf.types as libcudf_types +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view ctypedef bool underlying_type_t_order ctypedef bool underlying_type_t_null_order diff --git a/python/cudf/cudf/_lib/types.pyx b/python/cudf/cudf/_lib/types.pyx index 253fdf7b0d9..861bb063707 100644 --- a/python/cudf/cudf/_lib/types.pyx +++ b/python/cudf/cudf/_lib/types.pyx @@ -7,19 +7,19 @@ import pandas as pd from libcpp.memory cimport make_shared, shared_ptr -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) +cimport pylibcudf.libcudf.types as libcudf_types +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view + from cudf._lib.types cimport ( underlying_type_t_interpolation, underlying_type_t_order, underlying_type_t_sorted, ) +import pylibcudf + import cudf -from cudf._lib import pylibcudf class TypeId(IntEnum): diff --git a/python/cudf/cudf/_lib/unary.pyx b/python/cudf/cudf/_lib/unary.pyx index 2f58c4512d6..d5602fd5a1c 100644 --- a/python/cudf/cudf/_lib/unary.pyx +++ b/python/cudf/cudf/_lib/unary.pyx @@ -5,7 +5,8 @@ from cudf._lib.types cimport dtype_to_pylibcudf_type import numpy as np -from cudf._lib import pylibcudf +import pylibcudf + from cudf.api.types import is_decimal_dtype from cudf.core.buffer import acquire_spill_lock diff --git a/python/cudf/cudf/_lib/utils.pxd b/python/cudf/cudf/_lib/utils.pxd index 1d55f7218dc..ff97fe80310 100644 --- a/python/cudf/cudf/_lib/utils.pxd +++ b/python/cudf/cudf/_lib/utils.pxd @@ -4,8 +4,8 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector -from cudf._lib.pylibcudf.libcudf.column.column cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table, table_view +from pylibcudf.libcudf.column.column cimport column_view +from pylibcudf.libcudf.table.table cimport table, table_view cdef data_from_unique_ptr( diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index 267432a0182..cae28d02ef4 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -10,11 +10,12 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector +from pylibcudf.libcudf.column.column cimport column, column_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type + from cudf._lib.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column, column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type try: import ujson as json diff --git a/python/cudf/cudf/core/_internals/expressions.py b/python/cudf/cudf/core/_internals/expressions.py index 63714a78572..67bde5a72b2 100644 --- a/python/cudf/cudf/core/_internals/expressions.py +++ b/python/cudf/cudf/core/_internals/expressions.py @@ -6,8 +6,8 @@ import pyarrow as pa -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.expressions import ( +import pylibcudf as plc +from pylibcudf.expressions import ( ASTOperator, ColumnReference, Expression, diff --git a/python/cudf/cudf/core/buffer/buffer.py b/python/cudf/cudf/core/buffer/buffer.py index 80dbbe4c048..32ae8c5ee53 100644 --- a/python/cudf/cudf/core/buffer/buffer.py +++ b/python/cudf/cudf/core/buffer/buffer.py @@ -11,6 +11,7 @@ import numpy from typing_extensions import Self +import pylibcudf import rmm import cudf @@ -501,7 +502,7 @@ def get_ptr_and_size(array_interface: Mapping) -> tuple[int, int]: shape = array_interface["shape"] or (1,) strides = array_interface["strides"] itemsize = cudf.dtype(array_interface["typestr"]).itemsize - if strides is None or cudf._lib.pylibcudf.column.is_c_contiguous( + if strides is None or pylibcudf.column.is_c_contiguous( shape, strides, itemsize ): nelem = math.prod(shape) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index ac36813202a..a37355dfcda 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -9,9 +9,10 @@ import pandas as pd from typing_extensions import Self +import pylibcudf + import cudf from cudf import _lib as libcudf -from cudf._lib import pylibcudf from cudf.api.types import is_integer, is_scalar from cudf.core.column import ColumnBase, as_column, column, string from cudf.core.dtypes import CategoricalDtype diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 8eb6de79bce..2263dfd5c98 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -24,6 +24,8 @@ import pandas as pd from typing_extensions import Self +import pylibcudf + import cudf import cudf._lib as libcudf import cudf.core @@ -6311,7 +6313,7 @@ def rank( if method not in {"average", "min", "max", "first", "dense"}: raise KeyError(method) - method_enum = libcudf.pylibcudf.aggregation.RankMethod[method.upper()] + method_enum = pylibcudf.aggregation.RankMethod[method.upper()] if na_option not in {"keep", "top", "bottom"}: raise ValueError( "na_option must be one of 'keep', 'top', or 'bottom'" diff --git a/python/cudf/cudf/pandas/__init__.py b/python/cudf/cudf/pandas/__init__.py index e88e795671e..bacf1f7e77b 100644 --- a/python/cudf/cudf/pandas/__init__.py +++ b/python/cudf/cudf/pandas/__init__.py @@ -5,10 +5,9 @@ import os import warnings +import pylibcudf import rmm.mr -from cudf._lib import pylibcudf - from .fast_slow_proxy import is_proxy_object from .magics import load_ipython_extension from .profiler import Profiler diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 60ac171f3d7..9db52164eca 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pandas>=2.0,<2.2.3dev0", "ptxcompiler", "pyarrow>=16.1.0,<16.2.0a0", + "pylibcudf==24.10.*,>=0.0.0a0", "rich", "rmm==24.10.*,>=0.0.0a0", "typing_extensions>=4.0.0", @@ -88,6 +89,7 @@ known_dask = [ ] known_rapids = [ "rmm", + "pylibcudf" ] known_first_party = [ "cudf", @@ -127,6 +129,7 @@ requires = [ "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", + "pylibcudf==24.10.*,>=0.0.0a0", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt b/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt index 4f3b9220a4f..1b205537d73 100644 --- a/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt +++ b/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ) -include(../../../cudf/cmake/Modules/LinkPyarrowHeaders.cmake) +include(../../../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) link_to_pyarrow_headers("${RAPIDS_CYTHON_CREATED_TARGETS}") diff --git a/python/cudf_kafka/cudf_kafka/_lib/kafka.pxd b/python/cudf_kafka/cudf_kafka/_lib/kafka.pxd index 2de0bf39785..e65b0d233b9 100644 --- a/python/cudf_kafka/cudf_kafka/_lib/kafka.pxd +++ b/python/cudf_kafka/cudf_kafka/_lib/kafka.pxd @@ -6,9 +6,8 @@ from libcpp.map cimport map from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.io.datasource cimport Datasource -from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource +from pylibcudf.io.datasource cimport Datasource +from pylibcudf.libcudf.io.datasource cimport datasource cdef extern from "cudf_kafka/kafka_callback.hpp" \ diff --git a/python/cudf_kafka/cudf_kafka/_lib/kafka.pyx b/python/cudf_kafka/cudf_kafka/_lib/kafka.pyx index 2927dc0aa9a..20aa43b0134 100644 --- a/python/cudf_kafka/cudf_kafka/_lib/kafka.pyx +++ b/python/cudf_kafka/cudf_kafka/_lib/kafka.pyx @@ -6,8 +6,7 @@ from libcpp.map cimport map from libcpp.memory cimport make_unique, unique_ptr from libcpp.string cimport string from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource +from pylibcudf.libcudf.io.datasource cimport datasource from cudf_kafka._lib.kafka cimport kafka_consumer diff --git a/python/cudf_polars/cudf_polars/containers/column.py b/python/cudf_polars/cudf_polars/containers/column.py index 02018548b2c..dd3b771e305 100644 --- a/python/cudf_polars/cudf_polars/containers/column.py +++ b/python/cudf_polars/cudf_polars/containers/column.py @@ -8,7 +8,7 @@ import functools from typing import TYPE_CHECKING -import cudf._lib.pylibcudf as plc +import pylibcudf as plc if TYPE_CHECKING: from typing_extensions import Self diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index dba76855329..7c28e7b9a6c 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -10,11 +10,10 @@ from typing import TYPE_CHECKING, cast import pyarrow as pa +import pylibcudf as plc import polars as pl -import cudf._lib.pylibcudf as plc - from cudf_polars.containers.column import NamedColumn from cudf_polars.utils import dtypes diff --git a/python/cudf_polars/cudf_polars/dsl/expr.py b/python/cudf_polars/cudf_polars/dsl/expr.py index 9e0fca3f52f..e1b4d30b76b 100644 --- a/python/cudf_polars/cudf_polars/dsl/expr.py +++ b/python/cudf_polars/cudf_polars/dsl/expr.py @@ -21,11 +21,10 @@ from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple import pyarrow as pa +import pylibcudf as plc from polars.polars import _expr_nodes as pl_expr -import cudf._lib.pylibcudf as plc - from cudf_polars.containers import Column, NamedColumn from cudf_polars.utils import dtypes, sorting diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 3754addeb11..019f00f4fca 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -21,12 +21,11 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar import pyarrow as pa +import pylibcudf as plc from typing_extensions import assert_never import polars as pl -import cudf._lib.pylibcudf as plc - import cudf_polars.dsl.expr as expr from cudf_polars.containers import DataFrame, NamedColumn from cudf_polars.utils import sorting diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index dec45679c75..6dc97c7cb51 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -11,14 +11,13 @@ from typing import Any import pyarrow as pa +import pylibcudf as plc from typing_extensions import assert_never import polars as pl import polars.polars as plrs from polars.polars import _expr_nodes as pl_expr, _ir_nodes as pl_ir -import cudf._lib.pylibcudf as plc - from cudf_polars.dsl import expr, ir from cudf_polars.typing import NodeTraverser from cudf_polars.utils import dtypes diff --git a/python/cudf_polars/cudf_polars/typing/__init__.py b/python/cudf_polars/cudf_polars/typing/__init__.py index c04eac41bb7..02440e67fde 100644 --- a/python/cudf_polars/cudf_polars/typing/__init__.py +++ b/python/cudf_polars/cudf_polars/typing/__init__.py @@ -8,9 +8,9 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Literal, Protocol, Union -from polars.polars import _expr_nodes as pl_expr, _ir_nodes as pl_ir +import pylibcudf as plc -import cudf._lib.pylibcudf as plc +from polars.polars import _expr_nodes as pl_expr, _ir_nodes as pl_ir if TYPE_CHECKING: from typing import Callable diff --git a/python/cudf_polars/cudf_polars/utils/dtypes.py b/python/cudf_polars/cudf_polars/utils/dtypes.py index cd68d021286..7f6ea1edfd9 100644 --- a/python/cudf_polars/cudf_polars/utils/dtypes.py +++ b/python/cudf_polars/cudf_polars/utils/dtypes.py @@ -8,12 +8,11 @@ from functools import cache import pyarrow as pa +import pylibcudf as plc from typing_extensions import assert_never import polars as pl -import cudf._lib.pylibcudf as plc - __all__ = ["from_polars", "downcast_arrow_lists"] diff --git a/python/cudf_polars/cudf_polars/utils/sorting.py b/python/cudf_polars/cudf_polars/utils/sorting.py index 57f94c4ec4c..17ea44e5b1b 100644 --- a/python/cudf_polars/cudf_polars/utils/sorting.py +++ b/python/cudf_polars/cudf_polars/utils/sorting.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING -import cudf._lib.pylibcudf as plc +import pylibcudf as plc if TYPE_CHECKING: from collections.abc import Sequence diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 424c83a5199..c380853035d 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -19,8 +19,8 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cudf==24.10.*,>=0.0.0a0", "polars>=1.0,<1.3", + "pylibcudf==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/cudf_polars/tests/containers/test_column.py b/python/cudf_polars/tests/containers/test_column.py index 4f3c0de5975..19919877f84 100644 --- a/python/cudf_polars/tests/containers/test_column.py +++ b/python/cudf_polars/tests/containers/test_column.py @@ -6,10 +6,9 @@ from functools import partial import pyarrow +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - from cudf_polars.containers import Column, NamedColumn diff --git a/python/cudf_polars/tests/containers/test_dataframe.py b/python/cudf_polars/tests/containers/test_dataframe.py index 87508e17407..6b470268084 100644 --- a/python/cudf_polars/tests/containers/test_dataframe.py +++ b/python/cudf_polars/tests/containers/test_dataframe.py @@ -3,12 +3,11 @@ from __future__ import annotations +import pylibcudf as plc import pytest import polars as pl -import cudf._lib.pylibcudf as plc - from cudf_polars.containers import DataFrame, NamedColumn diff --git a/python/cudf_polars/tests/dsl/test_expr.py b/python/cudf_polars/tests/dsl/test_expr.py index ddc3ca66d86..b7d4672daca 100644 --- a/python/cudf_polars/tests/dsl/test_expr.py +++ b/python/cudf_polars/tests/dsl/test_expr.py @@ -3,10 +3,9 @@ from __future__ import annotations +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - from cudf_polars.dsl import expr diff --git a/python/cudf_polars/tests/expressions/test_literal.py b/python/cudf_polars/tests/expressions/test_literal.py index 5bd3131d1d7..ced49bdc254 100644 --- a/python/cudf_polars/tests/expressions/test_literal.py +++ b/python/cudf_polars/tests/expressions/test_literal.py @@ -2,12 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import pylibcudf as plc import pytest import polars as pl -import cudf._lib.pylibcudf as plc - from cudf_polars.testing.asserts import ( assert_gpu_result_equal, assert_ir_translation_raises, diff --git a/python/cudf_polars/tests/expressions/test_sort.py b/python/cudf_polars/tests/expressions/test_sort.py index d46df92db94..76c7648813a 100644 --- a/python/cudf_polars/tests/expressions/test_sort.py +++ b/python/cudf_polars/tests/expressions/test_sort.py @@ -4,12 +4,11 @@ import itertools +import pylibcudf as plc import pytest import polars as pl -import cudf._lib.pylibcudf as plc - from cudf_polars import translate_ir from cudf_polars.testing.asserts import assert_gpu_result_equal diff --git a/python/cudf_polars/tests/utils/test_broadcast.py b/python/cudf_polars/tests/utils/test_broadcast.py index 69ad1e519e2..35aaef44e1f 100644 --- a/python/cudf_polars/tests/utils/test_broadcast.py +++ b/python/cudf_polars/tests/utils/test_broadcast.py @@ -3,10 +3,9 @@ from __future__ import annotations +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - from cudf_polars.containers import NamedColumn from cudf_polars.dsl.ir import broadcast diff --git a/python/pylibcudf/CMakeLists.txt b/python/pylibcudf/CMakeLists.txt new file mode 100644 index 00000000000..424d8372280 --- /dev/null +++ b/python/pylibcudf/CMakeLists.txt @@ -0,0 +1,100 @@ +# ============================================================================= +# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) +include(rapids-cuda) +rapids_cuda_init_architectures(pylibcudf) + +project( + pylibcudf + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files" + OFF +) +option(USE_LIBARROW_FROM_PYARROW "Only use the libarrow contained in pyarrow" OFF) +mark_as_advanced(USE_LIBARROW_FROM_PYARROW) + +# Find Python early so that later commands can use it +find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) + +# If the user requested it we attempt to find CUDF. +if(FIND_CUDF_CPP) + include(rapids-cpm) + include(rapids-export) + include(rapids-find) + rapids_cpm_init() + + if(USE_LIBARROW_FROM_PYARROW) + # We need to find arrow before libcudf since libcudf requires it but doesn't bundle arrow + # libraries. These variables have no effect because we are always searching for arrow via + # pyarrow, but they must be set as they are required arguments to the function in + # get_arrow.cmake. + set(CUDF_USE_ARROW_STATIC OFF) + set(CUDF_ENABLE_ARROW_S3 OFF) + set(CUDF_ENABLE_ARROW_ORC OFF) + set(CUDF_ENABLE_ARROW_PYTHON OFF) + set(CUDF_ENABLE_ARROW_PARQUET OFF) + include(../../cpp/cmake/thirdparty/get_arrow.cmake) + endif() + + find_package(cudf "${RAPIDS_VERSION}" REQUIRED) + + # an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack + # for the interop.pyx + include(../../cpp/cmake/thirdparty/get_dlpack.cmake) +else() + set(cudf_FOUND OFF) +endif() + +include(rapids-cython-core) + +if(NOT cudf_FOUND) + set(BUILD_TESTS OFF) + set(BUILD_BENCHMARKS OFF) + set(CUDF_BUILD_TESTUTIL OFF) + set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) + set(CUDA_STATIC_RUNTIME ON) + + add_subdirectory(../../cpp cudf-cpp EXCLUDE_FROM_ALL) + + # libcudf targets are excluded by default above via EXCLUDE_FROM_ALL to remove extraneous + # components like headers from libcudacxx, but we do need the libraries. However, we want to + # control where they are installed to. Since there are multiple subpackages of pylibcudf that + # require access to libcudf, we place the library and all its dependent artifacts in the cudf + # directory as a single source of truth and modify the other rpaths appropriately. + set(cython_lib_dir pylibcudf) + include(cmake/Modules/WheelHelpers.cmake) + # TODO: This install is currently overzealous. We should only install the libraries that are + # downloaded by CPM during the build, not libraries that were found on the system. However, in + # practice right this would only be a problem is if libcudf was not found but some of the + # dependencies were, and we have no real use cases where that happens. + install_aliased_imported_targets( + TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp + DESTINATION ${cython_lib_dir} + ) +endif() + +rapids_cython_init() + +include(cmake/Modules/LinkPyarrowHeaders.cmake) +add_subdirectory(pylibcudf) + +if(DEFINED cython_lib_dir) + rapids_cython_add_rpath_entries(TARGET cudf PATHS "${cython_lib_dir}") +endif() diff --git a/python/pylibcudf/README.md b/python/pylibcudf/README.md new file mode 120000 index 00000000000..fe840054137 --- /dev/null +++ b/python/pylibcudf/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake b/python/pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake similarity index 100% rename from python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake rename to python/pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake diff --git a/python/cudf/cmake/Modules/WheelHelpers.cmake b/python/pylibcudf/cmake/Modules/WheelHelpers.cmake similarity index 100% rename from python/cudf/cmake/Modules/WheelHelpers.cmake rename to python/pylibcudf/cmake/Modules/WheelHelpers.cmake diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/pylibcudf/pylibcudf/CMakeLists.txt similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt rename to python/pylibcudf/pylibcudf/CMakeLists.txt index da32d530928..ab21bfe97ab 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/CMakeLists.txt @@ -54,7 +54,7 @@ rapids_cython_create_modules( ) include(${rapids-cmake-dir}/export/find_package_root.cmake) -include(../../../../../cpp/cmake/thirdparty/get_nanoarrow.cmake) +include(../../../cpp/cmake/thirdparty/get_nanoarrow.cmake) target_link_libraries(pylibcudf_interop PUBLIC nanoarrow) add_subdirectory(libcudf) diff --git a/python/pylibcudf/pylibcudf/VERSION b/python/pylibcudf/pylibcudf/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/pylibcudf/pylibcudf/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/pylibcudf/pylibcudf/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/__init__.pxd rename to python/pylibcudf/pylibcudf/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/pylibcudf/pylibcudf/__init__.py similarity index 99% rename from python/cudf/cudf/_lib/pylibcudf/__init__.py rename to python/pylibcudf/pylibcudf/__init__.py index 9705eba84b1..677fdaf80d0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/pylibcudf/pylibcudf/__init__.py @@ -12,6 +12,7 @@ filling, groupby, interop, + io, join, lists, merge, diff --git a/python/pylibcudf/pylibcudf/_version.py b/python/pylibcudf/pylibcudf/_version.py new file mode 100644 index 00000000000..d2765e5d53c --- /dev/null +++ b/python/pylibcudf/pylibcudf/_version.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd b/python/pylibcudf/pylibcudf/aggregation.pxd similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/aggregation.pxd rename to python/pylibcudf/pylibcudf/aggregation.pxd index 0981d0e855a..c9ab1eab21c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd +++ b/python/pylibcudf/pylibcudf/aggregation.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.aggregation cimport ( +from pylibcudf.libcudf.aggregation cimport ( Kind as kind_t, aggregation, correlation_type, @@ -15,7 +14,7 @@ from cudf._lib.pylibcudf.libcudf.aggregation cimport ( rolling_aggregation, scan_aggregation, ) -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.types cimport ( interpolation, nan_equality, null_equality, diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx b/python/pylibcudf/pylibcudf/aggregation.pyx similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/aggregation.pyx rename to python/pylibcudf/pylibcudf/aggregation.pyx index eed2f6de585..e510b738f70 100644 --- a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx +++ b/python/pylibcudf/pylibcudf/aggregation.pyx @@ -4,8 +4,7 @@ from cython.operator cimport dereference from libcpp.cast cimport dynamic_cast from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.aggregation cimport ( +from pylibcudf.libcudf.aggregation cimport ( aggregation, correlation_type, ewm_history, @@ -41,7 +40,7 @@ from cudf._lib.pylibcudf.libcudf.aggregation cimport ( rolling_aggregation, scan_aggregation, ) -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.types cimport ( interpolation, nan_equality, null_equality, @@ -51,18 +50,16 @@ from cudf._lib.pylibcudf.libcudf.types cimport ( size_type, ) -from cudf._lib.pylibcudf.libcudf.aggregation import Kind # no-cython-lint -from cudf._lib.pylibcudf.libcudf.aggregation import \ +from pylibcudf.libcudf.aggregation import Kind # no-cython-lint +from pylibcudf.libcudf.aggregation import \ correlation_type as CorrelationType # no-cython-lint -from cudf._lib.pylibcudf.libcudf.aggregation import \ +from pylibcudf.libcudf.aggregation import \ ewm_history as EWMHistory # no-cython-lint -from cudf._lib.pylibcudf.libcudf.aggregation import \ +from pylibcudf.libcudf.aggregation import \ rank_method as RankMethod # no-cython-lint -from cudf._lib.pylibcudf.libcudf.aggregation import \ +from pylibcudf.libcudf.aggregation import \ rank_percentage as RankPercentage # no-cython-lint -from cudf._lib.pylibcudf.libcudf.aggregation import ( # no-cython-lint - udf_type as UdfType, -) +from pylibcudf.libcudf.aggregation import udf_type as UdfType # no-cython-lint from .types cimport DataType @@ -71,7 +68,7 @@ cdef class Aggregation: """A type of aggregation to perform. Aggregations are passed to APIs like - :py:func:`~cudf._lib.pylibcudf.groupby.GroupBy.aggregate` to indicate what + :py:func:`~pylibcudf.groupby.GroupBy.aggregate` to indicate what operations to perform. Using a class for aggregations provides a unified API for handling parametrizable aggregations. This class should never be instantiated directly, only via one of the factory functions. diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd b/python/pylibcudf/pylibcudf/binaryop.pxd similarity index 90% rename from python/cudf/cudf/_lib/pylibcudf/binaryop.pxd rename to python/pylibcudf/pylibcudf/binaryop.pxd index 2411e28ac66..06625e9e2db 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd +++ b/python/pylibcudf/pylibcudf/binaryop.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp cimport bool - -from cudf._lib.pylibcudf.libcudf.binaryop cimport binary_operator +from pylibcudf.libcudf.binaryop cimport binary_operator from .column cimport Column from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx b/python/pylibcudf/pylibcudf/binaryop.pyx similarity index 86% rename from python/cudf/cudf/_lib/pylibcudf/binaryop.pyx rename to python/pylibcudf/pylibcudf/binaryop.pyx index 44d9f4ad04a..5a67f4d6cdb 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx +++ b/python/pylibcudf/pylibcudf/binaryop.pyx @@ -5,12 +5,11 @@ from cython.operator import dereference from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf cimport binaryop as cpp_binaryop +from pylibcudf.libcudf.binaryop cimport binary_operator +from pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf cimport binaryop as cpp_binaryop -from cudf._lib.pylibcudf.libcudf.binaryop cimport binary_operator -from cudf._lib.pylibcudf.libcudf.column.column cimport column - -from cudf._lib.pylibcudf.libcudf.binaryop import \ +from pylibcudf.libcudf.binaryop import \ binary_operator as BinaryOperator # no-cython-lint from .column cimport Column @@ -27,9 +26,9 @@ cpdef Column binary_operation( """Perform a binary operation between a column and another column or scalar. ``lhs`` and ``rhs`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`, but at least one must be a - :py:class:`~cudf._lib.pylibcudf.column.Column`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`, but at least one must be a + :py:class:`~pylibcudf.column.Column`. For details, see :cpp:func:`binary_operation`. diff --git a/python/cudf/cudf/_lib/pylibcudf/column.pxd b/python/pylibcudf/pylibcudf/column.pxd similarity index 84% rename from python/cudf/cudf/_lib/pylibcudf/column.pxd rename to python/pylibcudf/pylibcudf/column.pxd index 13ee0a70681..92d63e4e495 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column.pxd +++ b/python/pylibcudf/pylibcudf/column.pxd @@ -2,16 +2,13 @@ from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) -from cudf._lib.pylibcudf.libcudf.types cimport bitmask_type, size_type +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view +from pylibcudf.libcudf.types cimport bitmask_type, size_type from .gpumemoryview cimport gpumemoryview from .types cimport DataType diff --git a/python/cudf/cudf/_lib/pylibcudf/column.pyx b/python/pylibcudf/pylibcudf/column.pyx similarity index 98% rename from python/cudf/cudf/_lib/pylibcudf/column.pyx rename to python/pylibcudf/pylibcudf/column.pyx index 1d9902b0374..a37a12fc7e1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column.pyx +++ b/python/pylibcudf/pylibcudf/column.pyx @@ -3,16 +3,13 @@ from cython.operator cimport dereference from libcpp.memory cimport make_unique, unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf.column.column cimport column, column_contents +from pylibcudf.libcudf.column.column_factories cimport make_column_from_scalar +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport size_type from rmm._lib.device_buffer cimport DeviceBuffer -from cudf._lib.pylibcudf.libcudf.column.column cimport column, column_contents -from cudf._lib.pylibcudf.libcudf.column.column_factories cimport ( - make_column_from_scalar, -) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type - from .gpumemoryview cimport gpumemoryview from .scalar cimport Scalar from .types cimport DataType, size_of, type_id diff --git a/python/cudf/cudf/_lib/pylibcudf/column_factories.pxd b/python/pylibcudf/pylibcudf/column_factories.pxd similarity index 92% rename from python/cudf/cudf/_lib/pylibcudf/column_factories.pxd rename to python/pylibcudf/pylibcudf/column_factories.pxd index 9dbd74ab16c..fef02359240 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column_factories.pxd +++ b/python/pylibcudf/pylibcudf/column_factories.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.types cimport mask_state, size_type +from pylibcudf.libcudf.types cimport mask_state, size_type from .column cimport Column from .types cimport DataType, size_type, type_id diff --git a/python/cudf/cudf/_lib/pylibcudf/column_factories.pyx b/python/pylibcudf/pylibcudf/column_factories.pyx similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/column_factories.pyx rename to python/pylibcudf/pylibcudf/column_factories.pyx index ef7f512f0e5..4601cba515a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column_factories.pyx +++ b/python/pylibcudf/pylibcudf/column_factories.pyx @@ -1,9 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_factories cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_factories cimport ( make_duration_column as cpp_make_duration_column, make_empty_column as cpp_make_empty_column, make_fixed_point_column as cpp_make_fixed_point_column, @@ -11,7 +10,7 @@ from cudf._lib.pylibcudf.libcudf.column.column_factories cimport ( make_numeric_column as cpp_make_numeric_column, make_timestamp_column as cpp_make_timestamp_column, ) -from cudf._lib.pylibcudf.libcudf.types cimport mask_state, size_type +from pylibcudf.libcudf.types cimport mask_state, size_type from .types cimport DataType, type_id diff --git a/python/cudf/cudf/_lib/pylibcudf/concatenate.pxd b/python/pylibcudf/pylibcudf/concatenate.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/concatenate.pxd rename to python/pylibcudf/pylibcudf/concatenate.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/concatenate.pyx b/python/pylibcudf/pylibcudf/concatenate.pyx similarity index 80% rename from python/cudf/cudf/_lib/pylibcudf/concatenate.pyx rename to python/pylibcudf/pylibcudf/concatenate.pyx index 5e40f921b2c..8bdcc086e0f 100644 --- a/python/cudf/cudf/_lib/pylibcudf/concatenate.pyx +++ b/python/pylibcudf/pylibcudf/concatenate.pyx @@ -3,12 +3,11 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf cimport concatenate as cpp_concatenate -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf cimport concatenate as cpp_concatenate +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/copying.pxd b/python/pylibcudf/pylibcudf/copying.pxd similarity index 94% rename from python/cudf/cudf/_lib/pylibcudf/copying.pxd rename to python/pylibcudf/pylibcudf/copying.pxd index 06543d3ca92..7dfed437673 100644 --- a/python/cudf/cudf/_lib/pylibcudf/copying.pxd +++ b/python/pylibcudf/pylibcudf/copying.pxd @@ -1,12 +1,11 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. from libcpp cimport bool as cbool - -from cudf._lib.pylibcudf.libcudf.copying cimport ( +from pylibcudf.libcudf.copying cimport ( mask_allocation_policy, out_of_bounds_policy, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type from .column cimport Column from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/pylibcudf/copying.pyx b/python/pylibcudf/pylibcudf/copying.pyx similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/copying.pyx rename to python/pylibcudf/pylibcudf/copying.pyx index 2d59deb3864..9743119d92a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/copying.pyx +++ b/python/pylibcudf/pylibcudf/copying.pyx @@ -6,29 +6,28 @@ from libcpp.functional cimport reference_wrapper from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - # TODO: We want to make cpp a more full-featured package so that we can access # directly from that. It will make namespacing much cleaner in pylibcudf. What # we really want here would be # cimport libcudf... libcudf.copying.algo(...) -from cudf._lib.pylibcudf.libcudf cimport copying as cpp_copying -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf cimport copying as cpp_copying +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.copying cimport ( +from pylibcudf.libcudf.copying cimport ( mask_allocation_policy, out_of_bounds_policy, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type -from cudf._lib.pylibcudf.libcudf.copying import \ +from pylibcudf.libcudf.copying import \ mask_allocation_policy as MaskAllocationPolicy # no-cython-lint -from cudf._lib.pylibcudf.libcudf.copying import \ +from pylibcudf.libcudf.copying import \ out_of_bounds_policy as OutOfBoundsPolicy # no-cython-lint from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/datetime.pxd b/python/pylibcudf/pylibcudf/datetime.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/datetime.pxd rename to python/pylibcudf/pylibcudf/datetime.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/datetime.pyx b/python/pylibcudf/pylibcudf/datetime.pyx similarity index 78% rename from python/cudf/cudf/_lib/pylibcudf/datetime.pyx rename to python/pylibcudf/pylibcudf/datetime.pyx index 82351327de6..0ddc68bcb9d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/datetime.pyx +++ b/python/pylibcudf/pylibcudf/datetime.pyx @@ -1,11 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.datetime cimport ( - extract_year as cpp_extract_year, -) +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.datetime cimport extract_year as cpp_extract_year from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/exception_handler.pxd b/python/pylibcudf/pylibcudf/exception_handler.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/exception_handler.pxd rename to python/pylibcudf/pylibcudf/exception_handler.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/experimental.pxd b/python/pylibcudf/pylibcudf/experimental.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/experimental.pxd rename to python/pylibcudf/pylibcudf/experimental.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/experimental.pyx b/python/pylibcudf/pylibcudf/experimental.pyx similarity index 92% rename from python/cudf/cudf/_lib/pylibcudf/experimental.pyx rename to python/pylibcudf/pylibcudf/experimental.pyx index 1e2a682d879..b25a53e13b2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/experimental.pyx +++ b/python/pylibcudf/pylibcudf/experimental.pyx @@ -2,8 +2,7 @@ from libcpp cimport bool from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf cimport experimental as cpp_experimental +from pylibcudf.libcudf cimport experimental as cpp_experimental cpdef enable_prefetching(str key): diff --git a/python/cudf/cudf/_lib/pylibcudf/expressions.pxd b/python/pylibcudf/pylibcudf/expressions.pxd similarity index 91% rename from python/cudf/cudf/_lib/pylibcudf/expressions.pxd rename to python/pylibcudf/pylibcudf/expressions.pxd index 64825b89d9f..65660b7c449 100644 --- a/python/cudf/cudf/_lib/pylibcudf/expressions.pxd +++ b/python/pylibcudf/pylibcudf/expressions.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.expressions cimport ( +from pylibcudf.libcudf.expressions cimport ( ast_operator, expression, table_reference, diff --git a/python/cudf/cudf/_lib/pylibcudf/expressions.pyx b/python/pylibcudf/pylibcudf/expressions.pyx similarity index 94% rename from python/cudf/cudf/_lib/pylibcudf/expressions.pyx rename to python/pylibcudf/pylibcudf/expressions.pyx index b983a617533..a44c9e25987 100644 --- a/python/cudf/cudf/_lib/pylibcudf/expressions.pyx +++ b/python/pylibcudf/pylibcudf/expressions.pyx @@ -1,7 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.expressions import \ +from pylibcudf.libcudf.expressions import \ ast_operator as ASTOperator # no-cython-lint -from cudf._lib.pylibcudf.libcudf.expressions import \ +from pylibcudf.libcudf.expressions import \ table_reference as TableReference # no-cython-lint from cython.operator cimport dereference @@ -9,22 +9,21 @@ from libc.stdint cimport int32_t, int64_t from libcpp.memory cimport make_unique, unique_ptr from libcpp.string cimport string from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf cimport expressions as libcudf_exp -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport ( +from pylibcudf.libcudf cimport expressions as libcudf_exp +from pylibcudf.libcudf.scalar.scalar cimport ( duration_scalar, numeric_scalar, string_scalar, timestamp_scalar, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type, type_id -from cudf._lib.pylibcudf.libcudf.wrappers.durations cimport ( +from pylibcudf.libcudf.types cimport size_type, type_id +from pylibcudf.libcudf.wrappers.durations cimport ( duration_ms, duration_ns, duration_s, duration_us, ) -from cudf._lib.pylibcudf.libcudf.wrappers.timestamps cimport ( +from pylibcudf.libcudf.wrappers.timestamps cimport ( timestamp_ms, timestamp_ns, timestamp_s, diff --git a/python/cudf/cudf/_lib/pylibcudf/filling.pxd b/python/pylibcudf/pylibcudf/filling.pxd similarity index 90% rename from python/cudf/cudf/_lib/pylibcudf/filling.pxd rename to python/pylibcudf/pylibcudf/filling.pxd index 3560ebf2ea2..b9345f8cd42 100644 --- a/python/cudf/cudf/_lib/pylibcudf/filling.pxd +++ b/python/pylibcudf/pylibcudf/filling.pxd @@ -1,5 +1,5 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type from .column cimport Column from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/pylibcudf/filling.pyx b/python/pylibcudf/pylibcudf/filling.pyx similarity index 94% rename from python/cudf/cudf/_lib/pylibcudf/filling.pyx rename to python/pylibcudf/pylibcudf/filling.pyx index 05f67681428..61b430e64aa 100644 --- a/python/cudf/cudf/_lib/pylibcudf/filling.pyx +++ b/python/pylibcudf/pylibcudf/filling.pyx @@ -3,16 +3,15 @@ from cython.operator cimport dereference from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.filling cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.filling cimport ( fill as cpp_fill, fill_in_place as cpp_fill_in_place, repeat as cpp_repeat, sequence as cpp_sequence, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport size_type from .column cimport Column from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/pylibcudf/gpumemoryview.pxd b/python/pylibcudf/pylibcudf/gpumemoryview.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/gpumemoryview.pxd rename to python/pylibcudf/pylibcudf/gpumemoryview.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/gpumemoryview.pyx b/python/pylibcudf/pylibcudf/gpumemoryview.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/gpumemoryview.pyx rename to python/pylibcudf/pylibcudf/gpumemoryview.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pxd b/python/pylibcudf/pylibcudf/groupby.pxd similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/groupby.pxd rename to python/pylibcudf/pylibcudf/groupby.pxd index eaa05c26986..79af2f1b746 100644 --- a/python/cudf/cudf/_lib/pylibcudf/groupby.pxd +++ b/python/pylibcudf/pylibcudf/groupby.pxd @@ -3,20 +3,19 @@ from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.aggregation cimport ( +from pylibcudf.libcudf.aggregation cimport ( aggregation, groupby_aggregation, groupby_scan_aggregation, ) -from cudf._lib.pylibcudf.libcudf.groupby cimport ( +from pylibcudf.libcudf.groupby cimport ( aggregation_request, aggregation_result, groupby, scan_request, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport null_order, order +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport null_order, order from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx b/python/pylibcudf/pylibcudf/groupby.pyx similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/groupby.pyx rename to python/pylibcudf/pylibcudf/groupby.pyx index f5bb46ca6a2..ae5d33aaa46 100644 --- a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx +++ b/python/pylibcudf/pylibcudf/groupby.pyx @@ -6,18 +6,17 @@ from libcpp.memory cimport make_unique, unique_ptr from libcpp.pair cimport pair from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.groupby cimport ( +from pylibcudf.libcudf.groupby cimport ( aggregation_request, aggregation_result, groupby, groups, scan_request, ) -from cudf._lib.pylibcudf.libcudf.replace cimport replace_policy -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.replace cimport replace_policy +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport size_type from .aggregation cimport Aggregation from .column cimport Column @@ -156,7 +155,7 @@ cdef class GroupBy: Parameters ---------- requests : List[GroupByRequest] - The list of `~.cudf._lib.pylibcudf.groupby.GroupByRequest` , each + The list of `~.pylibcudf.groupby.GroupByRequest` , each representing a set of aggregations to perform on a given column of values. Returns @@ -188,7 +187,7 @@ cdef class GroupBy: Parameters ---------- requests : List[GroupByRequest] - The list of `~.cudf._lib.pylibcudf.groupby.GroupByRequest` , each + The list of `~.pylibcudf.groupby.GroupByRequest` , each representing a set of aggregations to perform on a given column of values. Returns diff --git a/python/cudf/cudf/_lib/pylibcudf/interop.pyx b/python/pylibcudf/pylibcudf/interop.pyx similarity index 98% rename from python/cudf/cudf/_lib/pylibcudf/interop.pyx rename to python/pylibcudf/pylibcudf/interop.pyx index caa19724786..d54e5b7ba1f 100644 --- a/python/cudf/cudf/_lib/pylibcudf/interop.pyx +++ b/python/pylibcudf/pylibcudf/interop.pyx @@ -11,8 +11,8 @@ from functools import singledispatch from pyarrow import lib as pa -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.interop cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.interop cimport ( ArrowArray, ArrowArrayStream, ArrowSchema, @@ -22,7 +22,7 @@ from cudf._lib.pylibcudf.libcudf.interop cimport ( to_arrow_host_raw, to_arrow_schema_raw, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table cimport table from . cimport copying from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt b/python/pylibcudf/pylibcudf/io/CMakeLists.txt similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt rename to python/pylibcudf/pylibcudf/io/CMakeLists.txt diff --git a/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd b/python/pylibcudf/pylibcudf/io/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd rename to python/pylibcudf/pylibcudf/io/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/io/__init__.py b/python/pylibcudf/pylibcudf/io/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/io/__init__.py rename to python/pylibcudf/pylibcudf/io/__init__.py diff --git a/python/pylibcudf/pylibcudf/io/avro.pxd b/python/pylibcudf/pylibcudf/io/avro.pxd new file mode 100644 index 00000000000..8696fcb3c15 --- /dev/null +++ b/python/pylibcudf/pylibcudf/io/avro.pxd @@ -0,0 +1,12 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +from pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from pylibcudf.libcudf.io.avro cimport avro_reader_options +from pylibcudf.libcudf.types cimport size_type + + +cpdef TableWithMetadata read_avro( + SourceInfo source_info, + list columns = *, + size_type skip_rows = *, + size_type num_rows = * +) diff --git a/python/cudf/cudf/_lib/pylibcudf/io/avro.pyx b/python/pylibcudf/pylibcudf/io/avro.pyx similarity index 89% rename from python/cudf/cudf/_lib/pylibcudf/io/avro.pyx rename to python/pylibcudf/pylibcudf/io/avro.pyx index 538bd8aa322..667c67f4c36 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/avro.pyx +++ b/python/pylibcudf/pylibcudf/io/avro.pyx @@ -3,13 +3,12 @@ from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata -from cudf._lib.pylibcudf.libcudf.io.avro cimport ( +from pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from pylibcudf.libcudf.io.avro cimport ( avro_reader_options, read_avro as cpp_read_avro, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type cpdef TableWithMetadata read_avro( diff --git a/python/cudf/cudf/_lib/pylibcudf/io/csv.pyx b/python/pylibcudf/pylibcudf/io/csv.pyx similarity index 97% rename from python/cudf/cudf/_lib/pylibcudf/io/csv.pyx rename to python/pylibcudf/pylibcudf/io/csv.pyx index e9efb5befee..b53d6771cd6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/csv.pyx +++ b/python/pylibcudf/pylibcudf/io/csv.pyx @@ -5,19 +5,18 @@ from libcpp.map cimport map from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata -from cudf._lib.pylibcudf.libcudf.io.csv cimport ( +from pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from pylibcudf.libcudf.io.csv cimport ( csv_reader_options, read_csv as cpp_read_csv, ) -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.libcudf.io.types cimport ( compression_type, quote_style, table_with_metadata, ) -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type -from cudf._lib.pylibcudf.types cimport DataType +from pylibcudf.libcudf.types cimport data_type, size_type +from pylibcudf.types cimport DataType cdef tuple _process_parse_dates_hex(list cols): diff --git a/python/cudf/cudf/_lib/pylibcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/io/datasource.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/io/datasource.pxd rename to python/pylibcudf/pylibcudf/io/datasource.pxd index a0a9c3fa0d4..05c03dceee2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/datasource.pxd +++ b/python/pylibcudf/pylibcudf/io/datasource.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport shared_ptr - -from cudf._lib.pylibcudf.libcudf.io.arrow_io_source cimport arrow_io_source -from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource +from pylibcudf.libcudf.io.arrow_io_source cimport arrow_io_source +from pylibcudf.libcudf.io.datasource cimport datasource cdef class Datasource: diff --git a/python/cudf/cudf/_lib/pylibcudf/io/datasource.pyx b/python/pylibcudf/pylibcudf/io/datasource.pyx similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/io/datasource.pyx rename to python/pylibcudf/pylibcudf/io/datasource.pyx index 8f265f585de..6cc509b74cb 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/datasource.pyx +++ b/python/pylibcudf/pylibcudf/io/datasource.pyx @@ -3,9 +3,8 @@ from libcpp.memory cimport shared_ptr from pyarrow.includes.libarrow cimport CRandomAccessFile from pyarrow.lib cimport NativeFile - -from cudf._lib.pylibcudf.libcudf.io.arrow_io_source cimport arrow_io_source -from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource +from pylibcudf.libcudf.io.arrow_io_source cimport arrow_io_source +from pylibcudf.libcudf.io.datasource cimport datasource import warnings diff --git a/python/cudf/cudf/_lib/pylibcudf/io/json.pxd b/python/pylibcudf/pylibcudf/io/json.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/io/json.pxd rename to python/pylibcudf/pylibcudf/io/json.pxd index 2e0e92a054f..ab9b5b99ce2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/json.pxd +++ b/python/pylibcudf/pylibcudf/io/json.pxd @@ -1,14 +1,13 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp cimport bool - -from cudf._lib.pylibcudf.io.types cimport ( +from pylibcudf.io.types cimport ( SinkInfo, SourceInfo, TableWithMetadata, compression_type, ) -from cudf._lib.pylibcudf.libcudf.io.json cimport json_recovery_mode_t -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.io.json cimport json_recovery_mode_t +from pylibcudf.libcudf.types cimport size_type cpdef TableWithMetadata read_json( diff --git a/python/cudf/cudf/_lib/pylibcudf/io/json.pyx b/python/pylibcudf/pylibcudf/io/json.pyx similarity index 95% rename from python/cudf/cudf/_lib/pylibcudf/io/json.pyx rename to python/pylibcudf/pylibcudf/io/json.pyx index 2710ee60075..ce086f4a489 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/json.pyx +++ b/python/pylibcudf/pylibcudf/io/json.pyx @@ -5,14 +5,9 @@ from libcpp.map cimport map from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.concatenate cimport concatenate -from cudf._lib.pylibcudf.io.types cimport ( - SinkInfo, - SourceInfo, - TableWithMetadata, -) -from cudf._lib.pylibcudf.libcudf.io.json cimport ( +from pylibcudf.concatenate cimport concatenate +from pylibcudf.io.types cimport SinkInfo, SourceInfo, TableWithMetadata +from pylibcudf.libcudf.io.json cimport ( json_reader_options, json_recovery_mode_t, json_writer_options, @@ -20,13 +15,13 @@ from cudf._lib.pylibcudf.libcudf.io.json cimport ( schema_element, write_json as cpp_write_json, ) -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.libcudf.io.types cimport ( compression_type, table_metadata, table_with_metadata, ) -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type -from cudf._lib.pylibcudf.types cimport DataType +from pylibcudf.libcudf.types cimport data_type, size_type +from pylibcudf.types cimport DataType cdef map[string, schema_element] _generate_schema_map(list dtypes): @@ -270,7 +265,7 @@ cpdef void write_json( str false_value = "false" ): """ - Writes a :py:class:`~cudf._lib.pylibcudf.table.Table` to JSON format. + Writes a :py:class:`~pylibcudf.table.Table` to JSON format. Parameters ---------- diff --git a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd b/python/pylibcudf/pylibcudf/io/parquet.pxd similarity index 72% rename from python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd rename to python/pylibcudf/pylibcudf/io/parquet.pxd index 93ef849b813..47458b00159 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd +++ b/python/pylibcudf/pylibcudf/io/parquet.pxd @@ -3,14 +3,13 @@ from libc.stdint cimport int64_t from libcpp cimport bool from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.expressions cimport Expression -from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata -from cudf._lib.pylibcudf.libcudf.io.parquet cimport ( +from pylibcudf.expressions cimport Expression +from pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_reader as cpp_chunked_parquet_reader, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type -from cudf._lib.pylibcudf.types cimport DataType +from pylibcudf.libcudf.types cimport size_type +from pylibcudf.types cimport DataType cdef class ChunkedParquetReader: diff --git a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx similarity index 93% rename from python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx rename to python/pylibcudf/pylibcudf/io/parquet.pyx index 84a79f9565f..fb5244a2a9e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -5,17 +5,16 @@ from libcpp cimport bool from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.expressions cimport Expression -from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata -from cudf._lib.pylibcudf.libcudf.expressions cimport expression -from cudf._lib.pylibcudf.libcudf.io.parquet cimport ( +from pylibcudf.expressions cimport Expression +from pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from pylibcudf.libcudf.expressions cimport expression +from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_reader as cpp_chunked_parquet_reader, parquet_reader_options, read_parquet as cpp_read_parquet, ) -from cudf._lib.pylibcudf.libcudf.io.types cimport table_with_metadata -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.io.types cimport table_with_metadata +from pylibcudf.libcudf.types cimport size_type cdef parquet_reader_options _setup_parquet_reader_options( @@ -169,7 +168,7 @@ cpdef read_parquet( row_groups : list[list[size_type]], default None List of row groups to be read. filters : Expression, default None - An AST :py:class:`cudf._lib.pylibcudf.expressions.Expression` + An AST :py:class:`pylibcudf.expressions.Expression` to use for predicate pushdown. convert_strings_to_categories : bool, default False Whether to convert string columns to the category type diff --git a/python/cudf/cudf/_lib/pylibcudf/io/types.pxd b/python/pylibcudf/pylibcudf/io/types.pxd similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/io/types.pxd rename to python/pylibcudf/pylibcudf/io/types.pxd index 0094bf6032c..0ab28cb0973 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/io/types.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.libcudf.io.data_sink cimport data_sink +from pylibcudf.libcudf.io.types cimport ( column_encoding, column_in_metadata, column_name_info, @@ -19,7 +18,7 @@ from cudf._lib.pylibcudf.libcudf.io.types cimport ( table_metadata, table_with_metadata, ) -from cudf._lib.pylibcudf.table cimport Table +from pylibcudf.table cimport Table cdef class TableWithMetadata: diff --git a/python/cudf/cudf/_lib/pylibcudf/io/types.pyx b/python/pylibcudf/pylibcudf/io/types.pyx similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/io/types.pyx rename to python/pylibcudf/pylibcudf/io/types.pyx index 95fa7d4c2ee..1600a805b37 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/types.pyx +++ b/python/pylibcudf/pylibcudf/io/types.pyx @@ -6,11 +6,10 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.io.datasource cimport Datasource -from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink -from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource -from cudf._lib.pylibcudf.libcudf.io.types cimport ( +from pylibcudf.io.datasource cimport Datasource +from pylibcudf.libcudf.io.data_sink cimport data_sink +from pylibcudf.libcudf.io.datasource cimport datasource +from pylibcudf.libcudf.io.types cimport ( column_name_info, host_buffer, source_info, @@ -22,9 +21,9 @@ import errno import io import os -from cudf._lib.pylibcudf.libcudf.io.json import \ +from pylibcudf.libcudf.io.json import \ json_recovery_mode_t as JSONRecoveryMode # no-cython-lint -from cudf._lib.pylibcudf.libcudf.io.types import \ +from pylibcudf.libcudf.io.types import \ compression_type as CompressionType # no-cython-lint diff --git a/python/cudf/cudf/_lib/pylibcudf/join.pxd b/python/pylibcudf/pylibcudf/join.pxd similarity index 91% rename from python/cudf/cudf/_lib/pylibcudf/join.pxd rename to python/pylibcudf/pylibcudf/join.pxd index 83b4776c16e..06969b4a2db 100644 --- a/python/cudf/cudf/_lib/pylibcudf/join.pxd +++ b/python/pylibcudf/pylibcudf/join.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.types cimport null_equality +from pylibcudf.libcudf.types cimport null_equality from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/join.pyx b/python/pylibcudf/pylibcudf/join.pyx similarity index 95% rename from python/cudf/cudf/_lib/pylibcudf/join.pyx rename to python/pylibcudf/pylibcudf/join.pyx index 2ded84d84d1..25664286f19 100644 --- a/python/cudf/cudf/_lib/pylibcudf/join.pyx +++ b/python/pylibcudf/pylibcudf/join.pyx @@ -4,14 +4,13 @@ from cython.operator import dereference from libcpp.memory cimport make_unique, unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf cimport join as cpp_join +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport null_equality from rmm._lib.device_buffer cimport device_buffer -from cudf._lib.pylibcudf.libcudf cimport join as cpp_join -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport null_equality - from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/CMakeLists.txt b/python/pylibcudf/pylibcudf/libcudf/CMakeLists.txt similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/CMakeLists.txt rename to python/pylibcudf/pylibcudf/libcudf/CMakeLists.txt diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/__init__.py b/python/pylibcudf/pylibcudf/libcudf/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/aggregation.pxd b/python/pylibcudf/pylibcudf/libcudf/aggregation.pxd similarity index 98% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/aggregation.pxd rename to python/pylibcudf/pylibcudf/libcudf/aggregation.pxd index fe04db52094..58c579b86de 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/aggregation.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/aggregation.pxd @@ -5,8 +5,7 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.types cimport ( data_type, interpolation, nan_equality, diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/aggregation.pyx b/python/pylibcudf/pylibcudf/libcudf/aggregation.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/aggregation.pyx rename to python/pylibcudf/pylibcudf/libcudf/aggregation.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd b/python/pylibcudf/pylibcudf/libcudf/binaryop.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd rename to python/pylibcudf/pylibcudf/libcudf/binaryop.pxd index 78da5980db4..d39767b4aa8 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/binaryop.pxd @@ -4,12 +4,11 @@ from libc.stdint cimport int32_t from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.exception_handler cimport libcudf_exception_handler -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.exception_handler cimport libcudf_exception_handler +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/binaryop.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pyx b/python/pylibcudf/pylibcudf/libcudf/binaryop.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pyx rename to python/pylibcudf/pylibcudf/libcudf/binaryop.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/column/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/column/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/column/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/__init__.py b/python/pylibcudf/pylibcudf/libcudf/column/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/column/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/column/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/column.pxd b/python/pylibcudf/pylibcudf/libcudf/column/column.pxd similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/column/column.pxd rename to python/pylibcudf/pylibcudf/libcudf/column/column.pxd index dd184d31cc6..7a369701bbd 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/column.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/column/column.pxd @@ -3,14 +3,13 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from rmm._lib.device_buffer cimport device_buffer - -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type +from pylibcudf.libcudf.types cimport data_type, size_type + +from rmm._lib.device_buffer cimport device_buffer cdef extern from "cudf/column/column.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/column_factories.pxd b/python/pylibcudf/pylibcudf/libcudf/column/column_factories.pxd similarity index 93% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/column/column_factories.pxd rename to python/pylibcudf/pylibcudf/libcudf/column/column_factories.pxd index 2faff21a77b..f1a326bcd40 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/column_factories.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/column/column_factories.pxd @@ -1,12 +1,9 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from rmm._lib.device_buffer cimport device_buffer - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport ( bitmask_type, data_type, mask_state, @@ -14,6 +11,8 @@ from cudf._lib.pylibcudf.libcudf.types cimport ( type_id, ) +from rmm._lib.device_buffer cimport device_buffer + cdef extern from "cudf/column/column_factories.hpp" namespace "cudf" nogil: cdef unique_ptr[column] make_numeric_column(data_type type, diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/column_view.pxd b/python/pylibcudf/pylibcudf/libcudf/column/column_view.pxd similarity index 97% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/column/column_view.pxd rename to python/pylibcudf/pylibcudf/libcudf/column/column_view.pxd index c6403babe89..c0e971eb5bd 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/column/column_view.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/column/column_view.pxd @@ -2,12 +2,7 @@ from libcpp cimport bool from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.types cimport ( - bitmask_type, - data_type, - size_type, -) +from pylibcudf.libcudf.types cimport bitmask_type, data_type, size_type cdef extern from "cudf/column/column_view.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/concatenate.pxd b/python/pylibcudf/pylibcudf/libcudf/concatenate.pxd similarity index 77% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/concatenate.pxd rename to python/pylibcudf/pylibcudf/libcudf/concatenate.pxd index 0c362390ff2..92f5a185a54 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/concatenate.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/concatenate.pxd @@ -2,13 +2,12 @@ from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector +from pylibcudf.libcudf.column.column cimport column, column_view +from pylibcudf.libcudf.table.table cimport table, table_view +from pylibcudf.libcudf.utilities.host_span cimport host_span from rmm._lib.device_buffer cimport device_buffer -from cudf._lib.pylibcudf.libcudf.column.column cimport column, column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table, table_view -from cudf._lib.pylibcudf.libcudf.utilities.host_span cimport host_span - cdef extern from "cudf/concatenate.hpp" namespace "cudf" nogil: # The versions of concatenate taking vectors don't exist in libcudf diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/contiguous_split.pxd b/python/pylibcudf/pylibcudf/libcudf/contiguous_split.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/contiguous_split.pxd rename to python/pylibcudf/pylibcudf/libcudf/contiguous_split.pxd index b06feacb016..cadac6a0022 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/contiguous_split.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/contiguous_split.pxd @@ -3,12 +3,11 @@ from libc.stdint cimport uint8_t from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type from rmm._lib.device_buffer cimport device_buffer -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type - cdef extern from "cudf/contiguous_split.hpp" namespace "cudf" nogil: cdef cppclass packed_columns: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pxd b/python/pylibcudf/pylibcudf/libcudf/copying.pxd similarity index 90% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pxd rename to python/pylibcudf/pylibcudf/libcudf/copying.pxd index af3a16ad01b..4d4a4ba9b89 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/copying.pxd @@ -5,19 +5,18 @@ from libcpp cimport bool from libcpp.functional cimport reference_wrapper from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from rmm._lib.device_buffer cimport device_buffer - -from cudf._lib.pylibcudf.exception_handler cimport libcudf_exception_handler -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.exception_handler cimport libcudf_exception_handler +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type + +from rmm._lib.device_buffer cimport device_buffer ctypedef const scalar constscalar diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pyx b/python/pylibcudf/pylibcudf/libcudf/copying.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/copying.pyx rename to python/pylibcudf/pylibcudf/libcudf/copying.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/datetime.pxd b/python/pylibcudf/pylibcudf/libcudf/datetime.pxd similarity index 92% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/datetime.pxd rename to python/pylibcudf/pylibcudf/libcudf/datetime.pxd index 7db77b9c7c5..a4465343197 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/datetime.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/datetime.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar cdef extern from "cudf/datetime.hpp" namespace "cudf::datetime" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/experimental.pxd b/python/pylibcudf/pylibcudf/libcudf/experimental.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/experimental.pxd rename to python/pylibcudf/pylibcudf/libcudf/experimental.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pxd b/python/pylibcudf/pylibcudf/libcudf/expressions.pxd similarity index 90% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pxd rename to python/pylibcudf/pylibcudf/libcudf/expressions.pxd index 427e16d4ff8..5ba2dff6074 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/expressions.pxd @@ -3,15 +3,14 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.scalar.scalar cimport ( duration_scalar, numeric_scalar, timestamp_scalar, ) -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/ast/expressions.hpp" namespace "cudf::ast" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pyx b/python/pylibcudf/pylibcudf/libcudf/expressions.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pyx rename to python/pylibcudf/pylibcudf/libcudf/expressions.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/filling.pxd b/python/pylibcudf/pylibcudf/libcudf/filling.pxd similarity index 74% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/filling.pxd rename to python/pylibcudf/pylibcudf/libcudf/filling.pxd index 16ed682f930..7bed80050d2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/filling.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/filling.pxd @@ -2,16 +2,15 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/filling.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/groupby.pxd b/python/pylibcudf/pylibcudf/libcudf/groupby.pxd similarity index 83% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/groupby.pxd rename to python/pylibcudf/pylibcudf/libcudf/groupby.pxd index 16607cc3711..848462131fe 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/groupby.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/groupby.pxd @@ -5,25 +5,24 @@ from libcpp.functional cimport reference_wrapper from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.aggregation cimport ( +from pylibcudf.libcudf.aggregation cimport ( groupby_aggregation, groupby_scan_aggregation, ) -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.replace cimport replace_policy -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.replace cimport replace_policy +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport ( null_order, null_policy, order, size_type, sorted, ) -from cudf._lib.pylibcudf.libcudf.utilities.host_span cimport host_span +from pylibcudf.libcudf.utilities.host_span cimport host_span # workaround for https://github.com/cython/cython/issues/3885 ctypedef const scalar constscalar diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/hash.pxd b/python/pylibcudf/pylibcudf/libcudf/hash.pxd similarity index 86% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/hash.pxd rename to python/pylibcudf/pylibcudf/libcudf/hash.pxd index 5346252df69..51678ba69d8 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/hash.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/hash.pxd @@ -3,10 +3,9 @@ from libc.stdint cimport uint32_t, uint64_t from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/hashing.hpp" namespace "cudf::hashing" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/interop.pxd rename to python/pylibcudf/pylibcudf/libcudf/interop.pxd index 24d96b602dc..c7efff2340d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -3,14 +3,11 @@ from libcpp.memory cimport shared_ptr, unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector - -from cudf._lib.types import cudf_to_np_types, np_to_cudf_types - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "dlpack/dlpack.h" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/CMakeLists.txt b/python/pylibcudf/pylibcudf/libcudf/io/CMakeLists.txt similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/CMakeLists.txt rename to python/pylibcudf/pylibcudf/libcudf/io/CMakeLists.txt diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/io/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/__init__.py b/python/pylibcudf/pylibcudf/libcudf/io/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/io/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/arrow_io_source.pxd b/python/pylibcudf/pylibcudf/libcudf/io/arrow_io_source.pxd similarity index 86% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/arrow_io_source.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/arrow_io_source.pxd index 1d2138f8d10..54a913a9ce3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/arrow_io_source.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/arrow_io_source.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.datasource as cudf_io_datasource from libcpp.memory cimport shared_ptr from libcpp.string cimport string from pyarrow.includes.libarrow cimport CRandomAccessFile -cimport cudf._lib.pylibcudf.libcudf.io.datasource as cudf_io_datasource - cdef extern from "cudf/io/arrow_io_source.hpp" \ namespace "cudf::io" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/avro.pxd b/python/pylibcudf/pylibcudf/libcudf/io/avro.pxd similarity index 91% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/avro.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/avro.pxd index 530df5aa8f1..2d76e2f6c80 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/avro.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/avro.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.types as cudf_io_types from libcpp.string cimport string from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/io/avro.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/csv.pxd b/python/pylibcudf/pylibcudf/libcudf/io/csv.pxd similarity index 98% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/csv.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/csv.pxd index b5ff6558cd8..73a6d98650c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/csv.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/csv.pxd @@ -1,15 +1,14 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.types as cudf_io_types +cimport pylibcudf.libcudf.table.table_view as cudf_table_view from libc.stdint cimport uint8_t from libcpp cimport bool from libcpp.map cimport map from libcpp.memory cimport shared_ptr, unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -cimport cudf._lib.pylibcudf.libcudf.table.table_view as cudf_table_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type +from pylibcudf.libcudf.types cimport data_type, size_type cdef extern from "cudf/io/csv.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/data_sink.pxd b/python/pylibcudf/pylibcudf/libcudf/io/data_sink.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/data_sink.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/data_sink.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/datasource.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/json.pxd b/python/pylibcudf/pylibcudf/libcudf/io/json.pxd similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/json.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/json.pxd index 86621ae184f..7514e6c5258 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/json.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/json.pxd @@ -1,15 +1,14 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.types as cudf_io_types +cimport pylibcudf.libcudf.table.table_view as cudf_table_view from libc.stdint cimport int32_t, uint8_t from libcpp cimport bool from libcpp.map cimport map from libcpp.memory cimport shared_ptr, unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -cimport cudf._lib.pylibcudf.libcudf.table.table_view as cudf_table_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type +from pylibcudf.libcudf.types cimport data_type, size_type cdef extern from "cudf/io/json.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/json.pyx b/python/pylibcudf/pylibcudf/libcudf/io/json.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/json.pyx rename to python/pylibcudf/pylibcudf/libcudf/io/json.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/orc.pxd b/python/pylibcudf/pylibcudf/libcudf/io/orc.pxd similarity index 97% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/orc.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/orc.pxd index 25f91849dea..e4a09b8feb2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/orc.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/orc.pxd @@ -1,5 +1,7 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.types as cudf_io_types +cimport pylibcudf.libcudf.table.table_view as cudf_table_view from libc.stdint cimport int64_t, uint8_t from libcpp cimport bool from libcpp.map cimport map @@ -7,10 +9,7 @@ from libcpp.memory cimport shared_ptr, unique_ptr from libcpp.optional cimport optional from libcpp.string cimport string from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -cimport cudf._lib.pylibcudf.libcudf.table.table_view as cudf_table_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type +from pylibcudf.libcudf.types cimport data_type, size_type cdef extern from "cudf/io/orc.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/orc_metadata.pxd b/python/pylibcudf/pylibcudf/libcudf/io/orc_metadata.pxd similarity index 94% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/orc_metadata.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/orc_metadata.pxd index a23655b06f8..db6cb0cdfa5 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/orc_metadata.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/orc_metadata.pxd @@ -1,13 +1,12 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.types as cudf_io_types from libc.stdint cimport int32_t, int64_t, uint32_t, uint64_t from libcpp cimport bool from libcpp.optional cimport optional from libcpp.string cimport string from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -from cudf._lib.variant cimport monostate, variant +from pylibcudf.variant cimport monostate, variant cdef extern from "cudf/io/orc_metadata.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd similarity index 80% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd index d86915c7da9..222d87defa0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd @@ -8,17 +8,25 @@ from libcpp.memory cimport shared_ptr, unique_ptr from libcpp.optional cimport optional from libcpp.string cimport string from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -cimport cudf._lib.pylibcudf.libcudf.table.table_view as cudf_table_view -from cudf._lib.pylibcudf.libcudf.expressions cimport expression -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type +from pylibcudf.libcudf.expressions cimport expression +from pylibcudf.libcudf.io.types cimport ( + compression_type, + dictionary_policy, + partition_info, + sink_info, + source_info, + statistics_freq, + table_input_metadata, + table_with_metadata, +) +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport data_type, size_type cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: cdef cppclass parquet_reader_options: parquet_reader_options() except + - cudf_io_types.source_info get_source_info() except + + source_info get_source_info() except + vector[vector[size_type]] get_row_groups() except + const optional[reference_wrapper[expression]]& get_filter() except + data_type get_timestamp_type() except + @@ -38,13 +46,13 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: @staticmethod parquet_reader_options_builder builder( - cudf_io_types.source_info src + source_info src ) except + cdef cppclass parquet_reader_options_builder: parquet_reader_options_builder() except + parquet_reader_options_builder( - cudf_io_types.source_info src + source_info src ) except + parquet_reader_options_builder& columns( vector[string] col_names @@ -69,15 +77,15 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: ) except + parquet_reader_options build() except + - cdef cudf_io_types.table_with_metadata read_parquet( + cdef table_with_metadata read_parquet( parquet_reader_options args) except + cdef cppclass parquet_writer_options_base: parquet_writer_options_base() except + - cudf_io_types.sink_info get_sink_info() except + - cudf_io_types.compression_type get_compression() except + - cudf_io_types.statistics_freq get_stats_level() except + - const optional[cudf_io_types.table_input_metadata]& get_metadata( + sink_info get_sink_info() except + + compression_type get_compression() except + + statistics_freq get_stats_level() except + + const optional[table_input_metadata]& get_metadata( ) except + size_t get_row_group_size_bytes() except + size_type get_row_group_size_rows() except + @@ -87,16 +95,16 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: bool is_enabled_write_arrow_schema() except + void set_metadata( - cudf_io_types.table_input_metadata m + table_input_metadata m ) except + void set_key_value_metadata( vector[map[string, string]] kvm ) except + void set_stats_level( - cudf_io_types.statistics_freq sf + statistics_freq sf ) except + void set_compression( - cudf_io_types.compression_type compression + compression_type compression ) except + void set_int96_timestamps( bool enabled @@ -111,14 +119,14 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: void set_max_dictionary_size(size_t val) except + void enable_write_v2_headers(bool val) except + void enable_write_arrow_schema(bool val) except + - void set_dictionary_policy(cudf_io_types.dictionary_policy policy) except + + void set_dictionary_policy(dictionary_policy policy) except + cdef cppclass parquet_writer_options(parquet_writer_options_base): parquet_writer_options() except + - cudf_table_view.table_view get_table() except + + table_view get_table() except + string get_column_chunks_file_paths() except + void set_partitions( - vector[cudf_io_types.partition_info] partitions + vector[partition_info] partitions ) except + void set_column_chunks_file_paths( vector[string] column_chunks_file_paths @@ -126,24 +134,24 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: @staticmethod parquet_writer_options_builder builder( - cudf_io_types.sink_info sink_, - cudf_table_view.table_view table_ + sink_info sink_, + table_view table_ ) except + cdef cppclass parquet_writer_options_builder_base[BuilderT, OptionsT]: parquet_writer_options_builder_base() except + BuilderT& metadata( - cudf_io_types.table_input_metadata m + table_input_metadata m ) except + BuilderT& key_value_metadata( vector[map[string, string]] kvm ) except + BuilderT& stats_level( - cudf_io_types.statistics_freq sf + statistics_freq sf ) except + BuilderT& compression( - cudf_io_types.compression_type compression + compression_type compression ) except + BuilderT& int96_timestamps( bool enabled @@ -173,7 +181,7 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: bool val ) except + BuilderT& dictionary_policy( - cudf_io_types.dictionary_policy val + dictionary_policy val ) except + OptionsT build() except + @@ -182,11 +190,11 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: parquet_writer_options]): parquet_writer_options_builder() except + parquet_writer_options_builder( - cudf_io_types.sink_info sink_, - cudf_table_view.table_view table_ + sink_info sink_, + table_view table_ ) except + parquet_writer_options_builder& partitions( - vector[cudf_io_types.partition_info] partitions + vector[partition_info] partitions ) except + parquet_writer_options_builder& column_chunks_file_paths( vector[string] column_chunks_file_paths @@ -201,7 +209,7 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: @staticmethod chunked_parquet_writer_options_builder builder( - cudf_io_types.sink_info sink_, + sink_info sink_, ) except + cdef cppclass chunked_parquet_writer_options_builder( @@ -210,18 +218,18 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: ): chunked_parquet_writer_options_builder() except + chunked_parquet_writer_options_builder( - cudf_io_types.sink_info sink_, + sink_info sink_, ) except + cdef cppclass parquet_chunked_writer: parquet_chunked_writer() except + parquet_chunked_writer(chunked_parquet_writer_options args) except + parquet_chunked_writer& write( - cudf_table_view.table_view table_, + table_view table_, ) except + parquet_chunked_writer& write( - const cudf_table_view.table_view& table_, - const vector[cudf_io_types.partition_info]& partitions, + const table_view& table_, + const vector[partition_info]& partitions, ) except + unique_ptr[vector[uint8_t]] close( vector[string] column_chunks_file_paths, @@ -237,7 +245,7 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: size_t pass_read_limit, const parquet_reader_options& options) except + bool has_next() except + - cudf_io_types.table_with_metadata read_chunk() except + + table_with_metadata read_chunk() except + cdef unique_ptr[vector[uint8_t]] merge_row_group_metadata( const vector[unique_ptr[vector[uint8_t]]]& metadata_list diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet_metadata.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd similarity index 89% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet_metadata.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd index 34a299b73ab..8e6da56c9a6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet_metadata.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet_metadata.pxd @@ -1,12 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.types as cudf_io_types from libc.stdint cimport int64_t from libcpp.string cimport string from libcpp.unordered_map cimport unordered_map from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/io/parquet_metadata.hpp" namespace "cudf::io" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/text.pxd b/python/pylibcudf/pylibcudf/libcudf/io/text.pxd similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/text.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/text.pxd index bec223d4079..14397ef970d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/text.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/text.pxd @@ -4,8 +4,7 @@ from libc.stdint cimport uint64_t from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column cdef extern from "cudf/io/text/byte_range_info.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/timezone.pxd b/python/pylibcudf/pylibcudf/libcudf/io/timezone.pxd similarity index 86% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/timezone.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/timezone.pxd index 88cb5544dc1..676901efcec 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/timezone.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/timezone.pxd @@ -4,8 +4,7 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.optional cimport optional from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table cimport table cdef extern from "cudf/timezone.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pxd b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd similarity index 92% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pxd rename to python/pylibcudf/pylibcudf/libcudf/io/types.pxd index 0a6bddcd907..a3d99807876 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd @@ -1,5 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.io.data_sink as cudf_io_data_sink +cimport pylibcudf.libcudf.io.datasource as cudf_io_datasource +cimport pylibcudf.libcudf.table.table_view as cudf_table_view from libc.stdint cimport int32_t, uint8_t from libcpp cimport bool from libcpp.map cimport map @@ -9,12 +12,8 @@ from libcpp.string cimport string from libcpp.unordered_map cimport unordered_map from libcpp.vector cimport vector from pyarrow.includes.libarrow cimport CRandomAccessFile - -cimport cudf._lib.pylibcudf.libcudf.io.data_sink as cudf_io_data_sink -cimport cudf._lib.pylibcudf.libcudf.io.datasource as cudf_io_datasource -cimport cudf._lib.pylibcudf.libcudf.table.table_view as cudf_table_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/io/types.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pyx b/python/pylibcudf/pylibcudf/libcudf/io/types.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pyx rename to python/pylibcudf/pylibcudf/libcudf/io/types.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/join.pxd b/python/pylibcudf/pylibcudf/libcudf/join.pxd similarity index 88% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/join.pxd rename to python/pylibcudf/pylibcudf/libcudf/join.pxd index 32cd17f7c11..6f6c145b23c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/join.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/join.pxd @@ -4,14 +4,13 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair from libcpp.vector cimport vector +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport null_equality, size_type from rmm._lib.device_uvector cimport device_uvector -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport null_equality, size_type - ctypedef unique_ptr[device_uvector[size_type]] gather_map_type ctypedef pair[gather_map_type, gather_map_type] gather_map_pair_type diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/labeling.pxd b/python/pylibcudf/pylibcudf/libcudf/labeling.pxd similarity index 78% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/labeling.pxd rename to python/pylibcudf/pylibcudf/libcudf/labeling.pxd index 54731bf29af..ec6ef6b2a41 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/labeling.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/labeling.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/labeling/label_bins.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/__init__.py b/python/pylibcudf/pylibcudf/libcudf/lists/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/lists/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/combine.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/combine.pxd similarity index 78% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/combine.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/combine.pxd index 728bd840f71..d077958ce03 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/combine.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/combine.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/lists/combine.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/contains.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/contains.pxd similarity index 75% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/contains.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/contains.pxd index 40bb2e78970..81a5ad46389 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/contains.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/contains.pxd @@ -2,14 +2,11 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.exception_handler cimport libcudf_exception_handler -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.exception_handler cimport libcudf_exception_handler +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar cdef extern from "cudf/lists/contains.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/count_elements.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/count_elements.pxd similarity index 61% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/count_elements.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/count_elements.pxd index ba57a839fbc..e283551ed0c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/count_elements.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/count_elements.pxd @@ -1,11 +1,8 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view cdef extern from "cudf/lists/count_elements.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/explode.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/explode.pxd similarity index 59% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/explode.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/explode.pxd index 622a866f593..c64b2715cca 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/explode.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/explode.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/lists/explode.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/extract.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/extract.pxd similarity index 64% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/extract.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/extract.pxd index 53609ba8830..2ea060d87de 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/extract.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/extract.pxd @@ -1,12 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column, column_view -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column, column_view +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/lists/extract.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/filling.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/filling.pxd similarity index 76% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/filling.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/filling.pxd index 8403fd179f7..54f5a8409b6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/filling.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/filling.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/lists/filling.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/gather.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/gather.pxd similarity index 67% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/gather.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/gather.pxd index ab7ed141365..a762c6aa333 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/gather.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/gather.pxd @@ -1,11 +1,8 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view cdef extern from "cudf/lists/gather.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/lists_column_view.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/lists_column_view.pxd similarity index 86% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/lists_column_view.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/lists_column_view.pxd index 8917a6ac899..f43340a78b0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/lists_column_view.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/lists_column_view.pxd @@ -1,10 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/lists/lists_column_view.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/reverse.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/reverse.pxd similarity index 62% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/reverse.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/reverse.pxd index 0382a5d42c3..43b671ebfa0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/reverse.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/reverse.pxd @@ -1,11 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view cdef extern from "cudf/lists/reverse.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/set_operations.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/set_operations.pxd similarity index 81% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/set_operations.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/set_operations.pxd index eb796897f87..266f04ef6b3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/set_operations.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/set_operations.pxd @@ -1,12 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) -from cudf._lib.pylibcudf.libcudf.types cimport nan_equality, null_equality +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view +from pylibcudf.libcudf.types cimport nan_equality, null_equality cdef extern from "cudf/lists/set_operations.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/sorting.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/sorting.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/sorting.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/sorting.pxd index 337ac73908b..ea45f999c47 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/sorting.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/sorting.pxd @@ -1,12 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) -from cudf._lib.pylibcudf.libcudf.types cimport null_order, order +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view +from pylibcudf.libcudf.types cimport null_order, order cdef extern from "cudf/lists/sorting.hpp" namespace "cudf::lists" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/stream_compaction.pxd b/python/pylibcudf/pylibcudf/libcudf/lists/stream_compaction.pxd similarity index 68% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/lists/stream_compaction.pxd rename to python/pylibcudf/pylibcudf/libcudf/lists/stream_compaction.pxd index b1fcf7800b0..d9df7c3ca2e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/stream_compaction.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/lists/stream_compaction.pxd @@ -1,12 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( - lists_column_view, -) -from cudf._lib.pylibcudf.libcudf.types cimport nan_equality, null_equality +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view +from pylibcudf.libcudf.types cimport nan_equality, null_equality cdef extern from "cudf/lists/stream_compaction.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/merge.pxd b/python/pylibcudf/pylibcudf/libcudf/merge.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/merge.pxd rename to python/pylibcudf/pylibcudf/libcudf/merge.pxd index dacb3dc2d74..6930b7a0d06 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/merge.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/merge.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.types as libcudf_types from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/merge.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/null_mask.pxd b/python/pylibcudf/pylibcudf/libcudf/null_mask.pxd similarity index 80% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/null_mask.pxd rename to python/pylibcudf/pylibcudf/libcudf/null_mask.pxd index 0cab404c05f..3fc2c7e8f1e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/null_mask.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/null_mask.pxd @@ -2,17 +2,12 @@ from libc.stdint cimport int32_t from libcpp.pair cimport pair +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport bitmask_type, mask_state, size_type from rmm._lib.device_buffer cimport device_buffer -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport ( - bitmask_type, - mask_state, - size_type, -) - ctypedef int32_t underlying_type_t_mask_state diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/__init__.py b/python/pylibcudf/pylibcudf/libcudf/nvtext/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/nvtext/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/byte_pair_encode.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/byte_pair_encode.pxd similarity index 73% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/byte_pair_encode.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/byte_pair_encode.pxd index 033a820d2ef..fd768d22704 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/byte_pair_encode.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/byte_pair_encode.pxd @@ -2,10 +2,9 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar cdef extern from "nvtext/byte_pair_encoding.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/edit_distance.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/edit_distance.pxd similarity index 75% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/edit_distance.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/edit_distance.pxd index ca1f6650a5a..d459372fb8f 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/edit_distance.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/edit_distance.pxd @@ -2,9 +2,8 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "nvtext/edit_distance.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/generate_ngrams.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/generate_ngrams.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/generate_ngrams.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/generate_ngrams.pxd index 2034b1c1ee5..eefae746662 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/generate_ngrams.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/generate_ngrams.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type cdef extern from "nvtext/generate_ngrams.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/jaccard.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/jaccard.pxd similarity index 61% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/jaccard.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/jaccard.pxd index 789a1a2c35a..16c5f7f575e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/jaccard.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/jaccard.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "nvtext/jaccard.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/minhash.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/minhash.pxd similarity index 70% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/minhash.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/minhash.pxd index fc5577bf3f9..0c352a5068b 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/minhash.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/minhash.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "nvtext/minhash.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/ngrams_tokenize.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/ngrams_tokenize.pxd similarity index 58% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/ngrams_tokenize.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/ngrams_tokenize.pxd index 229f4d8f5a3..89f6e5edfc4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/ngrams_tokenize.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/ngrams_tokenize.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type cdef extern from "nvtext/ngrams_tokenize.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/normalize.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pxd similarity index 75% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/normalize.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pxd index 65c63b089df..cbf121920e1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/normalize.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pxd @@ -2,9 +2,8 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "nvtext/normalize.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/replace.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/replace.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/replace.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/replace.pxd index aaad28d2684..6bcfa1d9380 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/replace.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/replace.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type cdef extern from "nvtext/replace.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/stemmer.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/stemmer.pxd similarity index 79% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/stemmer.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/stemmer.pxd index 040d4c9de63..673bffa28ae 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/stemmer.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/stemmer.pxd @@ -2,10 +2,9 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "nvtext/stemmer.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/subword_tokenize.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/subword_tokenize.pxd similarity index 92% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/subword_tokenize.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/subword_tokenize.pxd index cce40bcd3f6..aabac0a617b 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/subword_tokenize.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/subword_tokenize.pxd @@ -4,9 +4,8 @@ from libc.stdint cimport uint16_t, uint32_t from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "nvtext/subword_tokenize.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/tokenize.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/tokenize.pxd similarity index 84% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/tokenize.pxd rename to python/pylibcudf/pylibcudf/libcudf/nvtext/tokenize.pxd index 721a6cabd01..34c054cf36f 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/nvtext/tokenize.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/tokenize.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type cdef extern from "nvtext/tokenize.hpp" namespace "nvtext" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/partitioning.pxd b/python/pylibcudf/pylibcudf/libcudf/partitioning.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/partitioning.pxd rename to python/pylibcudf/pylibcudf/libcudf/partitioning.pxd index babb167d2a0..1ea10e8a194 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/partitioning.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/partitioning.pxd @@ -1,15 +1,14 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.types as libcudf_types from libc.stdint cimport uint32_t from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/partitioning.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/quantiles.pxd b/python/pylibcudf/pylibcudf/libcudf/quantiles.pxd similarity index 70% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/quantiles.pxd rename to python/pylibcudf/pylibcudf/libcudf/quantiles.pxd index 32cfec2d4fc..cf2350fc36c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/quantiles.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/quantiles.pxd @@ -3,12 +3,11 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport ( interpolation, null_order, order, diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/reduce.pxd b/python/pylibcudf/pylibcudf/libcudf/reduce.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/reduce.pxd rename to python/pylibcudf/pylibcudf/libcudf/reduce.pxd index 3ae1f1a2906..6d2f4bd23d1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/reduce.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/reduce.pxd @@ -3,15 +3,11 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport pair - -from cudf._lib.pylibcudf.libcudf.aggregation cimport ( - reduce_aggregation, - scan_aggregation, -) -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.aggregation cimport reduce_aggregation, scan_aggregation +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/reduction.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/reduce.pyx b/python/pylibcudf/pylibcudf/libcudf/reduce.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/reduce.pyx rename to python/pylibcudf/pylibcudf/libcudf/reduce.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/replace.pxd b/python/pylibcudf/pylibcudf/libcudf/replace.pxd similarity index 83% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/replace.pxd rename to python/pylibcudf/pylibcudf/libcudf/replace.pxd index e67efbdaba0..4ac44fc946e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/replace.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/replace.pxd @@ -2,15 +2,12 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr - -from cudf._lib.types import cudf_to_np_types, np_to_cudf_types - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.scalar.scalar cimport scalar cdef extern from "cudf/replace.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/replace.pyx b/python/pylibcudf/pylibcudf/libcudf/replace.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/replace.pyx rename to python/pylibcudf/pylibcudf/libcudf/replace.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/reshape.pxd b/python/pylibcudf/pylibcudf/libcudf/reshape.pxd similarity index 57% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/reshape.pxd rename to python/pylibcudf/pylibcudf/libcudf/reshape.pxd index dfd9a71c3d3..446a082ab1b 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/reshape.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/reshape.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2019-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/reshape.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/rolling.pxd b/python/pylibcudf/pylibcudf/libcudf/rolling.pxd similarity index 64% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/rolling.pxd rename to python/pylibcudf/pylibcudf/libcudf/rolling.pxd index d7844f99a73..9e76faa0eba 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/rolling.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/rolling.pxd @@ -1,13 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.types import cudf_to_np_types, np_to_cudf_types - -from cudf._lib.pylibcudf.libcudf.aggregation cimport rolling_aggregation -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.aggregation cimport rolling_aggregation +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/rolling.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/round.pxd b/python/pylibcudf/pylibcudf/libcudf/round.pxd similarity index 75% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/round.pxd rename to python/pylibcudf/pylibcudf/libcudf/round.pxd index 027c4634c9f..1b65133f275 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/round.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/round.pxd @@ -2,9 +2,8 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/round.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/round.pyx b/python/pylibcudf/pylibcudf/libcudf/round.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/round.pyx rename to python/pylibcudf/pylibcudf/libcudf/round.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/scalar/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/scalar/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/__init__.py b/python/pylibcudf/pylibcudf/libcudf/scalar/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/scalar/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/scalar.pxd b/python/pylibcudf/pylibcudf/libcudf/scalar/scalar.pxd similarity index 91% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/scalar.pxd rename to python/pylibcudf/pylibcudf/libcudf/scalar/scalar.pxd index 662eb90096e..4b40a8a26f6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/scalar.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/scalar/scalar.pxd @@ -3,11 +3,10 @@ from libc.stdint cimport int32_t, int64_t from libcpp cimport bool from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type -from cudf._lib.pylibcudf.libcudf.wrappers.decimals cimport scale_type +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.wrappers.decimals cimport scale_type cdef extern from "cudf/scalar/scalar.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/scalar_factories.pxd b/python/pylibcudf/pylibcudf/libcudf/scalar/scalar_factories.pxd similarity index 76% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/scalar_factories.pxd rename to python/pylibcudf/pylibcudf/libcudf/scalar/scalar_factories.pxd index 8092c3d637d..ee4b47935b2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/scalar/scalar_factories.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/scalar/scalar_factories.pxd @@ -2,9 +2,8 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar cdef extern from "cudf/scalar/scalar_factories.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/search.pxd b/python/pylibcudf/pylibcudf/libcudf/search.pxd similarity index 73% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/search.pxd rename to python/pylibcudf/pylibcudf/libcudf/search.pxd index e2247a1366f..5a6ad5384c9 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/search.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/search.pxd @@ -1,12 +1,11 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.types as libcudf_types from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/search.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/sorting.pxd b/python/pylibcudf/pylibcudf/libcudf/sorting.pxd similarity index 84% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/sorting.pxd rename to python/pylibcudf/pylibcudf/libcudf/sorting.pxd index 3d7d3aa2790..9e899855486 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/sorting.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/sorting.pxd @@ -1,17 +1,14 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +cimport pylibcudf.libcudf.types as libcudf_types from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from cudf._lib.types import cudf_to_np_types, np_to_cudf_types - -cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types -from cudf._lib.pylibcudf.libcudf.aggregation cimport rank_method -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.aggregation cimport rank_method +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/sorting.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/stream_compaction.pxd b/python/pylibcudf/pylibcudf/libcudf/stream_compaction.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/stream_compaction.pxd rename to python/pylibcudf/pylibcudf/libcudf/stream_compaction.pxd index 11d803e5b76..7830c9478c2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/stream_compaction.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/stream_compaction.pxd @@ -3,14 +3,11 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from cudf._lib.types import cudf_to_np_types, np_to_cudf_types - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport ( nan_equality, nan_policy, null_equality, diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/stream_compaction.pyx b/python/pylibcudf/pylibcudf/libcudf/stream_compaction.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/stream_compaction.pyx rename to python/pylibcudf/pylibcudf/libcudf/stream_compaction.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/CMakeLists.txt b/python/pylibcudf/pylibcudf/libcudf/strings/CMakeLists.txt similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/CMakeLists.txt rename to python/pylibcudf/pylibcudf/libcudf/strings/CMakeLists.txt diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/__init__.py b/python/pylibcudf/pylibcudf/libcudf/strings/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/strings/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/attributes.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/attributes.pxd similarity index 76% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/attributes.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/attributes.pxd index c4d52c83663..5e510339834 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/attributes.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/attributes.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/strings/attributes.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/capitalize.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/capitalize.pxd similarity index 63% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/capitalize.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/capitalize.pxd index b0771e16680..77e3f46d7ee 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/capitalize.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/capitalize.pxd @@ -1,12 +1,9 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.char_types cimport ( - string_character_types, -) +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.char_types cimport string_character_types cdef extern from "cudf/strings/capitalize.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/case.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/case.pxd similarity index 81% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/case.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/case.pxd index 82c146b0023..7869e90f387 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/case.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/case.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/strings/case.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/char_types.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/char_types.pxd similarity index 82% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/char_types.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/char_types.pxd index f63e1a93f91..5d54c1c3593 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/char_types.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/char_types.pxd @@ -2,10 +2,9 @@ from libc.stdint cimport uint32_t from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar cdef extern from "cudf/strings/char_types/char_types.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/char_types.pyx b/python/pylibcudf/pylibcudf/libcudf/strings/char_types.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/char_types.pyx rename to python/pylibcudf/pylibcudf/libcudf/strings/char_types.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/combine.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/combine.pxd similarity index 83% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/combine.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/combine.pxd index b05e46af0d6..e4c9fa5817a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/combine.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/combine.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/strings/combine.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/contains.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/contains.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/contains.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/contains.pxd index f8ed253ff3c..c2fb5f0dce4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/contains.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/contains.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.regex_program cimport regex_program cdef extern from "cudf/strings/contains.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/__init__.py b/python/pylibcudf/pylibcudf/libcudf/strings/convert/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_booleans.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_booleans.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_booleans.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_booleans.pxd index daac2b5be28..83a9573baad 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_booleans.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_booleans.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar cdef extern from "cudf/strings/convert/convert_booleans.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_datetime.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_datetime.pxd similarity index 76% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_datetime.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_datetime.pxd index 263cee4fe1e..fa8975c4df9 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_datetime.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_datetime.pxd @@ -2,10 +2,9 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/strings/convert/convert_datetime.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_durations.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_durations.pxd similarity index 72% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_durations.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_durations.pxd index af357b9bde4..ebe10574353 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_durations.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_durations.pxd @@ -2,10 +2,9 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/strings/convert/convert_durations.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_fixed_point.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_fixed_point.pxd similarity index 73% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_fixed_point.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_fixed_point.pxd index 91c1abdb5e4..6f820f3c9a4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_fixed_point.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_fixed_point.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/strings/convert/convert_fixed_point.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_floats.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_floats.pxd similarity index 71% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_floats.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_floats.pxd index 5fbf2be0244..f4fc4674506 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_floats.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_floats.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/strings/convert/convert_floats.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_integers.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_integers.pxd similarity index 80% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_integers.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_integers.pxd index 3d6c59cbfcf..f12aab0a2e4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_integers.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_integers.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/strings/convert/convert_integers.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_ipv4.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_ipv4.pxd similarity index 76% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_ipv4.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_ipv4.pxd index 86de956b6b6..fe571cfced6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_ipv4.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_ipv4.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/strings/convert/convert_ipv4.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_lists.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_lists.pxd similarity index 62% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_lists.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_lists.pxd index aba2dbcca64..109111568d8 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_lists.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_lists.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar cdef extern from "cudf/strings/convert/convert_lists.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_urls.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_urls.pxd similarity index 72% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_urls.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_urls.pxd index fb7e0cae6de..5c07b698454 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/convert/convert_urls.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_urls.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/strings/convert/convert_urls.hpp" namespace \ diff --git a/python/pylibcudf/pylibcudf/libcudf/strings/extract.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/extract.pxd new file mode 100644 index 00000000000..12cd628fc1f --- /dev/null +++ b/python/pylibcudf/pylibcudf/libcudf/strings/extract.pxd @@ -0,0 +1,14 @@ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.table.table cimport table + + +cdef extern from "cudf/strings/extract.hpp" namespace "cudf::strings" nogil: + + cdef unique_ptr[table] extract( + column_view source_strings, + regex_program) except + diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/find.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/find.pxd similarity index 83% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/find.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/find.pxd index 04e2ed554ee..1d1df1b8b8e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/find.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/find.pxd @@ -2,11 +2,10 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/find.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/find_multiple.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/find_multiple.pxd similarity index 68% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/find_multiple.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/find_multiple.pxd index 1f1adc8e99f..0491644a10a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/find_multiple.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/find_multiple.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view cdef extern from "cudf/strings/find_multiple.hpp" namespace "cudf::strings" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/findall.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/findall.pxd similarity index 56% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/findall.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/findall.pxd index 4bc450b8911..b25724586e1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/findall.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/findall.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2019-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.strings.regex_program cimport regex_program cdef extern from "cudf/strings/findall.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/json.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/json.pxd similarity index 79% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/json.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/json.pxd index 5926fa1d29f..571ba7be7af 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/json.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/json.pxd @@ -3,10 +3,9 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar, string_scalar +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport scalar, string_scalar cdef extern from "cudf/json/json.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/padding.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/padding.pxd similarity index 59% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/padding.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/padding.pxd index 26681a1aa00..657fe61eb14 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/padding.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/padding.pxd @@ -2,12 +2,11 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.side_type cimport side_type -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.side_type cimport side_type +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/padding.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/regex_flags.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/regex_flags.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/regex_flags.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/regex_flags.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/regex_flags.pyx b/python/pylibcudf/pylibcudf/libcudf/strings/regex_flags.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/regex_flags.pyx rename to python/pylibcudf/pylibcudf/libcudf/strings/regex_flags.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/regex_program.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/regex_program.pxd similarity index 84% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/regex_program.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/regex_program.pxd index e92c8bd7737..5d1d9e583d5 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/regex_program.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/regex_program.pxd @@ -2,8 +2,7 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags cdef extern from "cudf/strings/regex/regex_program.hpp" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/repeat.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/repeat.pxd similarity index 67% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/repeat.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/repeat.pxd index 9e128529406..410ff58f299 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/repeat.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/repeat.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/repeat_strings.hpp" namespace "cudf::strings" \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/replace.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/replace.pxd similarity index 73% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/replace.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/replace.pxd index 34e03eec638..fd5f4fc4751 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/replace.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/replace.pxd @@ -3,11 +3,10 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/replace.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/replace_re.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/replace_re.pxd similarity index 63% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/replace_re.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/replace_re.pxd index 739505cd51d..40f0e2fa50c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/replace_re.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/replace_re.pxd @@ -3,13 +3,12 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/replace_re.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/side_type.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/side_type.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/side_type.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/side_type.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/split/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/split/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/__init__.py b/python/pylibcudf/pylibcudf/libcudf/strings/split/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/strings/split/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/partition.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/split/partition.pxd similarity index 63% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/partition.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/split/partition.pxd index 5119124b3e3..4162e886a7d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/partition.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/split/partition.pxd @@ -2,11 +2,10 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.table.table cimport table cdef extern from "cudf/strings/split/partition.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/split.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/split/split.pxd similarity index 78% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/split.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/split/split.pxd index 4f75664e47a..3046149aebb 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/split/split.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/split/split.pxd @@ -2,13 +2,12 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/split/split.hpp" namespace \ diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/strip.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/strip.pxd similarity index 52% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/strip.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/strip.pxd index 2d6fd6a9e89..b0ca771762d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/strip.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/strip.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.strings.side_type cimport side_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.strings.side_type cimport side_type cdef extern from "cudf/strings/strip.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/substring.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/substring.pxd similarity index 66% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/substring.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/substring.pxd index 02123cc0807..576dae9387f 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/substring.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/substring.pxd @@ -1,11 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport numeric_scalar -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport numeric_scalar +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/slice.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/translate.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/translate.pxd similarity index 73% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/translate.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/translate.pxd index b23ac277216..85fa719128a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/translate.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/translate.pxd @@ -4,11 +4,10 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.types cimport char_utf8 +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport char_utf8 cdef extern from "cudf/strings/translate.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/wrap.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/wrap.pxd similarity index 58% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings/wrap.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings/wrap.pxd index 1d92d445634..c0053391328 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings/wrap.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/wrap.pxd @@ -1,10 +1,9 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/strings/wrap.hpp" namespace "cudf::strings" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings_udf.pxd b/python/pylibcudf/pylibcudf/libcudf/strings_udf.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/strings_udf.pxd rename to python/pylibcudf/pylibcudf/libcudf/strings_udf.pxd index 804ad30dfb1..0c8fe1060ac 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/strings_udf.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings_udf.pxd @@ -4,13 +4,12 @@ from libc.stdint cimport uint8_t, uint16_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport size_type from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport size_type - cdef extern from "cudf/strings/udf/udf_string.hpp" namespace \ "cudf::strings::udf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/table/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/table/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/table/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/table/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/table/__init__.py b/python/pylibcudf/pylibcudf/libcudf/table/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/table/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/table/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/table/table.pxd b/python/pylibcudf/pylibcudf/libcudf/table/table.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/table/table.pxd rename to python/pylibcudf/pylibcudf/libcudf/table/table.pxd index 737a1327d45..654c29b083a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/table/table.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/table/table.pxd @@ -2,13 +2,9 @@ from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table_view cimport ( - mutable_table_view, - table_view, -) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table_view cimport mutable_table_view, table_view +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/table/table.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/table/table_view.pxd b/python/pylibcudf/pylibcudf/libcudf/table/table_view.pxd similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/table/table_view.pxd rename to python/pylibcudf/pylibcudf/libcudf/table/table_view.pxd index 00e1a89c025..3af2f6a6c2c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/table/table_view.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/table/table_view.pxd @@ -1,12 +1,11 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column_view cimport ( +from pylibcudf.libcudf.column.column_view cimport ( column_view, mutable_column_view, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/table/table_view.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/transform.pxd b/python/pylibcudf/pylibcudf/libcudf/transform.pxd similarity index 73% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/transform.pxd rename to python/pylibcudf/pylibcudf/libcudf/transform.pxd index b0a978fe5c5..38298a7c1f1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/transform.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/transform.pxd @@ -4,20 +4,15 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair from libcpp.string cimport string +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.expressions cimport expression +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport bitmask_type, data_type, size_type from rmm._lib.device_buffer cimport device_buffer -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.expressions cimport expression -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport ( - bitmask_type, - data_type, - size_type, -) - cdef extern from "cudf/transform.hpp" namespace "cudf" nogil: cdef pair[unique_ptr[device_buffer], size_type] bools_to_mask ( diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/transpose.pxd b/python/pylibcudf/pylibcudf/libcudf/transpose.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/transpose.pxd rename to python/pylibcudf/pylibcudf/libcudf/transpose.pxd index 5dcb9c165ad..9c0e3c073b0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/transpose.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/transpose.pxd @@ -2,9 +2,8 @@ from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table_view cimport table_view cdef extern from "cudf/transpose.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/types.pxd b/python/pylibcudf/pylibcudf/libcudf/types.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/types.pxd rename to python/pylibcudf/pylibcudf/libcudf/types.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/types.pyx b/python/pylibcudf/pylibcudf/libcudf/types.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/types.pyx rename to python/pylibcudf/pylibcudf/libcudf/types.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/unary.pxd b/python/pylibcudf/pylibcudf/libcudf/unary.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/unary.pxd rename to python/pylibcudf/pylibcudf/libcudf/unary.pxd index 2a1b189af51..887f8c7fca4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/unary.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/unary.pxd @@ -3,10 +3,9 @@ from libc.stdint cimport int32_t from libcpp cimport bool from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/unary.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/unary.pyx b/python/pylibcudf/pylibcudf/libcudf/unary.pyx similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/unary.pyx rename to python/pylibcudf/pylibcudf/libcudf/unary.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/utilities/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/__init__.py b/python/pylibcudf/pylibcudf/libcudf/utilities/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/utilities/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/host_span.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/host_span.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/host_span.pxd rename to python/pylibcudf/pylibcudf/libcudf/utilities/host_span.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/traits.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/traits.pxd similarity index 93% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/traits.pxd rename to python/pylibcudf/pylibcudf/libcudf/utilities/traits.pxd index 0cc58af735b..69765e44274 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/traits.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/utilities/traits.pxd @@ -2,8 +2,7 @@ from libcpp cimport bool from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.types cimport data_type +from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/utilities/traits.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/type_dispatcher.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/type_dispatcher.pxd similarity index 73% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/type_dispatcher.pxd rename to python/pylibcudf/pylibcudf/libcudf/utilities/type_dispatcher.pxd index 890fca3a662..fbeb6e9db90 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/type_dispatcher.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/utilities/type_dispatcher.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.types cimport type_id +from pylibcudf.libcudf.types cimport type_id cdef extern from "cudf/utilities/type_dispatcher.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/__init__.pxd b/python/pylibcudf/pylibcudf/libcudf/wrappers/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/__init__.pxd rename to python/pylibcudf/pylibcudf/libcudf/wrappers/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/__init__.py b/python/pylibcudf/pylibcudf/libcudf/wrappers/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/__init__.py rename to python/pylibcudf/pylibcudf/libcudf/wrappers/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/decimals.pxd b/python/pylibcudf/pylibcudf/libcudf/wrappers/decimals.pxd similarity index 90% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/decimals.pxd rename to python/pylibcudf/pylibcudf/libcudf/wrappers/decimals.pxd index 09b0c87e4b8..558299501d6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/decimals.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/wrappers/decimals.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t, int64_t - -from cudf._lib.pylibcudf.libcudf.types cimport int128 +from pylibcudf.libcudf.types cimport int128 cdef extern from "cudf/fixed_point/fixed_point.hpp" namespace "numeric" nogil: diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/durations.pxd b/python/pylibcudf/pylibcudf/libcudf/wrappers/durations.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/durations.pxd rename to python/pylibcudf/pylibcudf/libcudf/wrappers/durations.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/timestamps.pxd b/python/pylibcudf/pylibcudf/libcudf/wrappers/timestamps.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/libcudf/wrappers/timestamps.pxd rename to python/pylibcudf/pylibcudf/libcudf/wrappers/timestamps.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/lists.pxd b/python/pylibcudf/pylibcudf/lists.pxd similarity index 94% rename from python/cudf/cudf/_lib/pylibcudf/lists.pxd rename to python/pylibcudf/pylibcudf/lists.pxd index 17619b489d2..e7d006e6e2e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/lists.pxd +++ b/python/pylibcudf/pylibcudf/lists.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp cimport bool - -from cudf._lib.pylibcudf.libcudf.types cimport null_order, size_type +from pylibcudf.libcudf.types cimport null_order, size_type from .column cimport Column from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/pylibcudf/lists.pyx b/python/pylibcudf/pylibcudf/lists.pyx similarity index 95% rename from python/cudf/cudf/_lib/pylibcudf/lists.pyx rename to python/pylibcudf/pylibcudf/lists.pyx index c944fc35800..947caddc485 100644 --- a/python/cudf/cudf/_lib/pylibcudf/lists.pyx +++ b/python/pylibcudf/pylibcudf/lists.pyx @@ -4,9 +4,8 @@ from cython.operator cimport dereference from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.lists cimport ( contains as cpp_contains, explode as cpp_explode, filling as cpp_filling, @@ -14,34 +13,34 @@ from cudf._lib.pylibcudf.libcudf.lists cimport ( reverse as cpp_reverse, set_operations as cpp_set_operations, ) -from cudf._lib.pylibcudf.libcudf.lists.combine cimport ( +from pylibcudf.libcudf.lists.combine cimport ( concatenate_list_elements as cpp_concatenate_list_elements, concatenate_null_policy, concatenate_rows as cpp_concatenate_rows, ) -from cudf._lib.pylibcudf.libcudf.lists.count_elements cimport ( +from pylibcudf.libcudf.lists.count_elements cimport ( count_elements as cpp_count_elements, ) -from cudf._lib.pylibcudf.libcudf.lists.extract cimport ( +from pylibcudf.libcudf.lists.extract cimport ( extract_list_element as cpp_extract_list_element, ) -from cudf._lib.pylibcudf.libcudf.lists.sorting cimport ( +from pylibcudf.libcudf.lists.sorting cimport ( sort_lists as cpp_sort_lists, stable_sort_lists as cpp_stable_sort_lists, ) -from cudf._lib.pylibcudf.libcudf.lists.stream_compaction cimport ( +from pylibcudf.libcudf.lists.stream_compaction cimport ( apply_boolean_mask as cpp_apply_boolean_mask, distinct as cpp_distinct, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport ( nan_equality, null_equality, null_order, order, size_type, ) -from cudf._lib.pylibcudf.lists cimport ColumnOrScalar, ColumnOrSizeType +from pylibcudf.lists cimport ColumnOrScalar, ColumnOrSizeType from .column cimport Column, ListColumnView from .scalar cimport Scalar @@ -131,8 +130,8 @@ cpdef Column contains(Column input, ColumnOrScalar search_key): the search_key is contained in the input. ``search_key`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`. For details, see :cpp:func:`contains`. @@ -192,8 +191,8 @@ cpdef Column index_of(Column input, ColumnOrScalar search_key, bool find_first_o key row within the corresponding list row in the lists column. ``search_key`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`. For details, see :cpp:func:`index_of`. diff --git a/python/cudf/cudf/_lib/pylibcudf/merge.pxd b/python/pylibcudf/pylibcudf/merge.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/merge.pxd rename to python/pylibcudf/pylibcudf/merge.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/merge.pyx b/python/pylibcudf/pylibcudf/merge.pyx similarity index 83% rename from python/cudf/cudf/_lib/pylibcudf/merge.pyx rename to python/pylibcudf/pylibcudf/merge.pyx index 5aa46c142f6..a7d43c9d158 100644 --- a/python/cudf/cudf/_lib/pylibcudf/merge.pyx +++ b/python/pylibcudf/pylibcudf/merge.pyx @@ -3,11 +3,10 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf cimport merge as cpp_merge -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport null_order, order, size_type +from pylibcudf.libcudf cimport merge as cpp_merge +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport null_order, order, size_type from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/quantiles.pxd b/python/pylibcudf/pylibcudf/quantiles.pxd similarity index 86% rename from python/cudf/cudf/_lib/pylibcudf/quantiles.pxd rename to python/pylibcudf/pylibcudf/quantiles.pxd index 70ff135ca77..fbc1dfb30a6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/quantiles.pxd +++ b/python/pylibcudf/pylibcudf/quantiles.pxd @@ -1,7 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.types cimport interpolation, sorted +from pylibcudf.libcudf.types cimport interpolation, sorted from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/quantiles.pyx b/python/pylibcudf/pylibcudf/quantiles.pyx similarity index 93% rename from python/cudf/cudf/_lib/pylibcudf/quantiles.pyx rename to python/pylibcudf/pylibcudf/quantiles.pyx index c1f0e30ccd3..b847ade774d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/quantiles.pyx +++ b/python/pylibcudf/pylibcudf/quantiles.pyx @@ -4,15 +4,14 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.quantiles cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.quantiles cimport ( quantile as cpp_quantile, quantiles as cpp_quantiles, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport null_order, order, sorted +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport null_order, order, sorted from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/reduce.pxd b/python/pylibcudf/pylibcudf/reduce.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/reduce.pxd rename to python/pylibcudf/pylibcudf/reduce.pxd index 935efd4acf2..047f08297e4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/reduce.pxd +++ b/python/pylibcudf/pylibcudf/reduce.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.reduce cimport scan_type +from pylibcudf.libcudf.reduce cimport scan_type from .aggregation cimport Aggregation from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/reduce.pyx b/python/pylibcudf/pylibcudf/reduce.pyx similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/reduce.pyx rename to python/pylibcudf/pylibcudf/reduce.pyx index c272f183007..b0212a5b9c1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/reduce.pyx +++ b/python/pylibcudf/pylibcudf/reduce.pyx @@ -3,23 +3,18 @@ from cython.operator cimport dereference from libcpp.memory cimport unique_ptr from libcpp.utility cimport move, pair - -from cudf._lib.pylibcudf.libcudf cimport reduce as cpp_reduce -from cudf._lib.pylibcudf.libcudf.aggregation cimport ( - reduce_aggregation, - scan_aggregation, -) -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.reduce cimport scan_type -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf cimport reduce as cpp_reduce +from pylibcudf.libcudf.aggregation cimport reduce_aggregation, scan_aggregation +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.reduce cimport scan_type +from pylibcudf.libcudf.scalar.scalar cimport scalar from .aggregation cimport Aggregation from .column cimport Column from .scalar cimport Scalar from .types cimport DataType -from cudf._lib.pylibcudf.libcudf.reduce import \ - scan_type as ScanType # no-cython-lint +from pylibcudf.libcudf.reduce import scan_type as ScanType # no-cython-lint cpdef Scalar reduce(Column col, Aggregation agg, DataType data_type): diff --git a/python/cudf/cudf/_lib/pylibcudf/replace.pxd b/python/pylibcudf/pylibcudf/replace.pxd similarity index 92% rename from python/cudf/cudf/_lib/pylibcudf/replace.pxd rename to python/pylibcudf/pylibcudf/replace.pxd index 40484c728db..cb9fa8bf960 100644 --- a/python/cudf/cudf/_lib/pylibcudf/replace.pxd +++ b/python/pylibcudf/pylibcudf/replace.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. from libcpp cimport bool - -from cudf._lib.pylibcudf.libcudf.replace cimport replace_policy +from pylibcudf.libcudf.replace cimport replace_policy from .column cimport Column from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/pylibcudf/replace.pyx b/python/pylibcudf/pylibcudf/replace.pyx similarity index 97% rename from python/cudf/cudf/_lib/pylibcudf/replace.pyx rename to python/pylibcudf/pylibcudf/replace.pyx index 6e08e8f64a9..115dee132fd 100644 --- a/python/cudf/cudf/_lib/pylibcudf/replace.pyx +++ b/python/pylibcudf/pylibcudf/replace.pyx @@ -6,11 +6,10 @@ from cython.operator import dereference from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf cimport replace as cpp_replace +from pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf cimport replace as cpp_replace -from cudf._lib.pylibcudf.libcudf.column.column cimport column - -from cudf._lib.pylibcudf.libcudf.replace import \ +from pylibcudf.libcudf.replace import \ replace_policy as ReplacePolicy # no-cython-lint from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/reshape.pxd b/python/pylibcudf/pylibcudf/reshape.pxd similarity index 80% rename from python/cudf/cudf/_lib/pylibcudf/reshape.pxd rename to python/pylibcudf/pylibcudf/reshape.pxd index a7cc45d7a08..c4d3d375f7a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/reshape.pxd +++ b/python/pylibcudf/pylibcudf/reshape.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type from .column cimport Column from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/pylibcudf/reshape.pyx b/python/pylibcudf/pylibcudf/reshape.pyx similarity index 86% rename from python/cudf/cudf/_lib/pylibcudf/reshape.pyx rename to python/pylibcudf/pylibcudf/reshape.pyx index b68eba48cd6..a99145be900 100644 --- a/python/cudf/cudf/_lib/pylibcudf/reshape.pyx +++ b/python/pylibcudf/pylibcudf/reshape.pyx @@ -2,14 +2,13 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.reshape cimport ( +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.reshape cimport ( interleave_columns as cpp_interleave_columns, tile as cpp_tile, ) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport size_type from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/rolling.pxd b/python/pylibcudf/pylibcudf/rolling.pxd similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/rolling.pxd rename to python/pylibcudf/pylibcudf/rolling.pxd index cdadee68d43..9fcda21a62f 100644 --- a/python/cudf/cudf/_lib/pylibcudf/rolling.pxd +++ b/python/pylibcudf/pylibcudf/rolling.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.types cimport size_type from .aggregation cimport Aggregation from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/rolling.pyx b/python/pylibcudf/pylibcudf/rolling.pyx similarity index 89% rename from python/cudf/cudf/_lib/pylibcudf/rolling.pyx rename to python/pylibcudf/pylibcudf/rolling.pyx index 7aa7828a5dd..a46540d7ffa 100644 --- a/python/cudf/cudf/_lib/pylibcudf/rolling.pyx +++ b/python/pylibcudf/pylibcudf/rolling.pyx @@ -3,11 +3,10 @@ from cython.operator cimport dereference from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf cimport rolling as cpp_rolling -from cudf._lib.pylibcudf.libcudf.aggregation cimport rolling_aggregation -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf cimport rolling as cpp_rolling +from pylibcudf.libcudf.aggregation cimport rolling_aggregation +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.types cimport size_type from .aggregation cimport Aggregation from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/round.pxd b/python/pylibcudf/pylibcudf/round.pxd similarity index 77% rename from python/cudf/cudf/_lib/pylibcudf/round.pxd rename to python/pylibcudf/pylibcudf/round.pxd index ccb64fc2847..c8501b03fad 100644 --- a/python/cudf/cudf/_lib/pylibcudf/round.pxd +++ b/python/pylibcudf/pylibcudf/round.pxd @@ -1,7 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t - -from cudf._lib.pylibcudf.libcudf.round cimport rounding_method +from pylibcudf.libcudf.round cimport rounding_method from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/round.pyx b/python/pylibcudf/pylibcudf/round.pyx similarity index 85% rename from python/cudf/cudf/_lib/pylibcudf/round.pyx rename to python/pylibcudf/pylibcudf/round.pyx index cfcc2aafbb8..dc60d53b07e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/round.pyx +++ b/python/pylibcudf/pylibcudf/round.pyx @@ -2,16 +2,12 @@ from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf.round cimport round as cpp_round, rounding_method -from cudf._lib.pylibcudf.libcudf.round cimport ( - round as cpp_round, - rounding_method, -) - -from cudf._lib.pylibcudf.libcudf.round import \ +from pylibcudf.libcudf.round import \ rounding_method as RoundingMethod # no-cython-lint -from cudf._lib.pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/scalar.pxd b/python/pylibcudf/pylibcudf/scalar.pxd similarity index 92% rename from python/cudf/cudf/_lib/pylibcudf/scalar.pxd rename to python/pylibcudf/pylibcudf/scalar.pxd index e6c9db2f1ac..8664dfa4b7e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/scalar.pxd +++ b/python/pylibcudf/pylibcudf/scalar.pxd @@ -2,11 +2,10 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr +from pylibcudf.libcudf.scalar.scalar cimport scalar from rmm._lib.memory_resource cimport DeviceMemoryResource -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar - from .column cimport Column from .types cimport DataType diff --git a/python/cudf/cudf/_lib/pylibcudf/scalar.pyx b/python/pylibcudf/pylibcudf/scalar.pyx similarity index 94% rename from python/cudf/cudf/_lib/pylibcudf/scalar.pyx rename to python/pylibcudf/pylibcudf/scalar.pyx index 67730be07d8..3e20938af0c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/scalar.pyx +++ b/python/pylibcudf/pylibcudf/scalar.pyx @@ -3,14 +3,11 @@ from cython cimport no_gc_clear from libcpp.memory cimport unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.scalar.scalar_factories cimport make_empty_scalar_like from rmm._lib.memory_resource cimport get_current_device_resource -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.scalar.scalar_factories cimport ( - make_empty_scalar_like, -) - from .column cimport Column from .types cimport DataType diff --git a/python/cudf/cudf/_lib/pylibcudf/search.pxd b/python/pylibcudf/pylibcudf/search.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/search.pxd rename to python/pylibcudf/pylibcudf/search.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/search.pyx b/python/pylibcudf/pylibcudf/search.pyx similarity index 93% rename from python/cudf/cudf/_lib/pylibcudf/search.pyx rename to python/pylibcudf/pylibcudf/search.pyx index 151a39f204f..ff2468f3f9c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/search.pyx +++ b/python/pylibcudf/pylibcudf/search.pyx @@ -3,10 +3,9 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf cimport search as cpp_search -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.types cimport null_order, order +from pylibcudf.libcudf cimport search as cpp_search +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.types cimport null_order, order from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/sorting.pxd b/python/pylibcudf/pylibcudf/sorting.pxd similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/sorting.pxd rename to python/pylibcudf/pylibcudf/sorting.pxd index a4ea541a03b..8127ab21ad1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/sorting.pxd +++ b/python/pylibcudf/pylibcudf/sorting.pxd @@ -1,14 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp cimport bool - -from cudf._lib.pylibcudf.libcudf.aggregation cimport rank_method -from cudf._lib.pylibcudf.libcudf.types cimport ( - null_order, - null_policy, - order, - size_type, -) +from pylibcudf.libcudf.aggregation cimport rank_method +from pylibcudf.libcudf.types cimport null_order, null_policy, order, size_type from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/sorting.pyx b/python/pylibcudf/pylibcudf/sorting.pyx similarity index 96% rename from python/cudf/cudf/_lib/pylibcudf/sorting.pyx rename to python/pylibcudf/pylibcudf/sorting.pyx index 8c5a8e26899..bd173eebacb 100644 --- a/python/cudf/cudf/_lib/pylibcudf/sorting.pyx +++ b/python/pylibcudf/pylibcudf/sorting.pyx @@ -3,12 +3,11 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf cimport sorting as cpp_sorting -from cudf._lib.pylibcudf.libcudf.aggregation cimport rank_method -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport null_order, null_policy, order +from pylibcudf.libcudf cimport sorting as cpp_sorting +from pylibcudf.libcudf.aggregation cimport rank_method +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport null_order, null_policy, order from .column cimport Column from .table cimport Table diff --git a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd b/python/pylibcudf/pylibcudf/stream_compaction.pxd similarity index 89% rename from python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd rename to python/pylibcudf/pylibcudf/stream_compaction.pxd index 6f89aaf90e7..a4f39792f0c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd +++ b/python/pylibcudf/pylibcudf/stream_compaction.pxd @@ -1,9 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.stream_compaction cimport ( - duplicate_keep_option, -) -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.stream_compaction cimport duplicate_keep_option +from pylibcudf.libcudf.types cimport ( nan_equality, nan_policy, null_equality, diff --git a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx b/python/pylibcudf/pylibcudf/stream_compaction.pyx similarity index 95% rename from python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx rename to python/pylibcudf/pylibcudf/stream_compaction.pyx index 43449d3690a..b574bfa9fa2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx +++ b/python/pylibcudf/pylibcudf/stream_compaction.pyx @@ -3,16 +3,11 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf cimport ( - stream_compaction as cpp_stream_compaction, -) -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.stream_compaction cimport ( - duplicate_keep_option, -) -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf cimport stream_compaction as cpp_stream_compaction +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.stream_compaction cimport duplicate_keep_option +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.types cimport ( nan_equality, nan_policy, null_equality, @@ -20,7 +15,7 @@ from cudf._lib.pylibcudf.libcudf.types cimport ( size_type, ) -from cudf._lib.pylibcudf.libcudf.stream_compaction import \ +from pylibcudf.libcudf.stream_compaction import \ duplicate_keep_option as DuplicateKeepOption # no-cython-lint, isort:skip from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/CMakeLists.txt b/python/pylibcudf/pylibcudf/strings/CMakeLists.txt similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/strings/CMakeLists.txt rename to python/pylibcudf/pylibcudf/strings/CMakeLists.txt diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/__init__.pxd b/python/pylibcudf/pylibcudf/strings/__init__.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/strings/__init__.pxd rename to python/pylibcudf/pylibcudf/strings/__init__.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/__init__.py b/python/pylibcudf/pylibcudf/strings/__init__.py similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/strings/__init__.py rename to python/pylibcudf/pylibcudf/strings/__init__.py diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pxd b/python/pylibcudf/pylibcudf/strings/capitalize.pxd similarity index 64% rename from python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pxd rename to python/pylibcudf/pylibcudf/strings/capitalize.pxd index 9acf189fc23..b45949d4eb4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pxd +++ b/python/pylibcudf/pylibcudf/strings/capitalize.pxd @@ -1,7 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.scalar cimport Scalar +from pylibcudf.column cimport Column +from pylibcudf.scalar cimport Scalar cpdef Column capitalize(Column input, Scalar delimiters=*) diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pyx b/python/pylibcudf/pylibcudf/strings/capitalize.pyx similarity index 84% rename from python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pyx rename to python/pylibcudf/pylibcudf/strings/capitalize.pyx index ccf84d25572..06b991c3cf1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/capitalize.pyx +++ b/python/pylibcudf/pylibcudf/strings/capitalize.pyx @@ -2,16 +2,15 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.scalar.scalar_factories cimport ( +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, ) -from cudf._lib.pylibcudf.libcudf.strings cimport capitalize as cpp_capitalize -from cudf._lib.pylibcudf.scalar cimport Scalar -from cudf._lib.pylibcudf.strings.char_types cimport string_character_types +from pylibcudf.libcudf.strings cimport capitalize as cpp_capitalize +from pylibcudf.scalar cimport Scalar +from pylibcudf.strings.char_types cimport string_character_types from cython.operator import dereference diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/case.pxd b/python/pylibcudf/pylibcudf/strings/case.pxd similarity index 76% rename from python/cudf/cudf/_lib/pylibcudf/strings/case.pxd rename to python/pylibcudf/pylibcudf/strings/case.pxd index 225d566fe06..d3c98d5e3dc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/case.pxd +++ b/python/pylibcudf/pylibcudf/strings/case.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.column cimport Column +from pylibcudf.column cimport Column cpdef Column to_lower(Column input) diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/case.pyx b/python/pylibcudf/pylibcudf/strings/case.pyx similarity index 79% rename from python/cudf/cudf/_lib/pylibcudf/strings/case.pyx rename to python/pylibcudf/pylibcudf/strings/case.pyx index 3a360fd6b10..9e6cd7717d3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/case.pyx +++ b/python/pylibcudf/pylibcudf/strings/case.pyx @@ -2,10 +2,9 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.strings cimport case as cpp_case +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.strings cimport case as cpp_case cpdef Column to_lower(Column input): diff --git a/python/pylibcudf/pylibcudf/strings/char_types.pxd b/python/pylibcudf/pylibcudf/strings/char_types.pxd new file mode 100644 index 00000000000..ad4e4cf61d8 --- /dev/null +++ b/python/pylibcudf/pylibcudf/strings/char_types.pxd @@ -0,0 +1,3 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from pylibcudf.libcudf.strings.char_types cimport string_character_types diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/char_types.pyx b/python/pylibcudf/pylibcudf/strings/char_types.pyx similarity index 64% rename from python/cudf/cudf/_lib/pylibcudf/strings/char_types.pyx rename to python/pylibcudf/pylibcudf/strings/char_types.pyx index d96161951c6..e7621fb4d84 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/char_types.pyx +++ b/python/pylibcudf/pylibcudf/strings/char_types.pyx @@ -1,4 +1,4 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.strings.char_types import \ +from pylibcudf.libcudf.strings.char_types import \ string_character_types as StringCharacterTypes # no-cython-lint diff --git a/python/pylibcudf/pylibcudf/strings/contains.pxd b/python/pylibcudf/pylibcudf/strings/contains.pxd new file mode 100644 index 00000000000..2cd4891a0ea --- /dev/null +++ b/python/pylibcudf/pylibcudf/strings/contains.pxd @@ -0,0 +1,7 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from pylibcudf.column cimport Column +from pylibcudf.strings.regex_program cimport RegexProgram + + +cpdef Column contains_re(Column input, RegexProgram prog) diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/contains.pyx b/python/pylibcudf/pylibcudf/strings/contains.pyx similarity index 75% rename from python/cudf/cudf/_lib/pylibcudf/strings/contains.pyx rename to python/pylibcudf/pylibcudf/strings/contains.pyx index 8c598b7c953..1a2446f6e2c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/contains.pyx +++ b/python/pylibcudf/pylibcudf/strings/contains.pyx @@ -1,11 +1,10 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.strings cimport contains as cpp_contains -from cudf._lib.pylibcudf.strings.regex_program cimport RegexProgram +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.strings cimport contains as cpp_contains +from pylibcudf.strings.regex_program cimport RegexProgram cpdef Column contains_re( diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/find.pxd b/python/pylibcudf/pylibcudf/strings/find.pxd similarity index 77% rename from python/cudf/cudf/_lib/pylibcudf/strings/find.pxd rename to python/pylibcudf/pylibcudf/strings/find.pxd index bb43069f190..e7524a9360b 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/find.pxd +++ b/python/pylibcudf/pylibcudf/strings/find.pxd @@ -1,8 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.types cimport size_type -from cudf._lib.pylibcudf.scalar cimport Scalar +from pylibcudf.column cimport Column +from pylibcudf.libcudf.types cimport size_type +from pylibcudf.scalar cimport Scalar ctypedef fused ColumnOrScalar: Column diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/find.pyx b/python/pylibcudf/pylibcudf/strings/find.pyx similarity index 90% rename from python/cudf/cudf/_lib/pylibcudf/strings/find.pyx rename to python/pylibcudf/pylibcudf/strings/find.pyx index a0214efd0a1..22d370bf7e8 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/find.pyx +++ b/python/pylibcudf/pylibcudf/strings/find.pyx @@ -1,15 +1,14 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.strings cimport find as cpp_find -from cudf._lib.pylibcudf.scalar cimport Scalar +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.strings cimport find as cpp_find +from pylibcudf.scalar cimport Scalar from cython.operator import dereference -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.scalar.scalar cimport string_scalar cpdef Column find( @@ -22,8 +21,8 @@ cpdef Column find( first found in each string of the provided column. ``target`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`. If ``target`` is a scalar, the scalar will be searched for in each string. If ``target`` is a column, the corresponding string in the column will be @@ -126,8 +125,8 @@ cpdef Column contains( column. ``target`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`. If ``target`` is a scalar, the scalar will be searched for in each string. If ``target`` is a column, the corresponding string in the column will be @@ -180,8 +179,8 @@ cpdef Column starts_with( column. ``target`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`. If ``target`` is a scalar, the scalar will be searched for in each string. If ``target`` is a column, the corresponding string in the column will be @@ -233,8 +232,8 @@ cpdef Column ends_with( target string was found at the end of the string in the provided column. ``target`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`. If ``target`` is a scalar, the scalar will be searched for in each string. If ``target`` is a column, the corresponding string in the column will be diff --git a/python/pylibcudf/pylibcudf/strings/regex_flags.pxd b/python/pylibcudf/pylibcudf/strings/regex_flags.pxd new file mode 100644 index 00000000000..1ce3cd07df8 --- /dev/null +++ b/python/pylibcudf/pylibcudf/strings/regex_flags.pxd @@ -0,0 +1,2 @@ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/regex_flags.pyx b/python/pylibcudf/pylibcudf/strings/regex_flags.pyx similarity index 59% rename from python/cudf/cudf/_lib/pylibcudf/strings/regex_flags.pyx rename to python/pylibcudf/pylibcudf/strings/regex_flags.pyx index 903c2ddd503..ce3b6b10a42 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/regex_flags.pyx +++ b/python/pylibcudf/pylibcudf/strings/regex_flags.pyx @@ -1,4 +1,4 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.libcudf.strings.regex_flags import \ +from pylibcudf.libcudf.strings.regex_flags import \ regex_flags as RegexFlags # no-cython-lint diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pxd b/python/pylibcudf/pylibcudf/strings/regex_program.pxd similarity index 70% rename from python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pxd rename to python/pylibcudf/pylibcudf/strings/regex_program.pxd index 61ed268fb2d..045cc1e1c6b 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pxd +++ b/python/pylibcudf/pylibcudf/strings/regex_program.pxd @@ -2,8 +2,7 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string - -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.libcudf.strings.regex_program cimport regex_program cdef class RegexProgram: diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pyx b/python/pylibcudf/pylibcudf/strings/regex_program.pyx similarity index 84% rename from python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pyx rename to python/pylibcudf/pylibcudf/strings/regex_program.pyx index 5f0b8868452..f426b6888ae 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/regex_program.pyx +++ b/python/pylibcudf/pylibcudf/strings/regex_program.pyx @@ -4,12 +4,12 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move +from pylibcudf.libcudf.strings.regex_flags cimport regex_flags +from pylibcudf.libcudf.strings.regex_program cimport regex_program -from cudf._lib.pylibcudf.libcudf.strings.regex_flags cimport regex_flags -from cudf._lib.pylibcudf.libcudf.strings.regex_program cimport regex_program +from pylibcudf.strings.regex_flags import RegexFlags -from cudf._lib.pylibcudf.strings.regex_flags import RegexFlags -from cudf._lib.pylibcudf.strings.regex_flags cimport regex_flags +from pylibcudf.strings.regex_flags cimport regex_flags cdef class RegexProgram: diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/replace.pxd b/python/pylibcudf/pylibcudf/strings/replace.pxd similarity index 71% rename from python/cudf/cudf/_lib/pylibcudf/strings/replace.pxd rename to python/pylibcudf/pylibcudf/strings/replace.pxd index 52e2dc3c738..26273b96c57 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/replace.pxd +++ b/python/pylibcudf/pylibcudf/strings/replace.pxd @@ -1,8 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.types cimport size_type -from cudf._lib.pylibcudf.scalar cimport Scalar +from pylibcudf.column cimport Column +from pylibcudf.libcudf.types cimport size_type +from pylibcudf.scalar cimport Scalar cpdef Column replace( diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/replace.pyx b/python/pylibcudf/pylibcudf/strings/replace.pyx similarity index 90% rename from python/cudf/cudf/_lib/pylibcudf/strings/replace.pyx rename to python/pylibcudf/pylibcudf/strings/replace.pyx index c757150a600..9d0ebf4a814 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/replace.pyx +++ b/python/pylibcudf/pylibcudf/strings/replace.pyx @@ -2,20 +2,19 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport string_scalar -from cudf._lib.pylibcudf.libcudf.scalar.scalar_factories cimport ( +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, ) -from cudf._lib.pylibcudf.libcudf.strings.replace cimport ( +from pylibcudf.libcudf.strings.replace cimport ( replace as cpp_replace, replace_multiple as cpp_replace_multiple, replace_slice as cpp_replace_slice, ) -from cudf._lib.pylibcudf.libcudf.types cimport size_type -from cudf._lib.pylibcudf.scalar cimport Scalar +from pylibcudf.libcudf.types cimport size_type +from pylibcudf.scalar cimport Scalar cpdef Column replace( diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/slice.pxd b/python/pylibcudf/pylibcudf/strings/slice.pxd similarity index 69% rename from python/cudf/cudf/_lib/pylibcudf/strings/slice.pxd rename to python/pylibcudf/pylibcudf/strings/slice.pxd index 7d8d0006ef4..01e9f2b3c88 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/slice.pxd +++ b/python/pylibcudf/pylibcudf/strings/slice.pxd @@ -1,7 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.scalar cimport Scalar +from pylibcudf.column cimport Column +from pylibcudf.scalar cimport Scalar ctypedef fused ColumnOrScalar: Column diff --git a/python/cudf/cudf/_lib/pylibcudf/strings/slice.pyx b/python/pylibcudf/pylibcudf/strings/slice.pyx similarity index 81% rename from python/cudf/cudf/_lib/pylibcudf/strings/slice.pyx rename to python/pylibcudf/pylibcudf/strings/slice.pyx index df75134fb71..70d10cab36c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/strings/slice.pyx +++ b/python/pylibcudf/pylibcudf/strings/slice.pyx @@ -2,16 +2,15 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move - -from cudf._lib.pylibcudf.column cimport Column -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport numeric_scalar -from cudf._lib.pylibcudf.libcudf.scalar.scalar_factories cimport ( +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.scalar.scalar cimport numeric_scalar +from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_fixed_width_scalar as cpp_make_fixed_width_scalar, ) -from cudf._lib.pylibcudf.libcudf.strings cimport substring as cpp_slice -from cudf._lib.pylibcudf.libcudf.types cimport size_type -from cudf._lib.pylibcudf.scalar cimport Scalar +from pylibcudf.libcudf.strings cimport substring as cpp_slice +from pylibcudf.libcudf.types cimport size_type +from pylibcudf.scalar cimport Scalar from cython.operator import dereference @@ -25,9 +24,9 @@ cpdef Column slice_strings( """Perform a slice operation on a strings column. ``start`` and ``stop`` may be a - :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. But ``step`` must be a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~pylibcudf.column.Column` or a + :py:class:`~pylibcudf.scalar.Scalar`. But ``step`` must be a + :py:class:`~pylibcudf.scalar.Scalar`. For details, see :cpp:func:`cudf::strings::slice_strings`. diff --git a/python/cudf/cudf/_lib/pylibcudf/table.pxd b/python/pylibcudf/pylibcudf/table.pxd similarity index 78% rename from python/cudf/cudf/_lib/pylibcudf/table.pxd rename to python/pylibcudf/pylibcudf/table.pxd index e476fc770e3..cf5c0aa80f2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/table.pxd +++ b/python/pylibcudf/pylibcudf/table.pxd @@ -1,9 +1,8 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr - -from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view cdef class Table: diff --git a/python/cudf/cudf/_lib/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx similarity index 93% rename from python/cudf/cudf/_lib/pylibcudf/table.pyx rename to python/pylibcudf/pylibcudf/table.pyx index d91fa0474b0..5f77b89a605 100644 --- a/python/cudf/cudf/_lib/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -4,10 +4,9 @@ from cython.operator cimport dereference from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view -from cudf._lib.pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table cimport table from .column cimport Column diff --git a/python/cudf/cudf/pylibcudf_tests/common/utils.py b/python/pylibcudf/pylibcudf/tests/common/utils.py similarity index 97% rename from python/cudf/cudf/pylibcudf_tests/common/utils.py rename to python/pylibcudf/pylibcudf/tests/common/utils.py index acb2b5be85c..babe6634318 100644 --- a/python/cudf/cudf/pylibcudf_tests/common/utils.py +++ b/python/pylibcudf/pylibcudf/tests/common/utils.py @@ -6,11 +6,11 @@ import numpy as np import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from pyarrow.parquet import write_table as pq_write_table - -from cudf._lib import pylibcudf as plc -from cudf._lib.pylibcudf.io.types import CompressionType +from pylibcudf.io.types import CompressionType def metadata_from_arrow_type( @@ -157,13 +157,13 @@ def _flatten_arrays(arr): for lh_arr, rh_arr in zip(lhs, rhs): # Check NaNs positions match # and then filter out nans - lhs_nans = pa.compute.is_nan(lh_arr) - rhs_nans = pa.compute.is_nan(rh_arr) + lhs_nans = pc.is_nan(lh_arr) + rhs_nans = pc.is_nan(rh_arr) assert lhs_nans.equals(rhs_nans) - if pa.compute.any(lhs_nans) or pa.compute.any(rhs_nans): + if pc.any(lhs_nans) or pc.any(rhs_nans): # masks must be equal at this point - mask = pa.compute.fill_null(pa.compute.invert(lhs_nans), True) + mask = pc.fill_null(pc.invert(lhs_nans), True) lh_arr = lh_arr.filter(mask) rh_arr = rh_arr.filter(mask) diff --git a/python/cudf/cudf/pylibcudf_tests/conftest.py b/python/pylibcudf/pylibcudf/tests/conftest.py similarity index 98% rename from python/cudf/cudf/pylibcudf_tests/conftest.py rename to python/pylibcudf/pylibcudf/tests/conftest.py index 945e1689229..fdce6f353ca 100644 --- a/python/cudf/cudf/pylibcudf_tests/conftest.py +++ b/python/pylibcudf/pylibcudf/tests/conftest.py @@ -8,10 +8,9 @@ import numpy as np import pyarrow as pa +import pylibcudf as plc import pytest - -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.io.types import CompressionType +from pylibcudf.io.types import CompressionType sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common")) diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_avro.py b/python/pylibcudf/pylibcudf/tests/io/test_avro.py similarity index 98% rename from python/cudf/cudf/pylibcudf_tests/io/test_avro.py rename to python/pylibcudf/pylibcudf/tests/io/test_avro.py index 061d6792ce3..0cd5064a697 100644 --- a/python/cudf/cudf/pylibcudf_tests/io/test_avro.py +++ b/python/pylibcudf/pylibcudf/tests/io/test_avro.py @@ -5,11 +5,10 @@ import fastavro import pyarrow as pa +import pylibcudf as plc import pytest from utils import assert_table_and_meta_eq -import cudf._lib.pylibcudf as plc - avro_dtype_pairs = [ ("boolean", pa.bool_()), ("int", pa.int32()), diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_csv.py b/python/pylibcudf/pylibcudf/tests/io/test_csv.py similarity index 98% rename from python/cudf/cudf/pylibcudf_tests/io/test_csv.py rename to python/pylibcudf/pylibcudf/tests/io/test_csv.py index 95326a8b681..ccd7eef54f3 100644 --- a/python/cudf/cudf/pylibcudf_tests/io/test_csv.py +++ b/python/pylibcudf/pylibcudf/tests/io/test_csv.py @@ -5,7 +5,9 @@ import pandas as pd import pyarrow as pa +import pylibcudf as plc import pytest +from pylibcudf.io.types import CompressionType from utils import ( _convert_numeric_types_to_floating, assert_table_and_meta_eq, @@ -13,9 +15,6 @@ write_source_str, ) -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.io.types import CompressionType - # Shared kwargs to pass to make_source _COMMON_CSV_SOURCE_KWARGS = { "format": "csv", diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_json.py b/python/pylibcudf/pylibcudf/tests/io/test_json.py similarity index 99% rename from python/cudf/cudf/pylibcudf_tests/io/test_json.py rename to python/pylibcudf/pylibcudf/tests/io/test_json.py index 4239f2438bb..9d976fedf00 100644 --- a/python/cudf/cudf/pylibcudf_tests/io/test_json.py +++ b/python/pylibcudf/pylibcudf/tests/io/test_json.py @@ -3,7 +3,9 @@ import pandas as pd import pyarrow as pa +import pylibcudf as plc import pytest +from pylibcudf.io.types import CompressionType from utils import ( assert_table_and_meta_eq, make_source, @@ -11,9 +13,6 @@ write_source_str, ) -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.io.types import CompressionType - # Shared kwargs to pass to make_source _COMMON_JSON_SOURCE_KWARGS = {"format": "json", "orient": "records"} diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py b/python/pylibcudf/pylibcudf/tests/io/test_parquet.py similarity index 97% rename from python/cudf/cudf/pylibcudf_tests/io/test_parquet.py rename to python/pylibcudf/pylibcudf/tests/io/test_parquet.py index dbd20cd473e..f6e843ccf66 100644 --- a/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py +++ b/python/pylibcudf/pylibcudf/tests/io/test_parquet.py @@ -1,18 +1,17 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa import pyarrow.compute as pc +import pylibcudf as plc import pytest from pyarrow.parquet import read_table -from utils import assert_table_and_meta_eq, make_source - -import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.expressions import ( +from pylibcudf.expressions import ( ASTOperator, ColumnNameReference, ColumnReference, Literal, Operation, ) +from utils import assert_table_and_meta_eq, make_source # Shared kwargs to pass to make_source _COMMON_PARQUET_SOURCE_KWARGS = {"format": "parquet"} diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py b/python/pylibcudf/pylibcudf/tests/io/test_source_sink_info.py similarity index 98% rename from python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py rename to python/pylibcudf/pylibcudf/tests/io/test_source_sink_info.py index 438c482b77a..747f58ec8cf 100644 --- a/python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py +++ b/python/pylibcudf/pylibcudf/tests/io/test_source_sink_info.py @@ -2,10 +2,9 @@ import io +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - @pytest.fixture(params=[plc.io.SourceInfo, plc.io.SinkInfo]) def io_class(request): diff --git a/python/cudf/cudf/pylibcudf_tests/pytest.ini b/python/pylibcudf/pylibcudf/tests/pytest.ini similarity index 100% rename from python/cudf/cudf/pylibcudf_tests/pytest.ini rename to python/pylibcudf/pylibcudf/tests/pytest.ini diff --git a/python/cudf/cudf/pylibcudf_tests/test_binaryops.py b/python/pylibcudf/pylibcudf/tests/test_binaryops.py similarity index 99% rename from python/cudf/cudf/pylibcudf_tests/test_binaryops.py rename to python/pylibcudf/pylibcudf/tests/test_binaryops.py index a83caf39ead..f784cb3c191 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_binaryops.py +++ b/python/pylibcudf/pylibcudf/tests/test_binaryops.py @@ -4,11 +4,10 @@ import numpy as np import pyarrow as pa +import pylibcudf as plc import pytest from utils import assert_column_eq -from cudf._lib import pylibcudf as plc - def idfn(param): ltype, rtype, outtype, plc_op, _ = param diff --git a/python/cudf/cudf/pylibcudf_tests/test_column_factories.py b/python/pylibcudf/pylibcudf/tests/test_column_factories.py similarity index 99% rename from python/cudf/cudf/pylibcudf_tests/test_column_factories.py rename to python/pylibcudf/pylibcudf/tests/test_column_factories.py index 4c05770a41f..8cedbc6d42f 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_column_factories.py +++ b/python/pylibcudf/pylibcudf/tests/test_column_factories.py @@ -1,11 +1,10 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pylibcudf as plc import pytest from utils import DEFAULT_STRUCT_TESTING_TYPE, assert_column_eq -from cudf._lib import pylibcudf as plc - EMPTY_COL_SIZE = 3 NUMERIC_TYPES = [ diff --git a/python/cudf/cudf/pylibcudf_tests/test_column_from_device.py b/python/pylibcudf/pylibcudf/tests/test_column_from_device.py similarity index 97% rename from python/cudf/cudf/pylibcudf_tests/test_column_from_device.py rename to python/pylibcudf/pylibcudf/tests/test_column_from_device.py index 78ee2cb100e..0e129fdf0ef 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_column_from_device.py +++ b/python/pylibcudf/pylibcudf/tests/test_column_from_device.py @@ -1,13 +1,12 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pylibcudf as plc import pytest from utils import assert_column_eq import rmm -from cudf._lib import pylibcudf as plc - VALID_TYPES = [ pa.int8(), pa.int16(), diff --git a/python/cudf/cudf/pylibcudf_tests/test_copying.py b/python/pylibcudf/pylibcudf/tests/test_copying.py similarity index 99% rename from python/cudf/cudf/pylibcudf_tests/test_copying.py rename to python/pylibcudf/pylibcudf/tests/test_copying.py index f27fe4e942e..628682d0a66 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_copying.py +++ b/python/pylibcudf/pylibcudf/tests/test_copying.py @@ -2,6 +2,7 @@ import pyarrow as pa import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import ( DEFAULT_STRUCT_TESTING_TYPE, @@ -15,8 +16,6 @@ metadata_from_arrow_type, ) -from cudf._lib import pylibcudf as plc - # TODO: consider moving this to conftest and "pairing" # it with pa_type, so that they don't get out of sync diff --git a/python/cudf/cudf/pylibcudf_tests/test_datetime.py b/python/pylibcudf/pylibcudf/tests/test_datetime.py similarity index 83% rename from python/cudf/cudf/pylibcudf_tests/test_datetime.py rename to python/pylibcudf/pylibcudf/tests/test_datetime.py index 75af0fa6ca1..d3aa6101e2d 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_datetime.py +++ b/python/pylibcudf/pylibcudf/tests/test_datetime.py @@ -3,11 +3,11 @@ import datetime import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture def column(has_nulls): @@ -25,6 +25,6 @@ def column(has_nulls): def test_extract_year(column): got = plc.datetime.extract_year(column) # libcudf produces an int16, arrow produces an int64 - expect = pa.compute.year(plc.interop.to_arrow(column)).cast(pa.int16()) + expect = pc.year(plc.interop.to_arrow(column)).cast(pa.int16()) assert_column_eq(expect, got) diff --git a/python/cudf/cudf/pylibcudf_tests/test_expressions.py b/python/pylibcudf/pylibcudf/tests/test_expressions.py similarity index 97% rename from python/cudf/cudf/pylibcudf_tests/test_expressions.py rename to python/pylibcudf/pylibcudf/tests/test_expressions.py index f661512caad..5894ef4624c 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_expressions.py +++ b/python/pylibcudf/pylibcudf/tests/test_expressions.py @@ -1,9 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - # We can't really evaluate these expressions, so just make sure # construction works properly diff --git a/python/cudf/cudf/pylibcudf_tests/test_interop.py b/python/pylibcudf/pylibcudf/tests/test_interop.py similarity index 98% rename from python/cudf/cudf/pylibcudf_tests/test_interop.py rename to python/pylibcudf/pylibcudf/tests/test_interop.py index 5c05f460e28..01c998f16d4 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_interop.py +++ b/python/pylibcudf/pylibcudf/tests/test_interop.py @@ -1,10 +1,9 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - def test_list_dtype_roundtrip(): list_type = pa.list_(pa.int32()) diff --git a/python/cudf/cudf/pylibcudf_tests/test_join.py b/python/pylibcudf/pylibcudf/tests/test_join.py similarity index 94% rename from python/cudf/cudf/pylibcudf_tests/test_join.py rename to python/pylibcudf/pylibcudf/tests/test_join.py index eb25ed915b1..61e02f4d28d 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_join.py +++ b/python/pylibcudf/pylibcudf/tests/test_join.py @@ -2,10 +2,9 @@ import numpy as np import pyarrow as pa +import pylibcudf as plc from utils import assert_table_eq -from cudf._lib import pylibcudf as plc - def test_cross_join(): left = pa.Table.from_arrays([[0, 1, 2], [3, 4, 5]], names=["a", "b"]) diff --git a/python/cudf/cudf/pylibcudf_tests/test_lists.py b/python/pylibcudf/pylibcudf/tests/test_lists.py similarity index 99% rename from python/cudf/cudf/pylibcudf_tests/test_lists.py rename to python/pylibcudf/pylibcudf/tests/test_lists.py index 33f95a7d364..2353a6ff8f9 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_lists.py +++ b/python/pylibcudf/pylibcudf/tests/test_lists.py @@ -2,11 +2,11 @@ import numpy as np import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -from cudf._lib import pylibcudf as plc - @pytest.fixture def test_data(): @@ -184,7 +184,7 @@ def test_extract_list_element_scalar(list_column): plc_column = plc.interop.from_arrow(pa.array(list_column)) res = plc.lists.extract_list_element(plc_column, 0) - expect = pa.compute.list_element(list_column, 0) + expect = pc.list_element(list_column, 0) assert_column_eq(expect, res) diff --git a/python/cudf/cudf/pylibcudf_tests/test_quantiles.py b/python/pylibcudf/pylibcudf/tests/test_quantiles.py similarity index 99% rename from python/cudf/cudf/pylibcudf_tests/test_quantiles.py rename to python/pylibcudf/pylibcudf/tests/test_quantiles.py index 13f3b037606..bac56691306 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_quantiles.py +++ b/python/pylibcudf/pylibcudf/tests/test_quantiles.py @@ -3,11 +3,10 @@ import numpy as np import pyarrow as pa import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq, assert_table_eq -import cudf._lib.pylibcudf as plc - # Map pylibcudf interpolation options to pyarrow options interp_mapping = { plc.types.Interpolation.LINEAR: "linear", diff --git a/python/cudf/cudf/pylibcudf_tests/test_regex_program.py b/python/pylibcudf/pylibcudf/tests/test_regex_program.py similarity index 89% rename from python/cudf/cudf/pylibcudf_tests/test_regex_program.py rename to python/pylibcudf/pylibcudf/tests/test_regex_program.py index 3a9bcec3616..777315df538 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_regex_program.py +++ b/python/pylibcudf/pylibcudf/tests/test_regex_program.py @@ -1,9 +1,8 @@ # Copyright (c) 2024, NVIDIA CORPORATION. +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - @pytest.mark.parametrize("pat", ["(", "*", "\\"]) def test_regex_program_invalid(pat): diff --git a/python/cudf/cudf/pylibcudf_tests/test_reshape.py b/python/pylibcudf/pylibcudf/tests/test_reshape.py similarity index 96% rename from python/cudf/cudf/pylibcudf_tests/test_reshape.py rename to python/pylibcudf/pylibcudf/tests/test_reshape.py index da1157e5832..01115bc363a 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_reshape.py +++ b/python/pylibcudf/pylibcudf/tests/test_reshape.py @@ -1,11 +1,10 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pylibcudf as plc import pytest from utils import assert_column_eq, assert_table_eq -from cudf._lib import pylibcudf as plc - @pytest.fixture(scope="module") def reshape_data(): diff --git a/python/cudf/cudf/pylibcudf_tests/test_round.py b/python/pylibcudf/pylibcudf/tests/test_round.py similarity index 86% rename from python/cudf/cudf/pylibcudf_tests/test_round.py rename to python/pylibcudf/pylibcudf/tests/test_round.py index 991e6ed310d..0b30316b9a0 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_round.py +++ b/python/pylibcudf/pylibcudf/tests/test_round.py @@ -1,11 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture(params=["float32", "float64"]) def column(request, has_nulls): @@ -26,8 +26,6 @@ def test_round(column, round_mode, decimals): "half_to_even": plc.round.RoundingMethod.HALF_EVEN, }[round_mode] got = plc.round.round(column, decimals, method) - expect = pa.compute.round( - plc.interop.to_arrow(column), decimals, round_mode - ) + expect = pc.round(plc.interop.to_arrow(column), decimals, round_mode) assert_column_eq(expect, got) diff --git a/python/cudf/cudf/pylibcudf_tests/test_string_capitalize.py b/python/pylibcudf/pylibcudf/tests/test_string_capitalize.py similarity index 86% rename from python/cudf/cudf/pylibcudf_tests/test_string_capitalize.py rename to python/pylibcudf/pylibcudf/tests/test_string_capitalize.py index c4e437fe5d9..176ccc55b96 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_string_capitalize.py +++ b/python/pylibcudf/pylibcudf/tests/test_string_capitalize.py @@ -1,11 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture(scope="module") def str_data(): @@ -34,7 +34,7 @@ def str_data(): def test_capitalize(str_data): pa_data, plc_data = str_data got = plc.strings.capitalize.capitalize(plc_data) - expected = pa.compute.utf8_capitalize(pa_data) + expected = pc.utf8_capitalize(pa_data) assert_column_eq(expected, got) @@ -43,12 +43,12 @@ def test_title(str_data): got = plc.strings.capitalize.title( plc_data, plc.strings.char_types.StringCharacterTypes.CASE_TYPES ) - expected = pa.compute.utf8_title(pa_data) + expected = pc.utf8_title(pa_data) assert_column_eq(expected, got) def test_is_title(str_data): pa_data, plc_data = str_data got = plc.strings.capitalize.is_title(plc_data) - expected = pa.compute.utf8_is_title(pa_data) + expected = pc.utf8_is_title(pa_data) assert_column_eq(expected, got) diff --git a/python/cudf/cudf/pylibcudf_tests/test_string_case.py b/python/pylibcudf/pylibcudf/tests/test_string_case.py similarity index 80% rename from python/cudf/cudf/pylibcudf_tests/test_string_case.py rename to python/pylibcudf/pylibcudf/tests/test_string_case.py index 1039859b2cf..233cc253b14 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_string_case.py +++ b/python/pylibcudf/pylibcudf/tests/test_string_case.py @@ -1,11 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture(scope="module") def string_col(): @@ -17,19 +17,19 @@ def string_col(): def test_to_upper(string_col): plc_col = plc.interop.from_arrow(string_col) got = plc.strings.case.to_upper(plc_col) - expected = pa.compute.utf8_upper(string_col) + expected = pc.utf8_upper(string_col) assert_column_eq(expected, got) def test_to_lower(string_col): plc_col = plc.interop.from_arrow(string_col) got = plc.strings.case.to_lower(plc_col) - expected = pa.compute.utf8_lower(string_col) + expected = pc.utf8_lower(string_col) assert_column_eq(expected, got) def test_swapcase(string_col): plc_col = plc.interop.from_arrow(string_col) got = plc.strings.case.swapcase(plc_col) - expected = pa.compute.utf8_swapcase(string_col) + expected = pc.utf8_swapcase(string_col) assert_column_eq(expected, got) diff --git a/python/cudf/cudf/pylibcudf_tests/test_string_contains.py b/python/pylibcudf/pylibcudf/tests/test_string_contains.py similarity index 92% rename from python/cudf/cudf/pylibcudf_tests/test_string_contains.py rename to python/pylibcudf/pylibcudf/tests/test_string_contains.py index fc8c6656b5d..4f88e09183f 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_string_contains.py +++ b/python/pylibcudf/pylibcudf/tests/test_string_contains.py @@ -1,11 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture(scope="module") def target_col(): @@ -44,7 +44,7 @@ def plc_target_pat(pa_target_scalar): def test_contains_re(target_col, pa_target_scalar, plc_target_pat): pa_target_col, plc_target_col = target_col got = plc.strings.contains.contains_re(plc_target_col, plc_target_pat) - expected = pa.compute.match_substring_regex( + expected = pc.match_substring_regex( pa_target_col, pa_target_scalar.as_py() ) assert_column_eq(got, expected) diff --git a/python/cudf/cudf/pylibcudf_tests/test_string_find.py b/python/pylibcudf/pylibcudf/tests/test_string_find.py similarity index 97% rename from python/cudf/cudf/pylibcudf_tests/test_string_find.py rename to python/pylibcudf/pylibcudf/tests/test_string_find.py index 95a1a3cf731..db3b13a5aae 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_string_find.py +++ b/python/pylibcudf/pylibcudf/tests/test_string_find.py @@ -1,11 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture(scope="module") def data_col(): @@ -223,7 +223,7 @@ def test_starts_with(data_col, target_scalar): pa_target_scalar, plc_target_scalar = target_scalar py_target = pa_target_scalar.as_py() got = plc.strings.find.starts_with(plc_data_col, plc_target_scalar) - expected = pa.compute.starts_with(pa_data_col, py_target) + expected = pc.starts_with(pa_data_col, py_target) assert_column_eq(expected, got) @@ -242,7 +242,7 @@ def test_ends_with(data_col, target_scalar): pa_target_scalar, plc_target_scalar = target_scalar py_target = pa_target_scalar.as_py() got = plc.strings.find.ends_with(plc_data_col, plc_target_scalar) - expected = pa.compute.ends_with(pa_data_col, py_target) + expected = pc.ends_with(pa_data_col, py_target) assert_column_eq(expected, got) diff --git a/python/cudf/cudf/pylibcudf_tests/test_string_replace.py b/python/pylibcudf/pylibcudf/tests/test_string_replace.py similarity index 95% rename from python/cudf/cudf/pylibcudf_tests/test_string_replace.py rename to python/pylibcudf/pylibcudf/tests/test_string_replace.py index f20edf6a506..5a9c2007b73 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_string_replace.py +++ b/python/pylibcudf/pylibcudf/tests/test_string_replace.py @@ -1,11 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pyarrow.compute as pc +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture(scope="module") def data_col(): @@ -64,7 +64,7 @@ def test_replace(data_col, scalar_repl_target, scalar_repl, maxrepl): plc_data_col, plc_target, plc_repl, maxrepl ) - expected = pa.compute.replace_substring( + expected = pc.replace_substring( pa_data_col, pattern=pa_target, replacement=pa_repl, @@ -90,7 +90,7 @@ def test_replace_slice(data_col, scalar_repl, startstop): # count_characters on the input, take the max and set stop to that stop = 1000 - expected = pa.compute.utf8_replace_slice(pa_data_col, start, stop, pa_repl) + expected = pc.utf8_replace_slice(pa_data_col, start, stop, pa_repl) assert_column_eq(expected, got) diff --git a/python/cudf/cudf/pylibcudf_tests/test_string_slice.py b/python/pylibcudf/pylibcudf/tests/test_string_slice.py similarity index 98% rename from python/cudf/cudf/pylibcudf_tests/test_string_slice.py rename to python/pylibcudf/pylibcudf/tests/test_string_slice.py index bd63987b30f..d9ce5591b98 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_string_slice.py +++ b/python/pylibcudf/pylibcudf/tests/test_string_slice.py @@ -1,11 +1,10 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pylibcudf as plc import pytest from utils import assert_column_eq -import cudf._lib.pylibcudf as plc - @pytest.fixture(scope="module") def pa_col(): diff --git a/python/cudf/cudf/pylibcudf_tests/test_table.py b/python/pylibcudf/pylibcudf/tests/test_table.py similarity index 93% rename from python/cudf/cudf/pylibcudf_tests/test_table.py rename to python/pylibcudf/pylibcudf/tests/test_table.py index cf1d51f6491..e822d6a97a8 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_table.py +++ b/python/pylibcudf/pylibcudf/tests/test_table.py @@ -1,10 +1,9 @@ # Copyright (c) 2024, NVIDIA CORPORATION. import pyarrow as pa +import pylibcudf as plc import pytest -import cudf._lib.pylibcudf as plc - @pytest.mark.parametrize( "arrow_tbl", diff --git a/python/cudf/cudf/pylibcudf_tests/test_traits.py b/python/pylibcudf/pylibcudf/tests/test_traits.py similarity index 98% rename from python/cudf/cudf/pylibcudf_tests/test_traits.py rename to python/pylibcudf/pylibcudf/tests/test_traits.py index 6c22cb02f21..2570e8abd51 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_traits.py +++ b/python/pylibcudf/pylibcudf/tests/test_traits.py @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib import pylibcudf as plc +import pylibcudf as plc def test_is_relationally_comparable(): diff --git a/python/cudf/cudf/pylibcudf_tests/test_transform.py b/python/pylibcudf/pylibcudf/tests/test_transform.py similarity index 95% rename from python/cudf/cudf/pylibcudf_tests/test_transform.py rename to python/pylibcudf/pylibcudf/tests/test_transform.py index 312939888dd..06fc35d8835 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_transform.py +++ b/python/pylibcudf/pylibcudf/tests/test_transform.py @@ -3,10 +3,9 @@ import math import pyarrow as pa +import pylibcudf as plc from utils import assert_column_eq -from cudf._lib import pylibcudf as plc - def test_nans_to_nulls(has_nans): if has_nans: diff --git a/python/cudf/cudf/pylibcudf_tests/test_unary.py b/python/pylibcudf/pylibcudf/tests/test_unary.py similarity index 93% rename from python/cudf/cudf/pylibcudf_tests/test_unary.py rename to python/pylibcudf/pylibcudf/tests/test_unary.py index b5e4f0cb0e8..9b8085d5c52 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_unary.py +++ b/python/pylibcudf/pylibcudf/tests/test_unary.py @@ -1,6 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from cudf._lib import pylibcudf as plc +import pylibcudf as plc def test_is_supported_cast(): diff --git a/python/cudf/cudf/_lib/pylibcudf/traits.pxd b/python/pylibcudf/pylibcudf/traits.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/traits.pxd rename to python/pylibcudf/pylibcudf/traits.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/traits.pyx b/python/pylibcudf/pylibcudf/traits.pyx similarity index 98% rename from python/cudf/cudf/_lib/pylibcudf/traits.pyx rename to python/pylibcudf/pylibcudf/traits.pyx index d2370f8d641..5a1c67e1f6c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/traits.pyx +++ b/python/pylibcudf/pylibcudf/traits.pyx @@ -1,8 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp cimport bool - -from cudf._lib.pylibcudf.libcudf.utilities cimport traits +from pylibcudf.libcudf.utilities cimport traits from .types cimport DataType diff --git a/python/cudf/cudf/_lib/pylibcudf/transform.pxd b/python/pylibcudf/pylibcudf/transform.pxd similarity index 100% rename from python/cudf/cudf/_lib/pylibcudf/transform.pxd rename to python/pylibcudf/pylibcudf/transform.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/transform.pyx b/python/pylibcudf/pylibcudf/transform.pyx similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/transform.pyx rename to python/pylibcudf/pylibcudf/transform.pyx index a734e71b820..100ccb580ce 100644 --- a/python/cudf/cudf/_lib/pylibcudf/transform.pyx +++ b/python/pylibcudf/pylibcudf/transform.pyx @@ -2,12 +2,11 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move, pair +from pylibcudf.libcudf cimport transform as cpp_transform +from pylibcudf.libcudf.types cimport size_type from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer -from cudf._lib.pylibcudf.libcudf cimport transform as cpp_transform -from cudf._lib.pylibcudf.libcudf.types cimport size_type - from .column cimport Column from .gpumemoryview cimport gpumemoryview diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pxd b/python/pylibcudf/pylibcudf/types.pxd similarity index 91% rename from python/cudf/cudf/_lib/pylibcudf/types.pxd rename to python/pylibcudf/pylibcudf/types.pxd index 1f3e1aa2fbb..aa48979d961 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pxd +++ b/python/pylibcudf/pylibcudf/types.pxd @@ -2,8 +2,7 @@ from libc.stdint cimport int32_t from libcpp cimport bool as cbool - -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.types cimport ( data_type, interpolation, mask_state, diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pyx b/python/pylibcudf/pylibcudf/types.pyx similarity index 66% rename from python/cudf/cudf/_lib/pylibcudf/types.pyx rename to python/pylibcudf/pylibcudf/types.pyx index 311f9ce4046..58c7d97e9bc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pyx +++ b/python/pylibcudf/pylibcudf/types.pyx @@ -1,25 +1,24 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t - -from cudf._lib.pylibcudf.libcudf.types cimport ( +from pylibcudf.libcudf.types cimport ( data_type, size_of as cpp_size_of, size_type, type_id, ) -from cudf._lib.pylibcudf.libcudf.utilities.type_dispatcher cimport type_to_id - -from cudf._lib.pylibcudf.libcudf.types import type_id as TypeId # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import nan_policy as NanPolicy # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import null_policy as NullPolicy # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import interpolation as Interpolation # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import mask_state as MaskState # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import nan_equality as NanEquality # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import null_equality as NullEquality # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import null_order as NullOrder # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import order as Order # no-cython-lint, isort:skip -from cudf._lib.pylibcudf.libcudf.types import sorted as Sorted # no-cython-lint, isort:skip +from pylibcudf.libcudf.utilities.type_dispatcher cimport type_to_id + +from pylibcudf.libcudf.types import type_id as TypeId # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import nan_policy as NanPolicy # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import null_policy as NullPolicy # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import interpolation as Interpolation # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import mask_state as MaskState # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import nan_equality as NanEquality # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import null_equality as NullEquality # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import null_order as NullOrder # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import order as Order # no-cython-lint, isort:skip +from pylibcudf.libcudf.types import sorted as Sorted # no-cython-lint, isort:skip cdef class DataType: diff --git a/python/cudf/cudf/_lib/pylibcudf/unary.pxd b/python/pylibcudf/pylibcudf/unary.pxd similarity index 87% rename from python/cudf/cudf/_lib/pylibcudf/unary.pxd rename to python/pylibcudf/pylibcudf/unary.pxd index d07df838172..9ee08653599 100644 --- a/python/cudf/cudf/_lib/pylibcudf/unary.pxd +++ b/python/pylibcudf/pylibcudf/unary.pxd @@ -1,8 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. from libcpp cimport bool - -from cudf._lib.pylibcudf.libcudf.unary cimport unary_operator +from pylibcudf.libcudf.unary cimport unary_operator from .column cimport Column from .types cimport DataType diff --git a/python/cudf/cudf/_lib/pylibcudf/unary.pyx b/python/pylibcudf/pylibcudf/unary.pyx similarity index 94% rename from python/cudf/cudf/_lib/pylibcudf/unary.pyx rename to python/pylibcudf/pylibcudf/unary.pyx index 8da46f0a832..839360ef406 100644 --- a/python/cudf/cudf/_lib/pylibcudf/unary.pyx +++ b/python/pylibcudf/pylibcudf/unary.pyx @@ -3,12 +3,11 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move +from pylibcudf.libcudf cimport unary as cpp_unary +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.unary cimport unary_operator -from cudf._lib.pylibcudf.libcudf cimport unary as cpp_unary -from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.unary cimport unary_operator - -from cudf._lib.pylibcudf.libcudf.unary import \ +from pylibcudf.libcudf.unary import \ unary_operator as UnaryOperator # no-cython-lint from .column cimport Column diff --git a/python/cudf/cudf/_lib/pylibcudf/utils.pxd b/python/pylibcudf/pylibcudf/utils.pxd similarity index 71% rename from python/cudf/cudf/_lib/pylibcudf/utils.pxd rename to python/pylibcudf/pylibcudf/utils.pxd index 77c05086397..6b994f20b61 100644 --- a/python/cudf/cudf/_lib/pylibcudf/utils.pxd +++ b/python/pylibcudf/pylibcudf/utils.pxd @@ -2,9 +2,8 @@ from libcpp.functional cimport reference_wrapper from libcpp.vector cimport vector - -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport bitmask_type +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport bitmask_type cdef void * int_to_void_ptr(Py_ssize_t ptr) nogil diff --git a/python/cudf/cudf/_lib/pylibcudf/utils.pyx b/python/pylibcudf/pylibcudf/utils.pyx similarity index 93% rename from python/cudf/cudf/_lib/pylibcudf/utils.pyx rename to python/pylibcudf/pylibcudf/utils.pyx index 42e3575ed44..ee4421ddeaf 100644 --- a/python/cudf/cudf/_lib/pylibcudf/utils.pyx +++ b/python/pylibcudf/pylibcudf/utils.pyx @@ -5,11 +5,10 @@ from cython.operator import dereference from libc.stdint cimport uintptr_t from libcpp.functional cimport reference_wrapper from libcpp.vector cimport vector - from cuda import cudart -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar -from cudf._lib.pylibcudf.libcudf.types cimport bitmask_type +from pylibcudf.libcudf.scalar.scalar cimport scalar +from pylibcudf.libcudf.types cimport bitmask_type from .scalar cimport Scalar diff --git a/python/cudf/cudf/_lib/variant.pxd b/python/pylibcudf/pylibcudf/variant.pxd similarity index 100% rename from python/cudf/cudf/_lib/variant.pxd rename to python/pylibcudf/pylibcudf/variant.pxd diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml new file mode 100644 index 00000000000..b037508d03f --- /dev/null +++ b/python/pylibcudf/pyproject.toml @@ -0,0 +1,123 @@ +# Copyright (c) 2021-2024, NVIDIA CORPORATION. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "pylibcudf" +dynamic = ["version"] +description = "pylibcudf - Python bindings for libcudf" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +dependencies = [ + "cuda-python>=11.7.1,<12.0a0", + "nvtx>=0.2.1", + "packaging", + "pyarrow>=16.1.0,<16.2.0a0", + "rmm==24.10.*,>=0.0.0a0", + "typing_extensions>=4.0.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] + +[project.optional-dependencies] +test = [ + "fastavro>=0.22.9", + "hypothesis", + "numpy", + "pandas", + "pytest-cov", + "pytest-xdist", + "pytest<8", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/cudf" +Documentation = "https://docs.rapids.ai/api/cudf/stable/" + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_dask = [ + "dask", + "distributed", + "dask_cuda", +] +known_rapids = [ + "rmm", +] +known_first_party = [ + "cudf", +] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "DASK", + "RAPIDS", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "__init__.py", +] + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "cython>=3.0.3", + "ninja", + "numpy==1.23.*", + "pyarrow==16.1.0.*", + "rmm==24.10.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.exclude = ["*tests*"] +sdist.reproducible = true +wheel.packages = ["pylibcudf"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "pylibcudf/VERSION" +regex = "(?P.*)" From 10cdd5fc5dcfc73404ae825f5d4bcf357c69ff24 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 16 Aug 2024 12:49:28 -0700 Subject: [PATCH 078/270] Reenable arrow tests (#16556) This PR reenables the tests that were disabled in #16379, converting them to use the new C data interface functions instead of the old libarrow-based ones. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/16556 --- cpp/tests/CMakeLists.txt | 4 - cpp/tests/interop/arrow_utils.hpp | 5 +- cpp/tests/interop/from_arrow_test.cpp | 145 +++++++++++++------ cpp/tests/interop/to_arrow_test.cpp | 192 ++++++++++++++++---------- cpp/tests/streams/interop_test.cpp | 78 ----------- 5 files changed, 224 insertions(+), 200 deletions(-) delete mode 100644 cpp/tests/streams/interop_test.cpp diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 8c4b0f1e367..006b36add0e 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -690,10 +690,6 @@ ConfigureTest(STREAM_DICTIONARY_TEST streams/dictionary_test.cpp STREAM_MODE tes ConfigureTest(STREAM_FILLING_TEST streams/filling_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_GROUPBY_TEST streams/groupby_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_HASHING_TEST streams/hash_test.cpp STREAM_MODE testing) -# Deprecation from 16297 and fixes in 16379 caused this test to be empty This will be reenabled once -# the deprecated APIs have been replaced in 24.10. -# -# ConfigureTest(STREAM_INTEROP_TEST streams/interop_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_JSONIO_TEST streams/io/json_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_LABELING_BINS_TEST streams/labeling_bins_test.cpp STREAM_MODE testing) ConfigureTest(STREAM_LISTS_TEST streams/lists_test.cpp STREAM_MODE testing) diff --git a/cpp/tests/interop/arrow_utils.hpp b/cpp/tests/interop/arrow_utils.hpp index 1fdf02e02f1..08eada632a5 100644 --- a/cpp/tests/interop/arrow_utils.hpp +++ b/cpp/tests/interop/arrow_utils.hpp @@ -32,6 +32,8 @@ #include +#include + #pragma once template @@ -154,8 +156,9 @@ std::shared_ptr get_arrow_list_array(std::vector data, "Failed to append values to buffer builder"); CUDF_EXPECTS(buff_builder.Finish(&offset_buffer).ok(), "Failed to allocate buffer"); + auto nullable = std::accumulate(list_validity.begin(), list_validity.end(), 0) > 0; return std::make_shared( - arrow::list(data_array->type()), + arrow::list(arrow::field("", data_array->type(), nullable)), offsets.size() - 1, offset_buffer, data_array, diff --git a/cpp/tests/interop/from_arrow_test.cpp b/cpp/tests/interop/from_arrow_test.cpp index 733e5814425..81c406c0faf 100644 --- a/cpp/tests/interop/from_arrow_test.cpp +++ b/cpp/tests/interop/from_arrow_test.cpp @@ -14,13 +14,6 @@ * limitations under the License. */ -// These interop functions are deprecated. We keep the code in this -// test and will migrate the tests to export the arrow C data -// interface which we consume with from_arrow_host. For now, the tests -// are commented out. - -#if 0 - #include #include @@ -43,6 +36,10 @@ #include +#include +#include +#include + std::unique_ptr get_cudf_table() { std::vector> columns; @@ -93,6 +90,45 @@ struct FromArrowTest : public cudf::test::BaseFixture {}; template struct FromArrowTestDurationsTest : public cudf::test::BaseFixture {}; +std::optional> export_table(std::shared_ptr arrow_table) +{ + ArrowSchema schema; + if (!arrow::ExportSchema(*arrow_table->schema(), &schema).ok()) { return std::nullopt; } + auto batch = arrow_table->CombineChunksToBatch().ValueOrDie(); + ArrowArray arr; + if (!arrow::ExportRecordBatch(*batch, &arr).ok()) { return std::nullopt; } + auto ret = cudf::from_arrow(&schema, &arr); + arr.release(&arr); + schema.release(&schema); + return {std::move(ret)}; +} + +std::optional> export_scalar(arrow::Scalar const& arrow_scalar) +{ + auto maybe_array = arrow::MakeArrayFromScalar(arrow_scalar, 1); + if (!maybe_array.ok()) { return std::nullopt; } + auto array = *maybe_array; + + ArrowSchema schema; + if (!arrow::ExportType(*array->type(), &schema).ok()) { return std::nullopt; } + + ArrowArray arr; + if (!arrow::ExportArray(*array, &arr).ok()) { return std::nullopt; } + + auto col = cudf::from_arrow_column(&schema, &arr); + auto ret = cudf::get_element(col->view(), 0); + + arr.release(&arr); + schema.release(&schema); + return {std::move(ret)}; +} + +std::optional> export_scalar( + std::shared_ptr const arrow_scalar) +{ + return export_scalar(*arrow_scalar); +} + TYPED_TEST_SUITE(FromArrowTestDurationsTest, cudf::test::DurationTypes); TEST_F(FromArrowTest, EmptyTable) @@ -102,9 +138,10 @@ TEST_F(FromArrowTest, EmptyTable) auto expected_cudf_table = tables.first->view(); auto arrow_table = tables.second; - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table, got_cudf_table.value()->view()); } TEST_F(FromArrowTest, DateTimeTable) @@ -127,9 +164,10 @@ TEST_F(FromArrowTest, DateTimeTable) auto arrow_table = arrow::Table::Make(schema, {arr}); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table.value()->view()); } TYPED_TEST(FromArrowTestDurationsTest, DurationTable) @@ -160,9 +198,10 @@ TYPED_TEST(FromArrowTestDurationsTest, DurationTable) auto arrow_table = arrow::Table::Make(schema, {arr}); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table.value()->view()); } TEST_F(FromArrowTest, NestedList) @@ -188,8 +227,9 @@ TEST_F(FromArrowTest, NestedList) auto arrow_table = arrow::Table::Make(schema, {nested_list_arr}); - auto got_cudf_table = cudf::from_arrow(*arrow_table); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table->view()); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table.value()->view()); } TEST_F(FromArrowTest, StructColumn) @@ -274,9 +314,10 @@ TEST_F(FromArrowTest, StructColumn) auto schema = std::make_shared(schema_vector); auto input = arrow::Table::Make(schema, {struct_array}); - auto got_cudf_table = cudf::from_arrow(*input); + auto got_cudf_table = export_table(input); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table, got_cudf_table.value()->view()); } TEST_F(FromArrowTest, DictionaryIndicesType) @@ -304,9 +345,10 @@ TEST_F(FromArrowTest, DictionaryIndicesType) cudf::table expected_table(std::move(columns)); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table.view(), got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table.view(), got_cudf_table.value()->view()); } TEST_F(FromArrowTest, ChunkedArray) @@ -369,9 +411,10 @@ TEST_F(FromArrowTest, ChunkedArray) auto expected_cudf_table = get_cudf_table(); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table->view(), got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table->view(), got_cudf_table.value()->view()); } struct FromArrowTestSlice @@ -388,13 +431,14 @@ TEST_P(FromArrowTestSlice, SliceTest) auto sliced_cudf_table = cudf::slice(cudf_table_view, {start, end})[0]; auto expected_cudf_table = cudf::table{sliced_cudf_table}; auto sliced_arrow_table = arrow_table->Slice(start, end - start); - auto got_cudf_table = cudf::from_arrow(*sliced_arrow_table); + auto got_cudf_table = export_table(sliced_arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); // This has been added to take-care of empty string column issue with no children - if (got_cudf_table->num_rows() == 0 and expected_cudf_table.num_rows() == 0) { - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected_cudf_table.view(), got_cudf_table->view()); + if (got_cudf_table.value()->num_rows() == 0 and expected_cudf_table.num_rows() == 0) { + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected_cudf_table.view(), got_cudf_table.value()->view()); } else { - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table.view(), got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table.view(), got_cudf_table.value()->view()); } } @@ -417,9 +461,10 @@ TEST_F(FromArrowTest, FixedPoint128Table) auto const schema = std::make_shared(schema_vector); auto const arrow_table = arrow::Table::Make(schema, {arr}); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); } } @@ -441,9 +486,10 @@ TEST_F(FromArrowTest, FixedPoint128TableLarge) auto const schema = std::make_shared(schema_vector); auto const arrow_table = arrow::Table::Make(schema, {arr}); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); } } @@ -466,9 +512,10 @@ TEST_F(FromArrowTest, FixedPoint128TableNulls) auto const schema = std::make_shared(schema_vector); auto const arrow_table = arrow::Table::Make(schema, {arr}); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); } } @@ -493,9 +540,10 @@ TEST_F(FromArrowTest, FixedPoint128TableNullsLarge) auto const schema = std::make_shared(schema_vector); auto const arrow_table = arrow::Table::Make(schema, {arr}); - auto got_cudf_table = cudf::from_arrow(*arrow_table); + auto got_cudf_table = export_table(arrow_table); + ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); } } @@ -519,9 +567,12 @@ TYPED_TEST(FromArrowNumericScalarTest, Basic) { TypeParam const value{42}; auto const arrow_scalar = arrow::MakeScalar(value); - auto const cudf_scalar = cudf::from_arrow(*arrow_scalar); + + auto const cudf_scalar = export_scalar(arrow_scalar); + ASSERT_TRUE(cudf_scalar.has_value()); + auto const cudf_numeric_scalar = - dynamic_cast*>(cudf_scalar.get()); + dynamic_cast*>(cudf_scalar.value().get()); if (cudf_numeric_scalar == nullptr) { CUDF_FAIL("Attempted to test with a non-numeric type."); } EXPECT_EQ(cudf_numeric_scalar->type(), cudf::data_type(cudf::type_to_id())); EXPECT_EQ(cudf_numeric_scalar->value(), value); @@ -535,12 +586,13 @@ TEST_F(FromArrowDecimalScalarTest, Basic) auto const value{42}; auto const precision{8}; auto const scale{4}; - auto arrow_scalar = arrow::Decimal128Scalar(value, arrow::decimal128(precision, -scale)); - auto cudf_scalar = cudf::from_arrow(arrow_scalar); + auto arrow_scalar = arrow::Decimal128Scalar(value, arrow::decimal128(precision, -scale)); + auto const cudf_scalar = export_scalar(arrow_scalar); + ASSERT_TRUE(cudf_scalar.has_value()); // Arrow offers a minimum of 128 bits for the Decimal type. auto const cudf_decimal_scalar = - dynamic_cast*>(cudf_scalar.get()); + dynamic_cast*>(cudf_scalar.value().get()); EXPECT_EQ(cudf_decimal_scalar->type(), cudf::data_type(cudf::type_to_id(), scale)); EXPECT_EQ(cudf_decimal_scalar->value(), value); @@ -552,9 +604,10 @@ TEST_F(FromArrowStringScalarTest, Basic) { auto const value = std::string("hello world"); auto const arrow_scalar = arrow::StringScalar(value); - auto const cudf_scalar = cudf::from_arrow(arrow_scalar); + auto const cudf_scalar = export_scalar(arrow_scalar); + ASSERT_TRUE(cudf_scalar.has_value()); - auto const cudf_string_scalar = dynamic_cast(cudf_scalar.get()); + auto const cudf_string_scalar = dynamic_cast(cudf_scalar.value().get()); EXPECT_EQ(cudf_string_scalar->type(), cudf::data_type(cudf::type_id::STRING)); EXPECT_EQ(cudf_string_scalar->to_string(), value); } @@ -572,9 +625,10 @@ TEST_F(FromArrowListScalarTest, Basic) auto const array = *maybe_array; auto const arrow_scalar = arrow::ListScalar(array); - auto const cudf_scalar = cudf::from_arrow(arrow_scalar); + auto const cudf_scalar = export_scalar(arrow_scalar); + ASSERT_TRUE(cudf_scalar.has_value()); - auto const cudf_list_scalar = dynamic_cast(cudf_scalar.get()); + auto const cudf_list_scalar = dynamic_cast(cudf_scalar.value().get()); EXPECT_EQ(cudf_list_scalar->type(), cudf::data_type(cudf::type_id::LIST)); cudf::test::fixed_width_column_wrapper const lhs( @@ -592,9 +646,10 @@ TEST_F(FromArrowStructScalarTest, Basic) auto const field = arrow::field("", underlying_arrow_scalar->type); auto const arrow_type = arrow::struct_({field}); auto const arrow_scalar = arrow::StructScalar({underlying_arrow_scalar}, arrow_type); - auto const cudf_scalar = cudf::from_arrow(arrow_scalar); + auto const cudf_scalar = export_scalar(arrow_scalar); + ASSERT_TRUE(cudf_scalar.has_value()); - auto const cudf_struct_scalar = dynamic_cast(cudf_scalar.get()); + auto const cudf_struct_scalar = dynamic_cast(cudf_scalar.value().get()); EXPECT_EQ(cudf_struct_scalar->type(), cudf::data_type(cudf::type_id::STRUCT)); cudf::test::fixed_width_column_wrapper const col({value}); @@ -602,5 +657,3 @@ TEST_F(FromArrowStructScalarTest, Basic) CUDF_TEST_EXPECT_TABLES_EQUAL(lhs, cudf_struct_scalar->view()); } - -#endif diff --git a/cpp/tests/interop/to_arrow_test.cpp b/cpp/tests/interop/to_arrow_test.cpp index 328ba210a3f..90ae12cdd90 100644 --- a/cpp/tests/interop/to_arrow_test.cpp +++ b/cpp/tests/interop/to_arrow_test.cpp @@ -14,13 +14,6 @@ * limitations under the License. */ -// These interop functions are deprecated. We keep the code in this -// test and will migrate the tests to export via the arrow C data -// interface with to_arrow_host which arrow can consume. For now, the -// test is commented out. - -#if 0 - #include #include @@ -38,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +39,8 @@ #include +#include + using vector_of_columns = std::vector>; std::pair, std::shared_ptr> get_tables( @@ -130,7 +126,7 @@ std::pair, std::shared_ptr> get_table auto keys = cudf::test::to_host(view.keys()).first; auto indices = cudf::test::to_host(view.indices()).first; auto dict_array = get_arrow_dict_array(std::vector(keys.begin(), keys.end()), - std::vector(indices.begin(), indices.end()), + std::vector(indices.begin(), indices.end()), validity); auto boolarray = get_arrow_array(bool_data, bool_validity); auto list_array = get_arrow_list_array( @@ -168,6 +164,21 @@ struct ToArrowTest : public cudf::test::BaseFixture {}; template struct ToArrowTestDurationsTest : public cudf::test::BaseFixture {}; +auto is_equal(cudf::table_view const& table, + cudf::host_span metadata, + std::shared_ptr expected_arrow_table) +{ + auto got_arrow_schema = cudf::to_arrow_schema(table, metadata); + auto got_arrow_table = cudf::to_arrow_host(table); + + for (auto i = 0; i < got_arrow_schema->n_children; ++i) { + auto arr = arrow::ImportArray(got_arrow_table->array.children[i], got_arrow_schema->children[i]) + .ValueOrDie(); + if (!expected_arrow_table->column(i)->Equals(arrow::ChunkedArray(arr))) { return false; } + } + return true; +} + TYPED_TEST_SUITE(ToArrowTestDurationsTest, cudf::test::DurationTypes); TEST_F(ToArrowTest, EmptyTable) @@ -179,10 +190,9 @@ TEST_F(ToArrowTest, EmptyTable) auto struct_meta = cudf::column_metadata{"f"}; struct_meta.children_meta = {{"integral"}, {"string"}}; - auto got_arrow_table = - cudf::to_arrow(cudf_table_view, {{"a"}, {"b"}, {"c"}, {"d"}, {"e"}, struct_meta}); - - ASSERT_EQ(expected_arrow_table->Equals(*got_arrow_table, true), true); + std::vector const metadata = { + {"a"}, {"b"}, {"c"}, {"d"}, {"e"}, struct_meta}; + ASSERT_TRUE(is_equal(cudf_table_view, metadata, expected_arrow_table)); } TEST_F(ToArrowTest, DateTimeTable) @@ -203,12 +213,10 @@ TEST_F(ToArrowTest, DateTimeTable) std::vector> schema_vector({arrow::field("a", arr->type())}); auto schema = std::make_shared(schema_vector); - auto expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input_view, {{"a"}}); - - ASSERT_EQ(expected_arrow_table->Equals(*got_arrow_table, true), true); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input_view, metadata, expected_arrow_table)); } TYPED_TEST(ToArrowTestDurationsTest, DurationTable) @@ -239,9 +247,8 @@ TYPED_TEST(ToArrowTestDurationsTest, DurationTable) auto expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input_view, {{"a"}}); - - ASSERT_EQ(expected_arrow_table->Equals(*got_arrow_table, true), true); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input_view, metadata, expected_arrow_table)); } TEST_F(ToArrowTest, NestedList) @@ -255,20 +262,20 @@ TEST_F(ToArrowTest, NestedList) auto list_arr = get_arrow_list_array({6, 7, 8, 9}, {0, 1, 4}, {1, 0, 1, 1}); std::vector offset{0, 0, 2}; auto mask_buffer = arrow::internal::BytesToBits({0, 1}).ValueOrDie(); - auto nested_list_arr = std::make_shared(arrow::list(list(arrow::int64())), - offset.size() - 1, - arrow::Buffer::Wrap(offset), - list_arr, - mask_buffer); + auto nested_list_arr = std::make_shared( + arrow::list(arrow::field("a", arrow::list(arrow::int64()), false)), + offset.size() - 1, + arrow::Buffer::Wrap(offset), + list_arr, + mask_buffer); std::vector> schema_vector( {arrow::field("a", nested_list_arr->type())}); auto schema = std::make_shared(schema_vector); - auto expected_arrow_table = arrow::Table::Make(schema, {nested_list_arr}); - auto got_arrow_table = cudf::to_arrow(input_view, {{"a"}}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + auto expected_arrow_table = arrow::Table::Make(schema, {nested_list_arr}); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input_view, metadata, expected_arrow_table)); } TEST_F(ToArrowTest, StructColumn) @@ -324,7 +331,10 @@ TEST_F(ToArrowTest, StructColumn) auto list_arr = get_arrow_list_array({1, 2, 3, 4, 5, 6, 7, 8, 9}, {0, 2, 4, 5, 6, 7, 9}); std::vector offset{0, 3, 4, 6}; auto nested_list_arr = std::make_shared( - arrow::list(list(arrow::int64())), offset.size() - 1, arrow::Buffer::Wrap(offset), list_arr); + arrow::list(arrow::field("a", arrow::list(arrow::field("a", arrow::int64(), false)), false)), + offset.size() - 1, + arrow::Buffer::Wrap(offset), + list_arr); std::vector> child_arrays2({str2_array, int2_array}); auto fields2 = std::vector>{ @@ -356,9 +366,8 @@ TEST_F(ToArrowTest, StructColumn) auto expected_arrow_table = arrow::Table::Make(schema, {struct_array}); - auto got_arrow_table = cudf::to_arrow(input_view, {metadata}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + std::vector const meta = {metadata}; + ASSERT_TRUE(is_equal(input_view, meta, expected_arrow_table)); } template @@ -380,9 +389,8 @@ TEST_F(ToArrowTest, FixedPoint64Table) auto const schema = std::make_shared(schema_vector); auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); } } @@ -402,9 +410,8 @@ TEST_F(ToArrowTest, FixedPoint128Table) auto const schema = std::make_shared(schema_vector); auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); } } @@ -431,9 +438,8 @@ TEST_F(ToArrowTest, FixedPoint64TableLarge) auto const schema = std::make_shared(schema_vector); auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); } } @@ -455,9 +461,8 @@ TEST_F(ToArrowTest, FixedPoint128TableLarge) auto const schema = std::make_shared(schema_vector); auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); } } @@ -479,9 +484,8 @@ TEST_F(ToArrowTest, FixedPoint64TableNullsSimple) auto const schema = std::make_shared(schema_vector); auto const arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, arrow_table)); } } @@ -503,9 +507,8 @@ TEST_F(ToArrowTest, FixedPoint128TableNullsSimple) auto const schema = std::make_shared(schema_vector); auto const arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, arrow_table)); } } @@ -529,9 +532,8 @@ TEST_F(ToArrowTest, FixedPoint64TableNulls) auto const schema = std::make_shared(schema_vector); auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); } } @@ -554,9 +556,8 @@ TEST_F(ToArrowTest, FixedPoint128TableNulls) auto const schema = std::make_shared(schema_vector); auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - auto const got_arrow_table = cudf::to_arrow(input, {{"a"}}); - - ASSERT_TRUE(expected_arrow_table->Equals(*got_arrow_table, true)); + std::vector const metadata = {{"a"}}; + ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); } } @@ -575,10 +576,10 @@ TEST_P(ToArrowTestSlice, SliceTest) auto expected_arrow_table = arrow_table->Slice(start, end - start); auto struct_meta = cudf::column_metadata{"f"}; struct_meta.children_meta = {{"integral"}, {"string"}}; - auto got_arrow_table = - cudf::to_arrow(sliced_cudf_table, {{"a"}, {"b"}, {"c"}, {"d"}, {"e"}, struct_meta}); - ASSERT_EQ(expected_arrow_table->Equals(*got_arrow_table, true), true); + std::vector const metadata = { + {"a"}, {"b"}, {"c"}, {"d"}, {"e"}, struct_meta}; + ASSERT_TRUE(is_equal(sliced_cudf_table, metadata, expected_arrow_table)); } INSTANTIATE_TEST_CASE_P(ToArrowTest, @@ -595,13 +596,58 @@ using NumericTypesNotBool = cudf::test::Concat; TYPED_TEST_SUITE(ToArrowNumericScalarTest, NumericTypesNotBool); +auto col_to_arrow_type(cudf::column_view const& col) +{ + switch (col.type().id()) { + case cudf::type_id::BOOL8: return arrow::boolean(); + case cudf::type_id::INT8: return arrow::int8(); + case cudf::type_id::INT16: return arrow::int16(); + case cudf::type_id::INT32: return arrow::int32(); + case cudf::type_id::INT64: return arrow::int64(); + case cudf::type_id::UINT8: return arrow::uint8(); + case cudf::type_id::UINT16: return arrow::uint16(); + case cudf::type_id::UINT32: return arrow::uint32(); + case cudf::type_id::UINT64: return arrow::uint64(); + case cudf::type_id::FLOAT32: return arrow::float32(); + case cudf::type_id::FLOAT64: return arrow::float64(); + case cudf::type_id::TIMESTAMP_DAYS: return arrow::date32(); + case cudf::type_id::STRING: return arrow::utf8(); + case cudf::type_id::LIST: + return arrow::list(col_to_arrow_type(col.child(cudf::lists_column_view::child_column_index))); + case cudf::type_id::DECIMAL128: return arrow::decimal(38, -col.type().scale()); + default: CUDF_FAIL("Unsupported type_id conversion to arrow type", cudf::data_type_error); + } +} + +std::optional> cudf_scalar_to_arrow( + cudf::scalar const& scalar, std::optional metadata = std::nullopt) +{ + auto const cudf_column = cudf::make_column_from_scalar(scalar, 1); + auto const c_arrow_array = cudf::to_arrow_host(*cudf_column); + auto const arrow_array = [&]() { + if (metadata.has_value()) { + auto const table = cudf::table_view({cudf_column->view()}); + std::vector const table_metadata = {metadata.value()}; + auto const arrow_schema = cudf::to_arrow_schema(table, table_metadata); + return arrow::ImportArray(&c_arrow_array->array, arrow_schema->children[0]).ValueOrDie(); + } else { + auto const arrow_type = col_to_arrow_type(cudf_column->view()); + return arrow::ImportArray(&c_arrow_array->array, arrow_type).ValueOrDie(); + } + }(); + auto const maybe_scalar = arrow_array->GetScalar(0); + if (!maybe_scalar.ok()) { return std::nullopt; } + return maybe_scalar.ValueOrDie(); +} + TYPED_TEST(ToArrowNumericScalarTest, Basic) { TypeParam const value{42}; auto const cudf_scalar = cudf::make_fixed_width_scalar(value); - cudf::column_metadata const metadata{""}; - auto const arrow_scalar = cudf::to_arrow(*cudf_scalar, metadata); + auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); + ASSERT_TRUE(maybe_scalar.has_value()); + auto const arrow_scalar = *maybe_scalar; auto const ref_arrow_scalar = arrow::MakeScalar(value); EXPECT_TRUE(arrow_scalar->Equals(*ref_arrow_scalar)); @@ -621,8 +667,9 @@ TEST_F(ToArrowDecimalScalarTest, Basic) auto const cudf_scalar = cudf::make_fixed_point_scalar(value, numeric::scale_type{scale}); - cudf::column_metadata const metadata{""}; - auto const arrow_scalar = cudf::to_arrow(*cudf_scalar, metadata); + auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); + ASSERT_TRUE(maybe_scalar.has_value()); + auto const arrow_scalar = *maybe_scalar; auto const maybe_ref_arrow_scalar = arrow::MakeScalar(arrow::decimal128(precision, -scale), value); @@ -636,9 +683,10 @@ struct ToArrowStringScalarTest : public cudf::test::BaseFixture {}; TEST_F(ToArrowStringScalarTest, Basic) { std::string const value{"hello world"}; - auto const cudf_scalar = cudf::make_string_scalar(value); - cudf::column_metadata const metadata{""}; - auto const arrow_scalar = cudf::to_arrow(*cudf_scalar, metadata); + auto const cudf_scalar = cudf::make_string_scalar(value); + auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); + ASSERT_TRUE(maybe_scalar.has_value()); + auto const arrow_scalar = *maybe_scalar; auto const ref_arrow_scalar = arrow::MakeScalar(value); EXPECT_TRUE(arrow_scalar->Equals(*ref_arrow_scalar)); @@ -656,8 +704,9 @@ TEST_F(ToArrowListScalarTest, Basic) auto const cudf_scalar = cudf::make_list_scalar(col); - cudf::column_metadata const metadata{""}; - auto const arrow_scalar = cudf::to_arrow(*cudf_scalar, metadata); + auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); + ASSERT_TRUE(maybe_scalar.has_value()); + auto const arrow_scalar = *maybe_scalar; arrow::Int64Builder builder; auto const status = builder.AppendValues(host_values, host_validity); @@ -682,7 +731,10 @@ TEST_F(ToArrowStructScalarTest, Basic) cudf::column_metadata metadata{""}; metadata.children_meta.emplace_back(field_name); - auto const arrow_scalar = cudf::to_arrow(*cudf_scalar, metadata); + + auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar, metadata); + ASSERT_TRUE(maybe_scalar.has_value()); + auto const arrow_scalar = *maybe_scalar; auto const underlying_arrow_scalar = arrow::MakeScalar(value); auto const field = arrow::field(field_name, underlying_arrow_scalar->type, false); @@ -693,5 +745,3 @@ TEST_F(ToArrowStructScalarTest, Basic) } CUDF_TEST_PROGRAM_MAIN() - -#endif diff --git a/cpp/tests/streams/interop_test.cpp b/cpp/tests/streams/interop_test.cpp deleted file mode 100644 index 9ba862585d0..00000000000 --- a/cpp/tests/streams/interop_test.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// These interop functions are deprecated. We keep the code in this -// test and will migrate the tests to export via the arrow C data -// interface with to_arrow_host which arrow can consume. For now, the -// test is commented out. - -#if 0 - -#include -#include -#include - -#include -#include -#include -#include - -struct ArrowTest : public cudf::test::BaseFixture {}; - -TEST_F(ArrowTest, ToArrow) -{ - int32_t const value{42}; - auto col = cudf::test::fixed_width_column_wrapper{{value}}; - cudf::table_view tbl{{col}}; - - std::vector metadata{{""}}; - cudf::to_arrow(tbl, metadata, cudf::test::get_default_stream()); -} - -TEST_F(ArrowTest, FromArrow) -{ - std::vector host_values = {1, 2, 3, 5, 6, 7, 8}; - std::vector host_validity = {true, true, true, false, true, true, true}; - - arrow::Int64Builder builder; - auto status = builder.AppendValues(host_values, host_validity); - auto maybe_array = builder.Finish(); - auto array = *maybe_array; - - auto field = arrow::field("", arrow::int32()); - auto schema = arrow::schema({field}); - auto table = arrow::Table::Make(schema, {array}); - cudf::from_arrow(*table, cudf::test::get_default_stream()); -} - -TEST_F(ArrowTest, ToArrowScalar) -{ - int32_t const value{42}; - auto cudf_scalar = - cudf::make_fixed_width_scalar(value, cudf::test::get_default_stream()); - - cudf::column_metadata metadata{""}; - cudf::to_arrow(*cudf_scalar, metadata, cudf::test::get_default_stream()); -} - -TEST_F(ArrowTest, FromArrowScalar) -{ - int32_t const value{42}; - auto arrow_scalar = arrow::MakeScalar(value); - cudf::from_arrow(*arrow_scalar, cudf::test::get_default_stream()); -} - -#endif From cb843dbdc2fc0c73c8af98909304c768bb65c16f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:02:06 -1000 Subject: [PATCH 079/270] Fix DataFrame reductions with median returning scalar instead of Series (#16527) xref https://github.com/rapidsai/cudf/issues/16507 This turned into a little bit of a refactor that also fixes the following: * `cudf.DataFrame.from_pandas` not preserving the `pandas.DataFrame.column.dtype` * `cudf.DataFrame.(axis=0)` not preserving the `.column` properties in the resulting `.index` Authors: - Matthew Roeschke (https://github.com/mroeschke) - Matthew Murray (https://github.com/Matt711) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16527 --- python/cudf/cudf/core/column_accessor.py | 3 + python/cudf/cudf/core/dataframe.py | 120 ++++++++-------------- python/cudf/cudf/core/indexed_frame.py | 36 +------ python/cudf/cudf/tests/test_dataframe.py | 6 ++ python/cudf/cudf/tests/test_reductions.py | 35 +++++++ 5 files changed, 90 insertions(+), 110 deletions(-) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 48bc84070b1..67c19f11e41 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -352,6 +352,9 @@ def insert( new_values = self.columns[:loc] + (value,) + self.columns[loc:] self._data = self._data.__class__(zip(new_keys, new_values)) self._clear_cache(old_ncols, old_ncols + 1) + if old_ncols == 0: + # The type(name) may no longer match the prior label_dtype + self.label_dtype = None def copy(self, deep=False) -> ColumnAccessor: """ diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 6ee3d69441f..97684129203 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5497,14 +5497,9 @@ def from_pandas(cls, dataframe, nan_as_null=no_default): ) if isinstance(dataframe, pd.DataFrame): - if not dataframe.columns.is_unique: - raise ValueError("Duplicate column names are not allowed") - data = { - col_name: column.as_column( - col_value.array, nan_as_null=nan_as_null - ) - for col_name, col_value in dataframe.items() + i: column.as_column(col_value.array, nan_as_null=nan_as_null) + for i, (_, col_value) in enumerate(dataframe.items()) } if isinstance(dataframe.index, pd.MultiIndex): index = cudf.MultiIndex.from_pandas( @@ -5515,14 +5510,8 @@ def from_pandas(cls, dataframe, nan_as_null=no_default): dataframe.index, nan_as_null=nan_as_null ) df = cls._from_data(data, index) - df._data._level_names = tuple(dataframe.columns.names) - - if isinstance(dataframe.columns, pd.RangeIndex): - df._data.rangeindex = True - # Set columns only if it is a MultiIndex - elif isinstance(dataframe.columns, pd.MultiIndex): - df.columns = dataframe.columns - + # Checks duplicate columns and sets column metadata + df.columns = dataframe.columns return df elif hasattr(dataframe, "__dataframe__"): # TODO: Probably should be handled in the constructor as @@ -6382,8 +6371,11 @@ def _reduce( source = self if axis is None: + assert PANDAS_LT_300, "Replace if/else with just axis=2" + # TODO(pandas3.0): Remove if/else for just axis = 2 if op in {"sum", "product", "std", "var"}: - # Do not remove until pandas 2.0 support is added. + # pandas only raises FutureWarning for these ops + # though it applies for all reductions warnings.warn( f"In a future version, {type(self).__name__}" f".{op}(axis=None) will return a scalar {op} over " @@ -6402,9 +6394,7 @@ def _reduce( if numeric_only: numeric_cols = ( - name - for name in self._data.names - if is_numeric_dtype(self._data[name].dtype) + name for name, dtype in self._dtypes if is_numeric_dtype(dtype) ) source = self._get_columns_by_label(numeric_cols) if source.empty: @@ -6414,62 +6404,41 @@ def _reduce( else source.index, dtype="float64", ) - if axis in {0, 2}: - if axis == 2 and op in ("kurtosis", "kurt", "skew"): - # TODO: concat + op can probably be done in the general case - # for axis == 2. - # https://github.com/rapidsai/cudf/issues/14930 - return getattr(concat_columns(source._data.columns), op)( - **kwargs - ) - try: - result = [ - getattr(source._data[col], op)(**kwargs) - for col in source._data.names - ] - except AttributeError: - numeric_ops = ( - "mean", - "min", - "max", - "sum", - "product", - "prod", - "std", - "var", - "kurtosis", - "kurt", - "skew", - ) - - if op in numeric_ops: + if ( + axis == 2 + and op in {"kurtosis", "skew"} + and self._num_rows < 4 + and self._num_columns > 1 + ): + # Total number of elements may satisfy the min number of values + # to compute skew/kurtosis + return getattr(concat_columns(source._columns), op)(**kwargs) + elif axis == 1: + return source._apply_cupy_method_axis_1(op, **kwargs) + else: + axis_0_results = [] + for col_label, col in source._data.items(): + try: + axis_0_results.append(getattr(col, op)(**kwargs)) + except AttributeError as err: if numeric_only: - try: - result = [ - getattr(source._data[col], op)(**kwargs) - for col in source._data.names - ] - except AttributeError: - raise NotImplementedError( - f"Not all column dtypes support op {op}" - ) - elif any( - not is_numeric_dtype(self._data[name].dtype) - for name in self._data.names - ): + raise NotImplementedError( + f"Column {col_label} with type {col.dtype} does not support {op}" + ) from err + elif not is_numeric_dtype(col.dtype): raise TypeError( "Non numeric columns passed with " "`numeric_only=False`, pass `numeric_only=True` " f"to perform DataFrame.{op}" - ) - else: - raise + ) from err + else: + raise if axis == 2: - return getattr(as_column(result, nan_as_null=False), op)( - **kwargs - ) + return getattr( + as_column(axis_0_results, nan_as_null=False), op + )(**kwargs) else: - source_dtypes = [c.dtype for c in source._data.columns] + source_dtypes = [dtype for _, dtype in source._dtypes] common_dtype = find_common_type(source_dtypes) if ( is_object_dtype(common_dtype) @@ -6483,17 +6452,14 @@ def _reduce( "Columns must all have the same dtype to " f"perform {op=} with {axis=}" ) + pd_index = source._data.to_pandas_index() if source._data.multiindex: - idx = MultiIndex.from_tuples( - source._data.names, names=source._data.level_names - ) + idx = MultiIndex.from_pandas(pd_index) else: - idx = cudf.Index(source._data.names) - return Series._from_column(as_column(result), index=idx) - elif axis == 1: - return source._apply_cupy_method_axis_1(op, **kwargs) - else: - raise ValueError(f"Invalid value of {axis=} received for {op}") + idx = cudf.Index.from_pandas(pd_index) + return Series._from_column( + as_column(axis_0_results), index=idx + ) @_performance_tracking def _scan( diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 2263dfd5c98..e46e24dd0d8 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1386,11 +1386,6 @@ def sum( a 10 b 34 dtype: int64 - - .. pandas-compat:: - :meth:`pandas.DataFrame.sum`, :meth:`pandas.Series.sum` - - Parameters currently not supported are `level`, `numeric_only`. """ return self._reduce( "sum", @@ -1447,11 +1442,6 @@ def product( a 24 b 5040 dtype: int64 - - .. pandas-compat:: - :meth:`pandas.DataFrame.product`, :meth:`pandas.Series.product` - - Parameters currently not supported are level`, `numeric_only`. """ return self._reduce( @@ -1508,7 +1498,9 @@ def mean(self, axis=0, skipna=True, numeric_only=False, **kwargs): **kwargs, ) - def median(self, axis=None, skipna=True, numeric_only=None, **kwargs): + def median( + self, axis=no_default, skipna=True, numeric_only=None, **kwargs + ): """ Return the median of the values for the requested axis. @@ -1542,11 +1534,6 @@ def median(self, axis=None, skipna=True, numeric_only=None, **kwargs): dtype: int64 >>> ser.median() 17.0 - - .. pandas-compat:: - :meth:`pandas.DataFrame.median`, :meth:`pandas.Series.median` - - Parameters currently not supported are `level` and `numeric_only`. """ return self._reduce( "median", @@ -1598,12 +1585,6 @@ def std( a 1.290994 b 1.290994 dtype: float64 - - .. pandas-compat:: - :meth:`pandas.DataFrame.std`, :meth:`pandas.Series.std` - - Parameters currently not supported are `level` and - `numeric_only` """ return self._reduce( @@ -1657,12 +1638,6 @@ def var( a 1.666667 b 1.666667 dtype: float64 - - .. pandas-compat:: - :meth:`pandas.DataFrame.var`, :meth:`pandas.Series.var` - - Parameters currently not supported are `level` and - `numeric_only` """ return self._reduce( "var", @@ -1713,11 +1688,6 @@ def kurtosis(self, axis=0, skipna=True, numeric_only=False, **kwargs): a -1.2 b -1.2 dtype: float64 - - .. pandas-compat:: - :meth:`pandas.DataFrame.kurtosis` - - Parameters currently not supported are `level` and `numeric_only` """ if axis not in (0, "index", None, no_default): raise NotImplementedError("Only axis=0 is currently supported.") diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 89eb5a12c71..9122a1074ac 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -11114,3 +11114,9 @@ def test_bool_raises(): lfunc_args_and_kwargs=[[cudf.DataFrame()]], rfunc_args_and_kwargs=[[pd.DataFrame()]], ) + + +def test_from_pandas_preserve_column_dtype(): + df = pd.DataFrame([[1, 2]], columns=pd.Index([1, 2], dtype="int8")) + result = cudf.DataFrame.from_pandas(df) + pd.testing.assert_index_equal(result.columns, df.columns, exact=True) diff --git a/python/cudf/cudf/tests/test_reductions.py b/python/cudf/cudf/tests/test_reductions.py index 8be6463c699..a70a2ea15dd 100644 --- a/python/cudf/cudf/tests/test_reductions.py +++ b/python/cudf/cudf/tests/test_reductions.py @@ -358,6 +358,30 @@ def test_reductions_axis_none_warning(op): assert_eq(expected, actual, check_dtype=False) +@pytest.mark.parametrize( + "op", + [ + "sum", + "product", + "std", + "var", + "kurt", + "kurtosis", + "skew", + "min", + "max", + "mean", + "median", + ], +) +def test_dataframe_reduction_no_args(op): + df = cudf.DataFrame({"a": range(10), "b": range(10)}) + pdf = df.to_pandas() + result = getattr(df, op)() + expected = getattr(pdf, op)() + assert_eq(result, expected) + + def test_reduction_column_multiindex(): idx = cudf.MultiIndex.from_tuples( [("a", 1), ("a", 2)], names=["foo", "bar"] @@ -374,3 +398,14 @@ def test_dtype_deprecated(op): with pytest.warns(FutureWarning): result = getattr(ser, op)(dtype=np.dtype(np.int8)) assert isinstance(result, np.int8) + + +@pytest.mark.parametrize( + "columns", [pd.RangeIndex(2), pd.Index([0, 1], dtype="int8")] +) +def test_dataframe_axis_0_preserve_column_type_in_index(columns): + pd_df = pd.DataFrame([[1, 2]], columns=columns) + cudf_df = cudf.DataFrame.from_pandas(pd_df) + result = cudf_df.sum(axis=0) + expected = pd_df.sum(axis=0) + assert_eq(result, expected, check_index_type=True) From fd44adc9e02dec4cdde9626f46ba231bda4a7ea6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 16 Aug 2024 13:02:49 -1000 Subject: [PATCH 080/270] Make CategoricalColumn.__init__ strict (#16456) This PR transfers some of the validation logic in `build_column` directly into `CategoricalColumn` just in case `CategoricalColumn` is called independently of `build_column`. Additionally adds stricter validation of `data`, `dtype` and `children` so the column doesn't represent an invalid state xref https://github.com/rapidsai/cudf/issues/16469 Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16456 --- python/cudf/cudf/_lib/column.pyx | 6 +-- python/cudf/cudf/core/column/categorical.py | 56 +++++++++++++-------- python/cudf/cudf/core/column/column.py | 9 +--- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index 2e400f775d3..e27c595edda 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -86,7 +86,7 @@ cdef class Column: object mask=None, int offset=0, object null_count=None, - object children=() + tuple children=() ): if size < 0: raise ValueError("size must be >=0") @@ -297,11 +297,11 @@ cdef class Column: dtypes = [ base_child.dtype for base_child in self.base_children ] - self._children = [ + self._children = tuple( child._with_type_metadata(dtype) for child, dtype in zip( children, dtypes ) - ] + ) return self._children def set_base_children(self, value): diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 66aed38bffd..1fdaf9f8c07 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -465,6 +465,18 @@ def reorder_categories( ) +def validate_categorical_children(children) -> None: + if not ( + len(children) == 1 + and isinstance(children[0], cudf.core.column.numerical.NumericalColumn) + and children[0].dtype.kind in "iu" + ): + # TODO: Enforce unsigned integer? + raise ValueError( + "Must specify exactly one child NumericalColumn of integers for representing the codes." + ) + + class CategoricalColumn(column.ColumnBase): """ Implements operations for Columns of Categorical type @@ -481,8 +493,7 @@ class CategoricalColumn(column.ColumnBase): respectively """ - dtype: cudf.core.dtypes.CategoricalDtype - _codes: NumericalColumn | None + dtype: CategoricalDtype _children: tuple[NumericalColumn] _VALID_REDUCTIONS = { "max", @@ -499,25 +510,29 @@ class CategoricalColumn(column.ColumnBase): def __init__( self, + data: None, + size: int | None, dtype: CategoricalDtype, mask: Buffer | None = None, - size: int | None = None, offset: int = 0, null_count: int | None = None, - children: tuple["column.ColumnBase", ...] = (), + children: tuple[NumericalColumn] = (), # type: ignore[assignment] ): + if data is not None: + raise ValueError(f"{data=} must be None") + validate_categorical_children(children) if size is None: - for child in children: - assert child.offset == 0 - assert child.base_mask is None - size = children[0].size + child = children[0] + assert child.offset == 0 + assert child.base_mask is None + size = child.size size = size - offset - if isinstance(dtype, pd.api.types.CategoricalDtype): - dtype = CategoricalDtype.from_pandas(dtype) if not isinstance(dtype, CategoricalDtype): - raise ValueError("dtype must be instance of CategoricalDtype") + raise ValueError( + f"{dtype=} must be cudf.CategoricalDtype instance." + ) super().__init__( - data=None, + data=data, size=size, dtype=dtype, mask=mask, @@ -525,7 +540,7 @@ def __init__( null_count=null_count, children=children, ) - self._codes = None + self._codes = self.children[0].set_mask(self.mask) @property def base_size(self) -> int: @@ -558,13 +573,14 @@ def _process_values_for_isin( rhs = cudf.core.column.as_column(values, dtype=self.dtype) return lhs, rhs - def set_base_mask(self, value: Buffer | None): + def set_base_mask(self, value: Buffer | None) -> None: super().set_base_mask(value) - self._codes = None + self._codes = self.children[0].set_mask(self.mask) - def set_base_children(self, value: tuple[ColumnBase, ...]): + def set_base_children(self, value: tuple[NumericalColumn]) -> None: # type: ignore[override] super().set_base_children(value) - self._codes = None + validate_categorical_children(value) + self._codes = value[0].set_mask(self.mask) @property def children(self) -> tuple[NumericalColumn]: @@ -586,9 +602,7 @@ def categories(self) -> ColumnBase: @property def codes(self) -> NumericalColumn: - if self._codes is None: - self._codes = self.children[0].set_mask(self.mask) - return cast(cudf.core.column.NumericalColumn, self._codes) + return self._codes @property def ordered(self) -> bool: @@ -1131,7 +1145,7 @@ def _mimic_inplace( ) -> Self | None: out = super()._mimic_inplace(other_col, inplace=inplace) if inplace and isinstance(other_col, CategoricalColumn): - self._codes = other_col._codes + self._codes = other_col.codes return out def view(self, dtype: Dtype) -> ColumnBase: diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 090c02da990..19d6bf84d3f 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1578,19 +1578,14 @@ def build_column( return col if isinstance(dtype, CategoricalDtype): - if not len(children) == 1: - raise ValueError( - "Must specify exactly one child column for CategoricalColumn" - ) - if not isinstance(children[0], ColumnBase): - raise TypeError("children must be a tuple of Columns") return cudf.core.column.CategoricalColumn( + data=data, # type: ignore[arg-type] dtype=dtype, mask=mask, size=size, offset=offset, null_count=null_count, - children=children, + children=children, # type: ignore[arg-type] ) elif dtype.type is np.datetime64: return cudf.core.column.DatetimeColumn( From b63ba70f2cf3724eeb118f9d2ec03a370c135f23 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 16 Aug 2024 18:27:07 -0700 Subject: [PATCH 081/270] Add build job for pylibcudf (#16587) This was missed in #16299 and is necessary to get builds published. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16587 --- .github/workflows/build.yaml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2fc39c06fad..9943b02a521 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,27 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-pylibcudf: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_pylibcudf.sh + wheel-publish-pylibcudf: + needs: wheel-build-pylibcudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: pylibcudf wheel-build-cudf: + needs: wheel-publish-pylibcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -109,7 +129,7 @@ jobs: date: ${{ inputs.date }} package-name: dask_cudf wheel-build-cudf-polars: - needs: wheel-publish-cudf + needs: wheel-publish-pylibcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: From dd2c12dd8a8682b562bb3b420e0982f79a99438d Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Fri, 16 Aug 2024 23:18:42 -0400 Subject: [PATCH 082/270] Revert "Make proxy NumPy arrays pass isinstance check in `cudf.pandas`" (#16586) Reverts rapidsai/cudf#16286 Authors: - Matthew Murray (https://github.com/Matt711) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16586 --- python/cudf/cudf/pandas/_wrappers/numpy.py | 3 --- python/cudf/cudf/pandas/fast_slow_proxy.py | 20 +--------------- python/cudf/cudf/pandas/proxy_base.py | 23 ------------------- .../cudf_pandas_tests/test_cudf_pandas.py | 8 ------- 4 files changed, 1 insertion(+), 53 deletions(-) delete mode 100644 python/cudf/cudf/pandas/proxy_base.py diff --git a/python/cudf/cudf/pandas/_wrappers/numpy.py b/python/cudf/cudf/pandas/_wrappers/numpy.py index eabea9713f1..3b012169676 100644 --- a/python/cudf/cudf/pandas/_wrappers/numpy.py +++ b/python/cudf/cudf/pandas/_wrappers/numpy.py @@ -14,7 +14,6 @@ make_final_proxy_type, make_intermediate_proxy_type, ) -from ..proxy_base import ProxyNDarrayBase from .common import ( array_interface, array_method, @@ -112,14 +111,12 @@ def wrap_ndarray(cls, arr: cupy.ndarray | numpy.ndarray, constructor): numpy.ndarray, fast_to_slow=cupy.ndarray.get, slow_to_fast=cupy.asarray, - bases=(ProxyNDarrayBase,), additional_attributes={ "__array__": array_method, # So that pa.array(wrapped-numpy-array) works "__arrow_array__": arrow_array_method, "__cuda_array_interface__": cuda_array_interface, "__array_interface__": array_interface, - "__array_ufunc__": _FastSlowAttribute("__array_ufunc__"), # ndarrays are unhashable "__hash__": None, # iter(cupy-array) produces an iterable of zero-dim device diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index 61aa6310082..bb678fd1efe 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -19,7 +19,6 @@ from ..options import _env_get_bool from ..testing import assert_eq from .annotation import nvtx -from .proxy_base import ProxyNDarrayBase def call_operator(fn, args, kwargs): @@ -565,11 +564,7 @@ def _fsproxy_wrap(cls, value, func): _FinalProxy subclasses can override this classmethod if they need particular behaviour when wrapped up. """ - base_class = _get_proxy_base_class(cls) - if base_class is object: - proxy = base_class.__new__(cls) - else: - proxy = base_class.__new__(cls, value) + proxy = object.__new__(cls) proxy._fsproxy_wrapped = value return proxy @@ -1198,19 +1193,6 @@ def is_proxy_object(obj: Any) -> bool: return False -def _get_proxy_base_class(cls): - """Returns the proxy base class if one exists""" - for proxy_class in PROXY_BASE_CLASSES: - if proxy_class in cls.__mro__: - return proxy_class - return object - - -PROXY_BASE_CLASSES: set[type] = { - ProxyNDarrayBase, -} - - NUMPY_TYPES: set[str] = set(np.sctypeDict.values()) diff --git a/python/cudf/cudf/pandas/proxy_base.py b/python/cudf/cudf/pandas/proxy_base.py deleted file mode 100644 index 61d9cde127c..00000000000 --- a/python/cudf/cudf/pandas/proxy_base.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import cupy as cp -import numpy as np - - -class ProxyNDarrayBase(np.ndarray): - def __new__(cls, arr): - if isinstance(arr, cp.ndarray): - obj = np.asarray(arr.get()).view(cls) - return obj - elif isinstance(arr, np.ndarray): - obj = np.asarray(arr).view(cls) - return obj - else: - raise TypeError( - "Unsupported array type. Must be numpy.ndarray or cupy.ndarray" - ) - - def __array_finalize__(self, obj): - self._fsproxy_wrapped = getattr(obj, "_fsproxy_wrapped", None) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index e5483fff913..6292022d8e4 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1632,11 +1632,3 @@ def test_change_index_name(index): assert s.index.name == name assert df.index.name == name - - -def test_numpy_ndarray_isinstancecheck(series): - s1, s2 = series - arr1 = s1.values - arr2 = s2.values - assert isinstance(arr1, np.ndarray) - assert isinstance(arr2, np.ndarray) From 592342c152af743390a923f125a380fe3b8f41c1 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:28:35 -0400 Subject: [PATCH 083/270] Remove invalid column_view usage in string-scalar-to-column function (#16530) Fixes the `make_column_from_scalar` function for `string_scalar` internal usage of a temporary `column_view` with non-zero size but no data or children to call `cudf::strings::detail::fill`. This relied too much on fragile internal logic which has cause several headaches including the recent work adding prefetch logic to libcudf. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16530 --- cpp/src/column/column_factories.cu | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/cpp/src/column/column_factories.cu b/cpp/src/column/column_factories.cu index bad20d6817c..ad9c5e4d3a0 100644 --- a/cpp/src/column/column_factories.cu +++ b/cpp/src/column/column_factories.cu @@ -20,11 +20,12 @@ #include #include #include -#include +#include #include #include +#include namespace cudf { @@ -57,15 +58,26 @@ std::unique_ptr column_from_scalar_dispatch::operator() const&>(value); + if (!value.is_valid(stream)) { + return make_strings_column( + size, + make_column_from_scalar(numeric_scalar(0), size + 1, stream, mr), + rmm::device_buffer{}, + size, + cudf::detail::create_null_mask(size, mask_state::ALL_NULL, stream, mr)); + } - // fill the column with the scalar - auto output = strings::detail::fill(strings_column_view(sc), 0, size, sv, stream, mr); + auto& ss = static_cast const&>(value); + auto const d_str = ss.value(stream); // no actual data is copied - return output; + // fill the column with the scalar + rmm::device_uvector indices(size, stream); + auto const row_value = + d_str.empty() ? cudf::strings::detail::string_index_pair{"", 0} + : cudf::strings::detail::string_index_pair{d_str.data(), d_str.size_bytes()}; + thrust::uninitialized_fill( + rmm::exec_policy_nosync(stream), indices.begin(), indices.end(), row_value); + return cudf::strings::detail::make_strings_column(indices.begin(), indices.end(), stream, mr); } template <> From 1b18cbc1e0b0e5dd7109228ce34c0fde5a2ddcb8 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Mon, 19 Aug 2024 08:03:54 -0700 Subject: [PATCH 084/270] Add `ToCudfBackend` expression to dask-cudf (#16573) Adds a `ToCudfBackend` expression for "pandas" to "cudf" conversion, preventing `to_backend("cudf")` operations from blocking useful optimizations like predicate pushdown. This is the dask-cudf component of https://github.com/dask/dask-expr/pull/1115 Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/cudf/pull/16573 --- python/dask_cudf/dask_cudf/backends.py | 20 +++++++----- python/dask_cudf/dask_cudf/expr/_expr.py | 31 ++++++++++++++++++- python/dask_cudf/dask_cudf/tests/test_core.py | 16 +++++++++- python/dask_cudf/dask_cudf/tests/utils.py | 4 +++ 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index a65ae819b44..16b2c8959e2 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -537,6 +537,12 @@ def to_cudf_dispatch_from_pandas(data, nan_as_null=None, **kwargs): return cudf.from_pandas(data, nan_as_null=nan_as_null) +@to_cudf_dispatch.register((cudf.DataFrame, cudf.Series, cudf.Index)) +def to_cudf_dispatch_from_cudf(data, **kwargs): + _unsupported_kwargs("cudf", "cudf", kwargs) + return data + + # Define "cudf" backend engine to be registered with Dask class CudfBackendEntrypoint(DataFrameBackendEntrypoint): """Backend-entrypoint class for Dask-DataFrame @@ -643,20 +649,20 @@ class CudfDXBackendEntrypoint(DataFrameBackendEntrypoint): Examples -------- >>> import dask - >>> import dask_expr + >>> import dask_expr as dx >>> with dask.config.set({"dataframe.backend": "cudf"}): ... ddf = dx.from_dict({"a": range(10)}) >>> type(ddf._meta) """ - @classmethod - def to_backend_dispatch(cls): - return CudfBackendEntrypoint.to_backend_dispatch() + @staticmethod + def to_backend(data, **kwargs): + import dask_expr as dx - @classmethod - def to_backend(cls, *args, **kwargs): - return CudfBackendEntrypoint.to_backend(*args, **kwargs) + from dask_cudf.expr._expr import ToCudfBackend + + return dx.new_collection(ToCudfBackend(data, kwargs)) @staticmethod def from_dict( diff --git a/python/dask_cudf/dask_cudf/expr/_expr.py b/python/dask_cudf/dask_cudf/expr/_expr.py index 8fccaccb695..8a2c50d3fe7 100644 --- a/python/dask_cudf/dask_cudf/expr/_expr.py +++ b/python/dask_cudf/dask_cudf/expr/_expr.py @@ -4,12 +4,41 @@ import dask_expr._shuffle as _shuffle_module from dask_expr import new_collection from dask_expr._cumulative import CumulativeBlockwise -from dask_expr._expr import Expr, VarColumns +from dask_expr._expr import Elemwise, Expr, VarColumns from dask_expr._reductions import Reduction, Var from dask.dataframe.core import is_dataframe_like, make_meta, meta_nonempty from dask.dataframe.dispatch import is_categorical_dtype +import cudf + +## +## Custom expressions +## + + +class ToCudfBackend(Elemwise): + # TODO: Inherit from ToBackend when rapids-dask-dependency + # is pinned to dask>=2024.8.1 + _parameters = ["frame", "options"] + _projection_passthrough = True + _filter_passthrough = True + _preserves_partitioning_information = True + + @staticmethod + def operation(df, options): + from dask_cudf.backends import to_cudf_dispatch + + return to_cudf_dispatch(df, **options) + + def _simplify_down(self): + if isinstance( + self.frame._meta, (cudf.DataFrame, cudf.Series, cudf.Index) + ): + # We already have cudf data + return self.frame + + ## ## Custom expression patching ## diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index 174923c2c7e..905d8c08135 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -15,7 +15,11 @@ import cudf import dask_cudf -from dask_cudf.tests.utils import skip_dask_expr, xfail_dask_expr +from dask_cudf.tests.utils import ( + require_dask_expr, + skip_dask_expr, + xfail_dask_expr, +) def test_from_dict_backend_dispatch(): @@ -993,3 +997,13 @@ def test_series_isin_error(): ser.isin([1, 5, "a"]) with pytest.raises(TypeError): ddf.isin([1, 5, "a"]).compute() + + +@require_dask_expr() +def test_to_backend_simplify(): + # Check that column projection is not blocked by to_backend + with dask.config.set({"dataframe.backend": "pandas"}): + df = dd.from_dict({"x": [1, 2, 3], "y": [4, 5, 6]}, npartitions=2) + df2 = df.to_backend("cudf")[["y"]].simplify() + df3 = df[["y"]].to_backend("cudf").to_backend("cudf").simplify() + assert df2._name == df3._name diff --git a/python/dask_cudf/dask_cudf/tests/utils.py b/python/dask_cudf/dask_cudf/tests/utils.py index c7dedbb6b4a..cc0c6899804 100644 --- a/python/dask_cudf/dask_cudf/tests/utils.py +++ b/python/dask_cudf/dask_cudf/tests/utils.py @@ -48,3 +48,7 @@ def xfail_dask_expr(reason=_default_reason, lt_version=None): else: xfail = QUERY_PLANNING_ON return pytest.mark.xfail(xfail, reason=reason) + + +def require_dask_expr(reason="requires dask-expr"): + return pytest.mark.skipif(not QUERY_PLANNING_ON, reason=reason) From 049177839e79dd28c776b5edfb2fd3f6c1b884a2 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 19 Aug 2024 17:05:16 +0200 Subject: [PATCH 085/270] MAINT: Adapt to numpy hiding flagsobject away (#16593) Authors: - Sebastian Berg (https://github.com/seberg) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16593 --- python/cudf/cudf/pandas/_wrappers/numpy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/pandas/_wrappers/numpy.py b/python/cudf/cudf/pandas/_wrappers/numpy.py index 3b012169676..90ac5198270 100644 --- a/python/cudf/cudf/pandas/_wrappers/numpy.py +++ b/python/cudf/cudf/pandas/_wrappers/numpy.py @@ -7,7 +7,7 @@ import cupy import cupy._core.flags import numpy -import numpy.core.multiarray +from packaging import version from ..fast_slow_proxy import ( _FastSlowAttribute, @@ -141,10 +141,15 @@ def wrap_ndarray(cls, arr: cupy.ndarray | numpy.ndarray, constructor): }, ) +if version.parse(numpy.__version__) >= version.parse("2.0"): + # NumPy 2 introduced `_core` and gives warnings for access to `core`. + from numpy._core.multiarray import flagsobj as _numpy_flagsobj +else: + from numpy.core.multiarray import flagsobj as _numpy_flagsobj # Mapping flags between slow and fast types _ndarray_flags = make_intermediate_proxy_type( "_ndarray_flags", cupy._core.flags.Flags, - numpy.core.multiarray.flagsobj, + _numpy_flagsobj, ) From c516fc48694b6bdbeeb5b31ebdc760034efdb285 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:55:03 -1000 Subject: [PATCH 086/270] Make ListColumn.__init__ strict (#16465) This PR makes `ListColumn.__init__` strict putting restrictions on data, dtype, size and children so these columns cannot be constructed into to an invalid state. It also aligns the signature with the base class. xref https://github.com/rapidsai/cudf/issues/16469 Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16465 --- python/cudf/cudf/core/column/column.py | 5 +- python/cudf/cudf/core/column/lists.py | 64 +++++++++++++++++--------- python/cudf/cudf/core/column/string.py | 1 + python/cudf/cudf/tests/test_list.py | 2 + 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 19d6bf84d3f..0857727d23f 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1625,12 +1625,13 @@ def build_column( ) elif isinstance(dtype, ListDtype): return cudf.core.column.ListColumn( - size=size, + data=None, + size=size, # type: ignore[arg-type] dtype=dtype, mask=mask, offset=offset, null_count=null_count, - children=children, + children=children, # type: ignore[arg-type] ) elif isinstance(dtype, IntervalDtype): return cudf.core.column.IntervalColumn( diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index 1b7cd95b3d0..302f04a0e71 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -3,7 +3,7 @@ from __future__ import annotations from functools import cached_property -from typing import TYPE_CHECKING, Sequence +from typing import TYPE_CHECKING, Sequence, cast import numpy as np import pandas as pd @@ -29,30 +29,46 @@ from cudf.api.types import _is_non_decimal_numeric_dtype, is_scalar from cudf.core.column import ColumnBase, as_column, column from cudf.core.column.methods import ColumnMethods, ParentType +from cudf.core.column.numerical import NumericalColumn from cudf.core.dtypes import ListDtype from cudf.core.missing import NA if TYPE_CHECKING: from cudf._typing import ColumnBinaryOperand, ColumnLike, Dtype, ScalarLike + from cudf.core.buffer import Buffer class ListColumn(ColumnBase): - dtype: ListDtype _VALID_BINARY_OPERATIONS = {"__add__", "__radd__"} def __init__( self, - size, - dtype, - mask=None, - offset=0, - null_count=None, - children=(), + data: None, + size: int, + dtype: ListDtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple[NumericalColumn, ColumnBase] = (), # type: ignore[assignment] ): + if data is not None: + raise ValueError("data must be None") + if not isinstance(dtype, ListDtype): + raise ValueError("dtype must be a cudf.ListDtype") + if not ( + len(children) == 2 + and isinstance(children[0], NumericalColumn) + # TODO: Enforce int32_t (size_type) used in libcudf? + and children[0].dtype.kind == "i" + and isinstance(children[1], ColumnBase) + ): + raise ValueError( + "children must a tuple of 2 columns of (signed integer offsets, list values)" + ) super().__init__( - None, - size, - dtype, + data=data, + size=size, + dtype=dtype, mask=mask, offset=offset, null_count=null_count, @@ -131,7 +147,7 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: raise TypeError("can only concatenate list to list") @property - def elements(self): + def elements(self) -> ColumnBase: """ Column containing the elements of each list (may itself be a ListColumn) @@ -139,11 +155,11 @@ def elements(self): return self.children[1] @property - def offsets(self): + def offsets(self) -> NumericalColumn: """ Integer offsets to elements specifying each row of the ListColumn """ - return self.children[0] + return cast(NumericalColumn, self.children[0]) def to_arrow(self): offsets = self.offsets.to_arrow() @@ -172,10 +188,9 @@ def set_base_data(self, value): else: super().set_base_data(value) - def set_base_children(self, value: tuple[ColumnBase, ...]): + def set_base_children(self, value: tuple[NumericalColumn, ColumnBase]): # type: ignore[override] super().set_base_children(value) - _, values = value - self._dtype = cudf.ListDtype(element_type=values.dtype) + self._dtype = cudf.ListDtype(element_type=value[1].dtype) @property def __cuda_array_interface__(self): @@ -196,12 +211,13 @@ def _with_type_metadata( dtype.element_type ) return ListColumn( + data=None, dtype=dtype, mask=self.base_mask, size=self.size, offset=self.offset, null_count=self.null_count, - children=(self.base_children[0], elements), + children=(self.base_children[0], elements), # type: ignore[arg-type] ) return self @@ -226,24 +242,25 @@ def from_sequences( """ data_col = column.column_empty(0) mask_col = [] - offset_col = [0] + offset_vals = [0] offset = 0 # Build Data, Mask & Offsets for data in arbitrary: if cudf._lib.scalar._is_null_host_scalar(data): mask_col.append(False) - offset_col.append(offset) + offset_vals.append(offset) else: mask_col.append(True) data_col = data_col.append(as_column(data)) offset += len(data) - offset_col.append(offset) + offset_vals.append(offset) - offset_col = column.as_column(offset_col, dtype=size_type_dtype) + offset_col = column.as_column(offset_vals, dtype=size_type_dtype) # Build ListColumn res = cls( + data=None, size=len(arbitrary), dtype=cudf.ListDtype(data_col.dtype), mask=cudf._lib.transform.bools_to_mask(as_column(mask_col)), @@ -283,12 +300,13 @@ def _transform_leaves(self, func, *args, **kwargs) -> Self: for c in cc: o = c.children[0] lc = cudf.core.column.ListColumn( # type: ignore + data=None, size=c.size, dtype=cudf.ListDtype(lc.dtype), mask=c.mask, offset=c.offset, null_count=c.null_count, - children=(o, lc), + children=(o, lc), # type: ignore[arg-type] ) return lc diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index a710a9f46c2..6f7508822d4 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -549,6 +549,7 @@ def _split_by_character(self): offset_col = col.children[0] return cudf.core.column.ListColumn( + data=None, size=len(col), dtype=cudf.ListDtype(col.dtype), mask=col.mask, diff --git a/python/cudf/cudf/tests/test_list.py b/python/cudf/cudf/tests/test_list.py index c4c883ca9f9..7d87fc73621 100644 --- a/python/cudf/cudf/tests/test_list.py +++ b/python/cudf/cudf/tests/test_list.py @@ -928,6 +928,7 @@ def test_empty_nested_list_uninitialized_offsets_memory_usage(): col = column_empty(0, cudf.ListDtype(cudf.ListDtype("int64"))) nested_col = col.children[1] empty_inner = type(nested_col)( + data=None, size=nested_col.size, dtype=nested_col.dtype, mask=nested_col.mask, @@ -939,6 +940,7 @@ def test_empty_nested_list_uninitialized_offsets_memory_usage(): ), ) col_empty_offset = type(col)( + data=None, size=col.size, dtype=col.dtype, mask=col.mask, From 074abcc0fa9eb9d2944b145f29fa02eb9edddc55 Mon Sep 17 00:00:00 2001 From: Nghia Truong <7416935+ttnghia@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:37:21 -0700 Subject: [PATCH 087/270] Add `public` qualifier for some member functions in Java class `Schema` (#16583) This adds the public qualifier for some member functions of `Schema` class in Java code, allowing them to be accessed outside of the `ai.rapids.cudf` package such as from spark-rapids-jni or Spark plugin. Java docs are also added for the newly became public functions as well as some existing public functions. Authors: - Nghia Truong (https://github.com/ttnghia) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) URL: https://github.com/rapidsai/cudf/pull/16583 --- java/src/main/java/ai/rapids/cudf/Schema.java | 56 +++++++++++++++---- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/Schema.java b/java/src/main/java/ai/rapids/cudf/Schema.java index 43603386649..76b2799aad6 100644 --- a/java/src/main/java/ai/rapids/cudf/Schema.java +++ b/java/src/main/java/ai/rapids/cudf/Schema.java @@ -120,7 +120,7 @@ private void flattenIfNeeded() { private int flattenedLength(int startingLength) { if (childSchemas != null) { - for (Schema child: childSchemas) { + for (Schema child : childSchemas) { startingLength++; startingLength = child.flattenedLength(startingLength); } @@ -150,11 +150,19 @@ public static Builder builder() { return new Builder(DType.STRUCT); } + /** + * Get names of the columns flattened from all levels in schema by depth-first traversal. + * @return An array containing names of all columns in schema. + */ public String[] getFlattenedColumnNames() { flattenIfNeeded(); return flattenedNames; } + /** + * Get names of the top level child columns in schema. + * @return An array containing names of top level child columns. + */ public String[] getColumnNames() { if (childNames == null) { return null; @@ -162,6 +170,10 @@ public String[] getColumnNames() { return childNames.toArray(new String[childNames.size()]); } + /** + * Check if the schema is nested (i.e., top level type is LIST or STRUCT). + * @return true if the schema is nested, false otherwise. + */ public boolean isNested() { return childSchemas != null && childSchemas.size() > 0; } @@ -173,7 +185,7 @@ public boolean isNested() { */ public boolean hasNestedChildren() { if (childSchemas != null) { - for (Schema child: childSchemas) { + for (Schema child : childSchemas) { if (child.isNested()) { return true; } @@ -182,7 +194,11 @@ public boolean hasNestedChildren() { return false; } - int[] getFlattenedTypeIds() { + /** + * Get type ids of the columns flattened from all levels in schema by depth-first traversal. + * @return An array containing type ids of all columns in schema. + */ + public int[] getFlattenedTypeIds() { flattenIfNeeded(); if (flattenedTypes == null) { return null; @@ -194,7 +210,11 @@ int[] getFlattenedTypeIds() { return ret; } - int[] getFlattenedTypeScales() { + /** + * Get scales of the columns' types flattened from all levels in schema by depth-first traversal. + * @return An array containing type scales of all columns in schema. + */ + public int[] getFlattenedTypeScales() { flattenIfNeeded(); if (flattenedTypes == null) { return null; @@ -206,11 +226,19 @@ int[] getFlattenedTypeScales() { return ret; } - DType[] getFlattenedTypes() { + /** + * Get the types of the columns in schema flattened from all levels by depth-first traversal. + * @return An array containing types of all columns in schema. + */ + public DType[] getFlattenedTypes() { flattenIfNeeded(); return flattenedTypes; } + /** + * Get types of the top level child columns in schema. + * @return An array containing types of top level child columns. + */ public DType[] getChildTypes() { if (childSchemas == null) { return null; @@ -222,6 +250,10 @@ public DType[] getChildTypes() { return ret; } + /** + * Get number of top level child columns in schema. + * @return Number of child columns. + */ public int getNumChildren() { if (childSchemas == null) { return 0; @@ -229,7 +261,11 @@ public int getNumChildren() { return childSchemas.size(); } - int[] getFlattenedNumChildren() { + /** + * Get numbers of child columns for each level in schema. + * @return Numbers of child columns for all levels flattened by depth-first traversal. + */ + public int[] getFlattenedNumChildren() { flattenIfNeeded(); return flattenedCounts; } @@ -253,7 +289,7 @@ public boolean isStructOrHasStructDescendant() { public HostColumnVector.DataType asHostDataType() { if (topLevelType == DType.LIST) { - assert(childSchemas != null && childSchemas.size() == 1); + assert (childSchemas != null && childSchemas.size() == 1); HostColumnVector.DataType element = childSchemas.get(0).asHostDataType(); return new HostColumnVector.ListType(true, element); } else if (topLevelType == DType.STRUCT) { @@ -261,7 +297,7 @@ public HostColumnVector.DataType asHostDataType() { return new HostColumnVector.StructType(true); } else { List childTypes = - childSchemas.stream().map(Schema::asHostDataType).collect(Collectors.toList()); + childSchemas.stream().map(Schema::asHostDataType).collect(Collectors.toList()); return new HostColumnVector.StructType(true, childTypes); } } else { @@ -269,7 +305,7 @@ public HostColumnVector.DataType asHostDataType() { } } - public static class Builder { + public static class Builder { private final DType topLevelType; private final List names; private final List types; @@ -326,7 +362,7 @@ public Schema build() { List children = null; if (types != null) { children = new ArrayList<>(types.size()); - for (Builder b: types) { + for (Builder b : types) { children.add(b.build()); } } From 79a5a97b2662bab6862ed895a6d802edd17d2502 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 19 Aug 2024 13:59:10 -0700 Subject: [PATCH 088/270] Remove NativeFile support from cudf Python (#16589) This PR removes all support for passing NativeFile objects through cudf's I/O routines. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Richard (Rick) Zamora (https://github.com/rjzamora) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16589 --- python/cudf/cudf/_lib/csv.pyx | 9 +- python/cudf/cudf/_lib/orc.pyx | 10 - python/cudf/cudf/_lib/parquet.pyx | 43 +--- python/cudf/cudf/io/csv.py | 11 +- python/cudf/cudf/io/orc.py | 33 +-- python/cudf/cudf/io/parquet.py | 102 ++------ python/cudf/cudf/tests/test_csv.py | 13 - python/cudf/cudf/tests/test_gcs.py | 6 +- python/cudf/cudf/tests/test_parquet.py | 33 +-- python/cudf/cudf/tests/test_s3.py | 168 +++---------- python/cudf/cudf/utils/ioutils.py | 234 ++---------------- python/cudf/cudf/utils/utils.py | 26 -- .../dask_cudf/dask_cudf/io/tests/test_s3.py | 48 ---- python/pylibcudf/pylibcudf/io/datasource.pxd | 7 - python/pylibcudf/pylibcudf/io/datasource.pyx | 24 -- 15 files changed, 86 insertions(+), 681 deletions(-) diff --git a/python/cudf/cudf/_lib/csv.pyx b/python/cudf/cudf/_lib/csv.pyx index e9aa97ecbc9..a90fe0f9ac6 100644 --- a/python/cudf/cudf/_lib/csv.pyx +++ b/python/cudf/cudf/_lib/csv.pyx @@ -7,7 +7,6 @@ from libcpp.utility cimport move from libcpp.vector cimport vector cimport pylibcudf.libcudf.types as libcudf_types -from pylibcudf.io.datasource cimport Datasource, NativeFileDatasource from cudf._lib.types cimport dtype_to_pylibcudf_type @@ -35,8 +34,6 @@ from pylibcudf.libcudf.table.table_view cimport table_view from cudf._lib.io.utils cimport make_sink_info from cudf._lib.utils cimport data_from_pylibcudf_io, table_view_from_table -from pyarrow.lib import NativeFile - import pylibcudf as plc from cudf.api.types import is_hashable @@ -127,9 +124,7 @@ def read_csv( cudf.read_csv """ - if not isinstance(datasource, (BytesIO, StringIO, bytes, - Datasource, - NativeFile)): + if not isinstance(datasource, (BytesIO, StringIO, bytes)): if not os.path.isfile(datasource): raise FileNotFoundError( errno.ENOENT, os.strerror(errno.ENOENT), datasource @@ -139,8 +134,6 @@ def read_csv( datasource = datasource.read().encode() elif isinstance(datasource, str) and not os.path.isfile(datasource): datasource = datasource.encode() - elif isinstance(datasource, NativeFile): - datasource = NativeFileDatasource(datasource) validate_args(delimiter, sep, delim_whitespace, decimal, thousands, nrows, skipfooter, byte_range, skiprows) diff --git a/python/cudf/cudf/_lib/orc.pyx b/python/cudf/cudf/_lib/orc.pyx index d506dcd4346..adeba6fffb1 100644 --- a/python/cudf/cudf/_lib/orc.pyx +++ b/python/cudf/cudf/_lib/orc.pyx @@ -22,7 +22,6 @@ except ImportError: import json cimport pylibcudf.libcudf.io.types as cudf_io_types -from pylibcudf.io.datasource cimport NativeFileDatasource from pylibcudf.libcudf.io.data_sink cimport data_sink from pylibcudf.libcudf.io.orc cimport ( chunked_orc_writer_options, @@ -71,8 +70,6 @@ from cudf._lib.types import SUPPORTED_NUMPY_TO_LIBCUDF_TYPES from cudf._lib.types cimport underlying_type_t_type_id from cudf._lib.utils cimport data_from_unique_ptr, table_view_from_table -from pyarrow.lib import NativeFile - from cudf._lib.utils import _index_level_name, generate_pandas_metadata @@ -204,10 +201,6 @@ cpdef read_parsed_orc_statistics(filepath_or_buffer): cudf.io.orc.read_orc_statistics """ - # Handle NativeFile input - if isinstance(filepath_or_buffer, NativeFile): - filepath_or_buffer = NativeFileDatasource(filepath_or_buffer) - cdef parsed_orc_statistics parsed = ( libcudf_read_parsed_orc_statistics(make_source_info([filepath_or_buffer])) ) @@ -490,9 +483,6 @@ cdef orc_reader_options make_orc_reader_options( bool use_index ) except*: - for i, datasource in enumerate(filepaths_or_buffers): - if isinstance(datasource, NativeFile): - filepaths_or_buffers[i] = NativeFileDatasource(datasource) cdef vector[vector[size_type]] strps = stripes cdef orc_reader_options opts cdef source_info src = make_source_info(filepaths_or_buffers) diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index 4bfb79ff651..c874a51e220 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -34,7 +34,6 @@ from libcpp.vector cimport vector cimport pylibcudf.libcudf.io.data_sink as cudf_io_data_sink cimport pylibcudf.libcudf.io.types as cudf_io_types from pylibcudf.expressions cimport Expression -from pylibcudf.io.datasource cimport NativeFileDatasource from pylibcudf.io.parquet cimport ChunkedParquetReader from pylibcudf.libcudf.io.parquet cimport ( chunked_parquet_writer_options, @@ -62,8 +61,6 @@ from cudf._lib.io.utils cimport ( ) from cudf._lib.utils cimport table_view_from_table -from pyarrow.lib import NativeFile - import pylibcudf as plc from pylibcudf cimport Table @@ -133,7 +130,6 @@ cdef object _process_metadata(object df, list per_file_user_data, object row_groups, object filepaths_or_buffers, - list pa_buffers, bool allow_range_index, bool use_pandas_metadata, size_type nrows=-1, @@ -199,9 +195,7 @@ cdef object _process_metadata(object df, pa.parquet.read_metadata( # Pyarrow cannot read directly from bytes io.BytesIO(s) if isinstance(s, bytes) else s - ) for s in ( - pa_buffers or filepaths_or_buffers - ) + ) for s in filepaths_or_buffers ] filtered_idx = [] @@ -274,27 +268,13 @@ def read_parquet_chunked( size_type nrows=-1, int64_t skip_rows=0 ): - # Convert NativeFile buffers to NativeFileDatasource, - # but save original buffers in case we need to use - # pyarrow for metadata processing - # (See: https://github.com/rapidsai/cudf/issues/9599) - - pa_buffers = [] - - new_bufs = [] - for i, datasource in enumerate(filepaths_or_buffers): - if isinstance(datasource, NativeFile): - new_bufs.append(NativeFileDatasource(datasource)) - else: - new_bufs.append(datasource) - # Note: If this function ever takes accepts filters # allow_range_index needs to be False when a filter is passed # (see read_parquet) allow_range_index = columns is not None and len(columns) != 0 reader = ChunkedParquetReader( - plc.io.SourceInfo(new_bufs), + plc.io.SourceInfo(filepaths_or_buffers), columns, row_groups, use_pandas_metadata, @@ -333,7 +313,7 @@ def read_parquet_chunked( ) df = _process_metadata(df, column_names, child_names, per_file_user_data, row_groups, - filepaths_or_buffers, pa_buffers, + filepaths_or_buffers, allow_range_index, use_pandas_metadata, nrows=nrows, skip_rows=skip_rows) return df @@ -356,16 +336,6 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, cudf.io.parquet.to_parquet """ - # Convert NativeFile buffers to NativeFileDatasource, - # but save original buffers in case we need to use - # pyarrow for metadata processing - # (See: https://github.com/rapidsai/cudf/issues/9599) - pa_buffers = [] - for i, datasource in enumerate(filepaths_or_buffers): - if isinstance(datasource, NativeFile): - pa_buffers.append(datasource) - filepaths_or_buffers[i] = NativeFileDatasource(datasource) - allow_range_index = True if columns is not None and len(columns) == 0 or filters: allow_range_index = False @@ -389,7 +359,7 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, df = _process_metadata(df, tbl_w_meta.column_names(include_children=False), tbl_w_meta.child_names, tbl_w_meta.per_file_user_data, - row_groups, filepaths_or_buffers, pa_buffers, + row_groups, filepaths_or_buffers, allow_range_index, use_pandas_metadata, nrows=nrows, skip_rows=skip_rows) return df @@ -403,11 +373,6 @@ cpdef read_parquet_metadata(filepaths_or_buffers): cudf.io.parquet.read_parquet cudf.io.parquet.to_parquet """ - # Convert NativeFile buffers to NativeFileDatasource - for i, datasource in enumerate(filepaths_or_buffers): - if isinstance(datasource, NativeFile): - filepaths_or_buffers[i] = NativeFileDatasource(datasource) - cdef cudf_io_types.source_info source = make_source_info(filepaths_or_buffers) args = move(source) diff --git a/python/cudf/cudf/io/csv.py b/python/cudf/cudf/io/csv.py index 0f2820a01e9..e61fc5063dc 100644 --- a/python/cudf/cudf/io/csv.py +++ b/python/cudf/cudf/io/csv.py @@ -5,7 +5,6 @@ from io import BytesIO, StringIO import numpy as np -from pyarrow.lib import NativeFile import cudf from cudf import _lib as libcudf @@ -50,7 +49,6 @@ def read_csv( comment=None, delim_whitespace=False, byte_range=None, - use_python_file_object=None, storage_options=None, bytes_per_thread=None, ): @@ -63,12 +61,6 @@ def read_csv( FutureWarning, ) - if use_python_file_object and bytes_per_thread is not None: - raise ValueError( - "bytes_per_thread is only supported when " - "`use_python_file_object=False`" - ) - if bytes_per_thread is None: bytes_per_thread = ioutils._BYTES_PER_THREAD_DEFAULT @@ -84,8 +76,7 @@ def read_csv( filepath_or_buffer, compression = ioutils.get_reader_filepath_or_buffer( path_or_data=filepath_or_buffer, compression=compression, - iotypes=(BytesIO, StringIO, NativeFile), - use_python_file_object=use_python_file_object, + iotypes=(BytesIO, StringIO), storage_options=storage_options, bytes_per_thread=bytes_per_thread, ) diff --git a/python/cudf/cudf/io/orc.py b/python/cudf/cudf/io/orc.py index 289292b5182..4f04caafc5d 100644 --- a/python/cudf/cudf/io/orc.py +++ b/python/cudf/cudf/io/orc.py @@ -10,7 +10,6 @@ from cudf._lib import orc as liborc from cudf.api.types import is_list_like from cudf.utils import ioutils -from cudf.utils.utils import maybe_filter_deprecation def _make_empty_df(filepath_or_buffer, columns): @@ -281,7 +280,6 @@ def read_orc( num_rows=None, use_index=True, timestamp_type=None, - use_python_file_object=None, storage_options=None, bytes_per_thread=None, ): @@ -321,9 +319,6 @@ def read_orc( ) filepaths_or_buffers = [] - have_nativefile = any( - isinstance(source, pa.NativeFile) for source in filepath_or_buffer - ) for source in filepath_or_buffer: if ioutils.is_directory( path_or_data=source, storage_options=storage_options @@ -339,7 +334,6 @@ def read_orc( tmp_source, compression = ioutils.get_reader_filepath_or_buffer( path_or_data=source, compression=None, - use_python_file_object=use_python_file_object, storage_options=storage_options, bytes_per_thread=bytes_per_thread, ) @@ -364,24 +358,17 @@ def read_orc( stripes = selected_stripes if engine == "cudf": - # Don't want to warn if use_python_file_object causes us to get - # a NativeFile (there is a separate deprecation warning for that) - with maybe_filter_deprecation( - not have_nativefile, - message="Support for reading pyarrow's NativeFile is deprecated", - category=FutureWarning, - ): - return DataFrame._from_data( - *liborc.read_orc( - filepaths_or_buffers, - columns, - stripes, - skiprows, - num_rows, - use_index, - timestamp_type, - ) + return DataFrame._from_data( + *liborc.read_orc( + filepaths_or_buffers, + columns, + stripes, + skiprows, + num_rows, + use_index, + timestamp_type, ) + ) else: from pyarrow import orc diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 4a419a2fbb6..fac51a9e471 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -15,7 +15,6 @@ import numpy as np import pandas as pd -import pyarrow as pa from pyarrow import dataset as ds import cudf @@ -24,7 +23,6 @@ from cudf.core.column import as_column, build_categorical_column, column_empty from cudf.utils import ioutils from cudf.utils.performance_tracking import _performance_tracking -from cudf.utils.utils import maybe_filter_deprecation BYTE_SIZES = { "kb": 1000, @@ -352,8 +350,6 @@ def read_parquet_metadata(filepath_or_buffer): path_or_data=source, compression=None, fs=fs, - use_python_file_object=None, - open_file_options=None, storage_options=None, bytes_per_thread=None, ) @@ -534,9 +530,7 @@ def read_parquet( filters=None, row_groups=None, use_pandas_metadata=True, - use_python_file_object=None, categorical_partitions=True, - open_file_options=None, bytes_per_thread=None, dataset_kwargs=None, nrows=None, @@ -549,16 +543,6 @@ def read_parquet( raise ValueError( f"Only supported engines are {{'cudf', 'pyarrow'}}, got {engine=}" ) - # Do not allow the user to set file-opening options - # when `use_python_file_object=False` is specified - if use_python_file_object is False: - if open_file_options: - raise ValueError( - "open_file_options is not currently supported when " - "use_python_file_object is set to False." - ) - open_file_options = {} - if bytes_per_thread is None: bytes_per_thread = ioutils._BYTES_PER_THREAD_DEFAULT @@ -612,23 +596,11 @@ def read_parquet( filepath_or_buffer = paths if paths else filepath_or_buffer filepaths_or_buffers = [] - if use_python_file_object: - open_file_options = _default_open_file_options( - open_file_options=open_file_options, - columns=columns, - row_groups=row_groups, - fs=fs, - ) - have_nativefile = any( - isinstance(source, pa.NativeFile) for source in filepath_or_buffer - ) for source in filepath_or_buffer: tmp_source, compression = ioutils.get_reader_filepath_or_buffer( path_or_data=source, compression=None, fs=fs, - use_python_file_object=use_python_file_object, - open_file_options=open_file_options, storage_options=storage_options, bytes_per_thread=bytes_per_thread, ) @@ -669,28 +641,20 @@ def read_parquet( ) # Convert parquet data to a cudf.DataFrame - - # Don't want to warn if use_python_file_object causes us to get - # a NativeFile (there is a separate deprecation warning for that) - with maybe_filter_deprecation( - not have_nativefile, - message="Support for reading pyarrow's NativeFile is deprecated", - category=FutureWarning, - ): - df = _parquet_to_frame( - filepaths_or_buffers, - engine, - *args, - columns=columns, - row_groups=row_groups, - use_pandas_metadata=use_pandas_metadata, - partition_keys=partition_keys, - partition_categories=partition_categories, - dataset_kwargs=dataset_kwargs, - nrows=nrows, - skip_rows=skip_rows, - **kwargs, - ) + df = _parquet_to_frame( + filepaths_or_buffers, + engine, + *args, + columns=columns, + row_groups=row_groups, + use_pandas_metadata=use_pandas_metadata, + partition_keys=partition_keys, + partition_categories=partition_categories, + dataset_kwargs=dataset_kwargs, + nrows=nrows, + skip_rows=skip_rows, + **kwargs, + ) # Apply filters row-wise (if any are defined), and return df = _apply_post_filters(df, filters) if projected_columns: @@ -1570,44 +1534,6 @@ def __exit__(self, *args): self.close() -def _default_open_file_options( - open_file_options, columns, row_groups, fs=None -): - """ - Set default fields in open_file_options. - - Copies and updates `open_file_options` to - include column and row-group information - under the "precache_options" key. By default, - we set "method" to "parquet", but precaching - will be disabled if the user chooses `method=None` - - Parameters - ---------- - open_file_options : dict or None - columns : list - row_groups : list - fs : fsspec.AbstractFileSystem, Optional - """ - if fs and ioutils._is_local_filesystem(fs): - # Quick return for local fs - return open_file_options or {} - # Assume remote storage if `fs` was not specified - open_file_options = (open_file_options or {}).copy() - precache_options = open_file_options.pop("precache_options", {}).copy() - if precache_options.get("method", "parquet") == "parquet": - precache_options.update( - { - "method": "parquet", - "engine": precache_options.get("engine", "pyarrow"), - "columns": columns, - "row_groups": row_groups, - } - ) - open_file_options["precache_options"] = precache_options - return open_file_options - - def _hive_dirname(name, val): # Simple utility to produce hive directory name if pd.isna(val): diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 6a21cb1b9d7..40ba415e681 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -13,7 +13,6 @@ import numpy as np import pandas as pd import pytest -from pyarrow import fs as pa_fs import cudf from cudf import read_csv @@ -1080,18 +1079,6 @@ def test_csv_reader_filepath_or_buffer(tmpdir, path_or_buf, src): assert_eq(expect, got) -def test_csv_reader_arrow_nativefile(path_or_buf): - # Check that we can read a file opened with the - # Arrow FileSystem interface - expect = cudf.read_csv(path_or_buf("filepath")) - fs, path = pa_fs.FileSystem.from_uri(path_or_buf("filepath")) - with pytest.warns(FutureWarning): - with fs.open_input_file(path) as fil: - got = cudf.read_csv(fil) - - assert_eq(expect, got) - - def test_small_zip(tmpdir): df = pd.DataFrame( { diff --git a/python/cudf/cudf/tests/test_gcs.py b/python/cudf/cudf/tests/test_gcs.py index 28fdfb5c2f1..82ecd356bbf 100644 --- a/python/cudf/cudf/tests/test_gcs.py +++ b/python/cudf/cudf/tests/test_gcs.py @@ -42,12 +42,8 @@ def mock_size(*args): monkeypatch.setattr(gcsfs.core.GCSFileSystem, "size", mock_size) # Test read from explicit path. - # Since we are monkey-patching, we cannot use - # use_python_file_object=True, because the pyarrow - # `open_input_file` command will fail (since it doesn't - # use the monkey-patched `open` definition) with pytest.warns(FutureWarning): - got = cudf.read_csv(f"gcs://{fpath}", use_python_file_object=False) + got = cudf.read_csv(f"gcs://{fpath}") assert_eq(pdf, got) # AbstractBufferedFile -> PythonFile conversion diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 879a2c50db7..db4f1c9c8bd 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -19,7 +19,7 @@ import pytest from fsspec.core import get_fs_token_paths from packaging import version -from pyarrow import fs as pa_fs, parquet as pq +from pyarrow import parquet as pq import cudf from cudf._lib.parquet import read_parquet_chunked @@ -705,40 +705,17 @@ def test_parquet_reader_filepath_or_buffer(parquet_path_or_buf, src): assert_eq(expect, got) -def test_parquet_reader_arrow_nativefile(parquet_path_or_buf): - # Check that we can read a file opened with the - # Arrow FileSystem interface - expect = cudf.read_parquet(parquet_path_or_buf("filepath")) - fs, path = pa_fs.FileSystem.from_uri(parquet_path_or_buf("filepath")) - with fs.open_input_file(path) as fil: - with pytest.warns(FutureWarning): - got = cudf.read_parquet(fil) - - assert_eq(expect, got) - - -@pytest.mark.parametrize("use_python_file_object", [True, False]) -def test_parquet_reader_use_python_file_object( - parquet_path_or_buf, use_python_file_object -): - # Check that the non-default `use_python_file_object=True` - # option works as expected +def test_parquet_reader_file_types(parquet_path_or_buf): expect = cudf.read_parquet(parquet_path_or_buf("filepath")) fs, _, paths = get_fs_token_paths(parquet_path_or_buf("filepath")) # Pass open fsspec file - with pytest.warns(FutureWarning): - with fs.open(paths[0], mode="rb") as fil: - got1 = cudf.read_parquet( - fil, use_python_file_object=use_python_file_object - ) + with fs.open(paths[0], mode="rb") as fil: + got1 = cudf.read_parquet(fil) assert_eq(expect, got1) # Pass path only - with pytest.warns(FutureWarning): - got2 = cudf.read_parquet( - paths[0], use_python_file_object=use_python_file_object - ) + got2 = cudf.read_parquet(paths[0]) assert_eq(expect, got2) diff --git a/python/cudf/cudf/tests/test_s3.py b/python/cudf/cudf/tests/test_s3.py index 3ae318d3bf5..6579fd23634 100644 --- a/python/cudf/cudf/tests/test_s3.py +++ b/python/cudf/cudf/tests/test_s3.py @@ -7,7 +7,6 @@ import numpy as np import pandas as pd -import pyarrow.fs as pa_fs import pytest from fsspec.core import get_fs_token_paths @@ -138,48 +137,17 @@ def test_read_csv(s3_base, s3so, pdf, bytes_per_thread): buffer = pdf.to_csv(index=False) # Use fsspec file object - with pytest.warns(FutureWarning): - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got = cudf.read_csv( - f"s3://{bucket}/{fname}", - storage_options=s3so, - bytes_per_thread=bytes_per_thread, - use_python_file_object=False, - ) - assert_eq(pdf, got) - - # Use Arrow PythonFile object - with pytest.warns(FutureWarning): - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got = cudf.read_csv( - f"s3://{bucket}/{fname}", - storage_options=s3so, - use_python_file_object=True, - ) - assert_eq(pdf, got) - - -def test_read_csv_arrow_nativefile(s3_base, s3so, pdf): - # Write to buffer - fname = "test_csv_reader_arrow_nativefile.csv" - bucket = "csv" - buffer = pdf.to_csv(index=False) with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - fs = pa_fs.S3FileSystem( - endpoint_override=s3so["client_kwargs"]["endpoint_url"], + got = cudf.read_csv( + f"s3://{bucket}/{fname}", + storage_options=s3so, + bytes_per_thread=bytes_per_thread, ) - with pytest.warns(FutureWarning): - with fs.open_input_file(f"{bucket}/{fname}") as fil: - got = cudf.read_csv(fil) - assert_eq(pdf, got) @pytest.mark.parametrize("bytes_per_thread", [32, 1024]) -@pytest.mark.parametrize("use_python_file_object", [True, False]) -def test_read_csv_byte_range( - s3_base, s3so, pdf, bytes_per_thread, use_python_file_object -): +def test_read_csv_byte_range(s3_base, s3so, pdf, bytes_per_thread): # Write to buffer fname = "test_csv_reader_byte_range.csv" bucket = "csv" @@ -187,18 +155,14 @@ def test_read_csv_byte_range( # Use fsspec file object with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - with pytest.warns(FutureWarning): - got = cudf.read_csv( - f"s3://{bucket}/{fname}", - storage_options=s3so, - byte_range=(74, 73), - bytes_per_thread=bytes_per_thread - if not use_python_file_object - else None, - header=None, - names=["Integer", "Float", "Integer2", "String", "Boolean"], - use_python_file_object=use_python_file_object, - ) + got = cudf.read_csv( + f"s3://{bucket}/{fname}", + storage_options=s3so, + byte_range=(74, 73), + bytes_per_thread=bytes_per_thread, + header=None, + names=["Integer", "Float", "Integer2", "String", "Boolean"], + ) assert_eq(pdf.iloc[-2:].reset_index(drop=True), got) @@ -226,16 +190,12 @@ def test_write_csv(s3_base, s3so, pdf, chunksize): @pytest.mark.parametrize("bytes_per_thread", [32, 1024]) @pytest.mark.parametrize("columns", [None, ["Float", "String"]]) -@pytest.mark.parametrize("precache", [None, "parquet"]) -@pytest.mark.parametrize("use_python_file_object", [True, False]) def test_read_parquet( s3_base, s3so, pdf, bytes_per_thread, columns, - precache, - use_python_file_object, ): fname = "test_parquet_reader.parquet" bucket = "parquet" @@ -245,19 +205,12 @@ def test_read_parquet( # Check direct path handling buffer.seek(0) with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - with pytest.warns(FutureWarning): - got1 = cudf.read_parquet( - f"s3://{bucket}/{fname}", - open_file_options=( - {"precache_options": {"method": precache}} - if use_python_file_object - else None - ), - storage_options=s3so, - bytes_per_thread=bytes_per_thread, - columns=columns, - use_python_file_object=use_python_file_object, - ) + got1 = cudf.read_parquet( + f"s3://{bucket}/{fname}", + storage_options=s3so, + bytes_per_thread=bytes_per_thread, + columns=columns, + ) expect = pdf[columns] if columns else pdf assert_eq(expect, got1) @@ -268,13 +221,11 @@ def test_read_parquet( f"s3://{bucket}/{fname}", storage_options=s3so )[0] with fs.open(f"s3://{bucket}/{fname}", mode="rb") as f: - with pytest.warns(FutureWarning): - got2 = cudf.read_parquet( - f, - bytes_per_thread=bytes_per_thread, - columns=columns, - use_python_file_object=use_python_file_object, - ) + got2 = cudf.read_parquet( + f, + bytes_per_thread=bytes_per_thread, + columns=columns, + ) assert_eq(expect, got2) @@ -350,28 +301,7 @@ def test_read_parquet_multi_file(s3_base, s3so, pdf): assert_eq(expect, got) -@pytest.mark.parametrize("columns", [None, ["Float", "String"]]) -def test_read_parquet_arrow_nativefile(s3_base, s3so, pdf, columns): - # Write to buffer - fname = "test_parquet_reader_arrow_nativefile.parquet" - bucket = "parquet" - buffer = BytesIO() - pdf.to_parquet(path=buffer) - buffer.seek(0) - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - with pytest.warns(FutureWarning): - fs = pa_fs.S3FileSystem( - endpoint_override=s3so["client_kwargs"]["endpoint_url"], - ) - with fs.open_input_file(f"{bucket}/{fname}") as fil: - got = cudf.read_parquet(fil, columns=columns) - - expect = pdf[columns] if columns else pdf - assert_eq(expect, got) - - -@pytest.mark.parametrize("precache", [None, "parquet"]) -def test_read_parquet_filters(s3_base, s3so, pdf_ext, precache): +def test_read_parquet_filters(s3_base, s3so, pdf_ext): fname = "test_parquet_reader_filters.parquet" bucket = "parquet" buffer = BytesIO() @@ -379,13 +309,11 @@ def test_read_parquet_filters(s3_base, s3so, pdf_ext, precache): buffer.seek(0) filters = [("String", "==", "Omega")] with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - with pytest.warns(FutureWarning): - got = cudf.read_parquet( - f"s3://{bucket}/{fname}", - storage_options=s3so, - filters=filters, - open_file_options={"precache_options": {"method": precache}}, - ) + got = cudf.read_parquet( + f"s3://{bucket}/{fname}", + storage_options=s3so, + filters=filters, + ) # All row-groups should be filtered out assert_eq(pdf_ext.iloc[:0], got.reset_index(drop=True)) @@ -445,33 +373,8 @@ def test_read_json(s3_base, s3so): assert_eq(expect, got) -@pytest.mark.parametrize("use_python_file_object", [False, True]) -@pytest.mark.parametrize("columns", [None, ["string1"]]) -def test_read_orc(s3_base, s3so, datadir, use_python_file_object, columns): - source_file = str(datadir / "orc" / "TestOrcFile.testSnappy.orc") - fname = "test_orc_reader.orc" - bucket = "orc" - expect = pd.read_orc(source_file) - - with open(source_file, "rb") as f: - buffer = f.read() - - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - with pytest.warns(FutureWarning): - got = cudf.read_orc( - f"s3://{bucket}/{fname}", - columns=columns, - storage_options=s3so, - use_python_file_object=use_python_file_object, - ) - - if columns: - expect = expect[columns] - assert_eq(expect, got) - - @pytest.mark.parametrize("columns", [None, ["string1"]]) -def test_read_orc_arrow_nativefile(s3_base, s3so, datadir, columns): +def test_read_orc(s3_base, s3so, datadir, columns): source_file = str(datadir / "orc" / "TestOrcFile.testSnappy.orc") fname = "test_orc_reader.orc" bucket = "orc" @@ -481,12 +384,11 @@ def test_read_orc_arrow_nativefile(s3_base, s3so, datadir, columns): buffer = f.read() with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - fs = pa_fs.S3FileSystem( - endpoint_override=s3so["client_kwargs"]["endpoint_url"], + got = cudf.read_orc( + f"s3://{bucket}/{fname}", + columns=columns, + storage_options=s3so, ) - with pytest.warns(FutureWarning): - with fs.open_input_file(f"{bucket}/{fname}") as fil: - got = cudf.read_orc(fil, columns=columns) if columns: expect = expect[columns] diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 448a815fe1b..4ac9b63985f 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -13,19 +13,10 @@ import numpy as np import pandas as pd from fsspec.core import get_fs_token_paths -from pyarrow import PythonFile as ArrowPythonFile -from pyarrow.lib import NativeFile -from cudf.api.extensions import no_default from cudf.core._compat import PANDAS_LT_300 from cudf.utils.docutils import docfmt_partial -try: - import fsspec.parquet as fsspec_parquet - -except ImportError: - fsspec_parquet = None - _BYTES_PER_THREAD_DEFAULT = 256 * 1024 * 1024 _ROW_GROUP_SIZE_BYTES_DEFAULT = 128 * 1024 * 1024 @@ -173,32 +164,12 @@ use_pandas_metadata : boolean, default True If True and dataset has custom PANDAS schema metadata, ensure that index columns are also loaded. -use_python_file_object : boolean, default True - If True, Arrow-backed PythonFile objects will be used in place of fsspec - AbstractBufferedFile objects at IO time. - - .. deprecated:: 24.08 - `use_python_file_object` is deprecated and will be removed in a future - version of cudf, as PyArrow NativeFiles will no longer be accepted as - input/output in cudf readers/writers in the future. -open_file_options : dict, optional - Dictionary of key-value pairs to pass to the function used to open remote - files. By default, this will be `fsspec.parquet.open_parquet_file`. To - deactivate optimized precaching, set the "method" to `None` under the - "precache_options" key. Note that the `open_file_func` key can also be - used to specify a custom file-open function. - - .. deprecated:: 24.08 - `open_file_options` is deprecated as it was intended for - pyarrow file inputs, which will no longer be accepted as - input/output cudf readers/writers in the future. bytes_per_thread : int, default None Determines the number of bytes to be allocated per thread to read the files in parallel. When there is a file of large size, we get slightly better throughput by decomposing it and transferring multiple "blocks" in parallel (using a python thread pool). Default allocation is {bytes_per_thread} bytes. - This parameter is functional only when `use_python_file_object=False`. skiprows : int, default None If not None, the number of rows to skip from the start of the file. @@ -485,14 +456,6 @@ This parameter is deprecated. use_index : bool, default True If True, use row index if available for faster seeking. -use_python_file_object : boolean, default True - If True, Arrow-backed PythonFile objects will be used in place of fsspec - AbstractBufferedFile objects at IO time. - - .. deprecated:: 24.08 - `use_python_file_object` is deprecated and will be removed in a future - version of cudf, as PyArrow NativeFiles will no longer be accepted as - input/output in cudf readers/writers in the future. storage_options : dict, optional, default None Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value @@ -506,7 +469,6 @@ better throughput by decomposing it and transferring multiple "blocks" in parallel (using a python thread pool). Default allocation is {bytes_per_thread} bytes. - This parameter is functional only when `use_python_file_object=False`. Returns ------- @@ -1209,14 +1171,6 @@ size to zero to read all data after the offset location. Reads the row that starts before or at the end of the range, even if it ends after the end of the range. -use_python_file_object : boolean, default True - If True, Arrow-backed PythonFile objects will be used in place of fsspec - AbstractBufferedFile objects at IO time. - - .. deprecated:: 24.08 - `use_python_file_object` is deprecated and will be removed in a future - version of cudf, as PyArrow NativeFiles will no longer be accepted as - input/output in cudf readers/writers in the future. storage_options : dict, optional, default None Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value @@ -1230,7 +1184,6 @@ better throughput by decomposing it and transferring multiple "blocks" in parallel (using a python thread pool). Default allocation is {bytes_per_thread} bytes. - This parameter is functional only when `use_python_file_object=False`. Returns ------- GPU ``DataFrame`` object. @@ -1454,22 +1407,6 @@ Mode in which file is opened iotypes : (), default (BytesIO) Object type to exclude from file-like check -use_python_file_object : boolean, default False - If True, Arrow-backed PythonFile objects will be used in place - of fsspec AbstractBufferedFile objects. - - .. deprecated:: 24.08 - `use_python_file_object` is deprecated and will be removed in a future - version of cudf, as PyArrow NativeFiles will no longer be accepted as - input/output in cudf readers/writers. -open_file_options : dict, optional - Optional dictionary of keyword arguments to pass to - `_open_remote_files` (used for remote storage only). - - .. deprecated:: 24.08 - `open_file_options` is deprecated as it was intended for - pyarrow file inputs, which will no longer be accepted as - input/output cudf readers/writers in the future. allow_raw_text_input : boolean, default False If True, this indicates the input `path_or_data` could be a raw text input and will not check for its existence in the filesystem. If False, @@ -1490,7 +1427,6 @@ better throughput by decomposing it and transferring multiple "blocks" in parallel (using a Python thread pool). Default allocation is {bytes_per_thread} bytes. - This parameter is functional only when `use_python_file_object=False`. Returns ------- @@ -1635,119 +1571,13 @@ def _get_filesystem_and_paths(path_or_data, storage_options): return fs, return_paths -def _set_context(obj, stack): - # Helper function to place open file on context stack - if stack is None: - return obj - return stack.enter_context(obj) - - -def _open_remote_files( - paths, - fs, - context_stack=None, - open_file_func=None, - precache_options=None, - **kwargs, -): - """Return a list of open file-like objects given - a list of remote file paths. - - Parameters - ---------- - paths : list(str) - List of file-path strings. - fs : fsspec.AbstractFileSystem - Fsspec file-system object. - context_stack : contextlib.ExitStack, Optional - Context manager to use for open files. - open_file_func : Callable, Optional - Call-back function to use for opening. If this argument - is specified, all other arguments will be ignored. - precache_options : dict, optional - Dictionary of key-word arguments to pass to use for - precaching. Unless the input contains ``{"method": None}``, - ``fsspec.parquet.open_parquet_file`` will be used for remote - storage. - **kwargs : - Key-word arguments to be passed to format-specific - open functions. - """ - - # Just use call-back function if one was specified - if open_file_func is not None: - return [ - _set_context(open_file_func(path, **kwargs), context_stack) - for path in paths - ] - - # Check if the "precache" option is supported. - # In the future, fsspec should do this check for us - precache_options = (precache_options or {}).copy() - precache = precache_options.pop("method", None) - if precache not in ("parquet", None): - raise ValueError(f"{precache} not a supported `precache` option.") - - # Check that "parts" caching (used for all format-aware file handling) - # is supported by the installed fsspec/s3fs version - if precache == "parquet" and not fsspec_parquet: - warnings.warn( - f"This version of fsspec ({fsspec.__version__}) does " - f"not support parquet-optimized precaching. Please upgrade " - f"to the latest fsspec version for better performance." - ) - precache = None - - if precache == "parquet": - # Use fsspec.parquet module. - # TODO: Use `cat_ranges` to collect "known" - # parts for all files at once. - row_groups = precache_options.pop("row_groups", None) or ( - [None] * len(paths) - ) - return [ - ArrowPythonFile( - _set_context( - fsspec_parquet.open_parquet_file( - path, - fs=fs, - row_groups=rgs, - **precache_options, - **kwargs, - ), - context_stack, - ) - ) - for path, rgs in zip(paths, row_groups) - ] - - # Avoid top-level pyarrow.fs import. - # Importing pyarrow.fs initializes a S3 SDK with a finalizer - # that runs atexit. In some circumstances it appears this - # runs a call into a logging system that is already shutdown. - # To avoid this, we only import this subsystem if it is - # really needed. - # See https://github.com/aws/aws-sdk-cpp/issues/2681 - from pyarrow.fs import FSSpecHandler, PyFileSystem - - # Default open - Use pyarrow filesystem API - pa_fs = PyFileSystem(FSSpecHandler(fs)) - return [ - _set_context(pa_fs.open_input_file(fpath), context_stack) - for fpath in paths - ] - - @doc_get_reader_filepath_or_buffer() def get_reader_filepath_or_buffer( path_or_data, compression, mode="rb", fs=None, - iotypes=(BytesIO, NativeFile), - # no_default aliases to False - use_python_file_object=no_default, - open_file_options=None, + iotypes=(BytesIO,), allow_raw_text_input=False, storage_options=None, bytes_per_thread=_BYTES_PER_THREAD_DEFAULT, @@ -1758,30 +1588,6 @@ def get_reader_filepath_or_buffer( path_or_data = stringify_pathlike(path_or_data) - if use_python_file_object is no_default: - use_python_file_object = False - elif use_python_file_object is not None: - warnings.warn( - "The 'use_python_file_object' keyword is deprecated and " - "will be removed in a future version.", - FutureWarning, - ) - else: - # Preserve the readers (e.g. read_csv) default of True - # if no use_python_file_object option is specified by the user - # for now (note: this is different from the default for this - # function of False) - # TODO: when non-pyarrow file reading perf is good enough - # we can default this to False - use_python_file_object = True - - if open_file_options is not None: - warnings.warn( - "The 'open_file_options' keyword is deprecated and " - "will be removed in a future version.", - FutureWarning, - ) - if isinstance(path_or_data, str): # Get a filesystem object if one isn't already available paths = [path_or_data] @@ -1866,38 +1672,28 @@ def get_reader_filepath_or_buffer( raise FileNotFoundError( f"{path_or_data} could not be resolved to any files" ) - if use_python_file_object: - path_or_data = _open_remote_files( - paths, - fs, - **(open_file_options or {}), - ) - else: - path_or_data = [ - BytesIO( - _fsspec_data_transfer( - fpath, - fs=fs, - mode=mode, - bytes_per_thread=bytes_per_thread, - ) + path_or_data = [ + BytesIO( + _fsspec_data_transfer( + fpath, + fs=fs, + mode=mode, + bytes_per_thread=bytes_per_thread, ) - for fpath in paths - ] + ) + for fpath in paths + ] if len(path_or_data) == 1: path_or_data = path_or_data[0] elif not isinstance(path_or_data, iotypes) and is_file_like(path_or_data): if isinstance(path_or_data, TextIOWrapper): path_or_data = path_or_data.buffer - if use_python_file_object: - path_or_data = ArrowPythonFile(path_or_data) - else: - path_or_data = BytesIO( - _fsspec_data_transfer( - path_or_data, mode=mode, bytes_per_thread=bytes_per_thread - ) + path_or_data = BytesIO( + _fsspec_data_transfer( + path_or_data, mode=mode, bytes_per_thread=bytes_per_thread ) + ) return path_or_data, compression diff --git a/python/cudf/cudf/utils/utils.py b/python/cudf/cudf/utils/utils.py index c9b343e0f9f..7347ec7866a 100644 --- a/python/cudf/cudf/utils/utils.py +++ b/python/cudf/cudf/utils/utils.py @@ -6,7 +6,6 @@ import os import traceback import warnings -from contextlib import contextmanager import numpy as np import pandas as pd @@ -404,28 +403,3 @@ def _all_bools_with_nulls(lhs, rhs, bool_fill_value): if result_mask is not None: result_col = result_col.set_mask(result_mask.as_mask()) return result_col - - -@contextmanager -def maybe_filter_deprecation( - condition: bool, message: str, category: type[Warning] -): - """Conditionally filter a warning category. - - Parameters - ---------- - condition - If true, filter the warning - message - Message to match, passed to :func:`warnings.filterwarnings` - category - Category of warning, passed to :func:`warnings.filterwarnings` - """ - with warnings.catch_warnings(): - if condition: - warnings.filterwarnings( - "ignore", - message, - category=category, - ) - yield diff --git a/python/dask_cudf/dask_cudf/io/tests/test_s3.py b/python/dask_cudf/dask_cudf/io/tests/test_s3.py index 99f19917424..a14ffbc37dc 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_s3.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_s3.py @@ -119,22 +119,6 @@ def test_read_csv(s3_base, s3so): assert df.a.sum().compute() == 4 -def test_read_csv_warns(s3_base, s3so): - with s3_context( - s3_base=s3_base, - bucket="daskcsv_warns", - files={"a.csv": b"a,b\n1,2\n3,4\n"}, - ): - with pytest.warns(FutureWarning): - df = dask_cudf.read_csv( - "s3://daskcsv_warns/*.csv", - blocksize="50 B", - storage_options=s3so, - use_python_file_object=True, - ) - assert df.a.sum().compute() == 4 - - def test_read_parquet_open_file_options_raises(): with pytest.raises(ValueError): dask_cudf.read_parquet( @@ -198,22 +182,6 @@ def test_read_parquet(s3_base, s3so, pdf): assert_eq(pdf, got) -def test_read_parquet_use_python_file_object(s3_base, s3so, pdf): - fname = "test_parquet_use_python_file_object.parquet" - bucket = "parquet" - buffer = BytesIO() - pdf.to_parquet(path=buffer) - buffer.seek(0) - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - with pytest.warns(FutureWarning): - got = dask_cudf.read_parquet( - f"s3://{bucket}/{fname}", - storage_options=s3so, - read={"use_python_file_object": True}, - ).head() - assert_eq(pdf, got) - - def test_read_orc(s3_base, s3so, pdf): fname = "test_orc_reader_dask.orc" bucket = "orc" @@ -226,19 +194,3 @@ def test_read_orc(s3_base, s3so, pdf): storage_options=s3so, ) assert_eq(pdf, got) - - -def test_read_orc_use_python_file_object(s3_base, s3so, pdf): - fname = "test_orc_use_python_file_object.orc" - bucket = "orc" - buffer = BytesIO() - pdf.to_orc(path=buffer) - buffer.seek(0) - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - with pytest.warns(FutureWarning): - got = dask_cudf.read_orc( - f"s3://{bucket}/{fname}", - storage_options=s3so, - use_python_file_object=True, - ).head() - assert_eq(pdf, got) diff --git a/python/pylibcudf/pylibcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/io/datasource.pxd index 05c03dceee2..c08f36693c7 100644 --- a/python/pylibcudf/pylibcudf/io/datasource.pxd +++ b/python/pylibcudf/pylibcudf/io/datasource.pxd @@ -1,14 +1,7 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. -from libcpp.memory cimport shared_ptr -from pylibcudf.libcudf.io.arrow_io_source cimport arrow_io_source from pylibcudf.libcudf.io.datasource cimport datasource cdef class Datasource: cdef datasource* get_datasource(self) except * nogil - - -cdef class NativeFileDatasource(Datasource): - cdef shared_ptr[arrow_io_source] c_datasource - cdef datasource* get_datasource(self) nogil diff --git a/python/pylibcudf/pylibcudf/io/datasource.pyx b/python/pylibcudf/pylibcudf/io/datasource.pyx index 6cc509b74cb..02418444caa 100644 --- a/python/pylibcudf/pylibcudf/io/datasource.pyx +++ b/python/pylibcudf/pylibcudf/io/datasource.pyx @@ -1,34 +1,10 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. -from libcpp.memory cimport shared_ptr -from pyarrow.includes.libarrow cimport CRandomAccessFile -from pyarrow.lib cimport NativeFile -from pylibcudf.libcudf.io.arrow_io_source cimport arrow_io_source from pylibcudf.libcudf.io.datasource cimport datasource -import warnings - cdef class Datasource: cdef datasource* get_datasource(self) except * nogil: with gil: raise NotImplementedError("get_datasource() should not " + "be directly invoked here") - -cdef class NativeFileDatasource(Datasource): - - def __cinit__(self, NativeFile native_file): - - cdef shared_ptr[CRandomAccessFile] ra_src - - warnings.warn( - "Support for reading pyarrow's NativeFile is deprecated " - "and will be removed in a future release of cudf.", - FutureWarning, - ) - - ra_src = native_file.get_random_access_file() - self.c_datasource.reset(new arrow_io_source(ra_src)) - - cdef datasource* get_datasource(self) nogil: - return (self.c_datasource.get()) From 6ccc2c2e4d7b4cda0bb4f844a28d69254049b795 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 16:39:34 -0500 Subject: [PATCH 089/270] standardize and consolidate wheel installations in testing scripts (#16575) I noticed some common changes to wheel-testing scripts in the PRs splitting off `pylibcudf` (#16299) and `libcudf` (#15483). * consolidating multiple `pip install`'s into 1 - *(this is safer, as it removes the risk of `pip` replacing a previously-installed CI package with another one from a remote package repository)* * standardizing the approach used for "install some wheels built earlier in this same CI run" These can go onto `branch-24.10` right now, so proposing them in a separate PR so that `cudf` CI can benefit from them without having to wait on those large PRs. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16575 --- ci/cudf_pandas_scripts/pandas-tests/run.sh | 13 +++++++++---- ci/cudf_pandas_scripts/run_tests.sh | 13 +++++++++---- ci/test_wheel_cudf.sh | 11 ++++++----- ci/test_wheel_cudf_polars.sh | 15 ++++++++++----- ci/test_wheel_dask_cudf.sh | 14 +++++++------- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 8deaeab78a3..97c3139080f 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -11,10 +11,15 @@ rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install $(ls ./local-pylibcudf-dep/pylibcudf*.whl) -python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] + +# Download the cudf and pylibcudf built in the previous step +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" \ + "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index bfb655db3ca..8215ce729b3 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -36,10 +36,15 @@ if [ "$no_cudf" = true ]; then echo "Skipping cudf install" else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep - RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep - python -m pip install $(ls ./local-pylibcudf-dep/pylibcudf*.whl) - python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests] + + # Download the cudf and pylibcudf built in the previous step + RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + + # echo to expand wildcard before adding `[extra]` requires for pip + python -m pip install \ + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" \ + "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" fi python -m pytest -p cudf.pandas \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 5a2c3ccac8f..19131952098 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -3,15 +3,16 @@ set -eou pipefail -# Download the pylibcudf built in the previous step RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep + +# Download the cudf and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# Install both pylibcudf and cudf +# echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - "$(echo ./local-pylibcudf-dep/pylibcudf*.whl)[test]" \ - "$(echo ./dist/cudf*.whl)[test]" + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 357d4170d47..6438d13c4b7 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -20,12 +20,17 @@ fi RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep -python -m pip install ./local-pylibcudf-dep/pylibcudf*.whl +# Download the cudf and pylibcudf built in the previous step +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -rapids-logger "Install cudf_polars" -python -m pip install $(echo ./dist/cudf_polars*.whl)[test] +rapids-logger "Installing cudf_polars and its dependencies" + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" rapids-logger "Run cudf_polars tests" diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 4d045472604..ff893a08e27 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -6,15 +6,15 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcudf-dep -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install \ - "$(echo ./local-pylibcudf-dep/pylibcudf*.whl)" \ - "$(echo ./local-cudf-dep/cudf*.whl)" +# Download the cudf and pylibcudf built in the previous step +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/dask_cudf*.whl)[test] +python -m pip install \ + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ From f2d13c9dbe957cd2a5cbf93a339149ab3edc0240 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 17:23:23 -0500 Subject: [PATCH 090/270] make more use of YAML anchors in dependencies.yaml (#16597) Contributes to https://github.com/rapidsai/build-planning/issues/33 Follow-up to #16299 This proposes some simplifications to `dependencies.yaml`. It's not intended to change any behavior. * more use of YAML anchors for requirements that are intended to be identical to each other * eliminating the `pylibcudf_build_dep` dependency group that was introduced in #16299, in favor of just tracking the `pylibcudf` build dependency alongside `cudf`'s `rmm` build dependency in the existing `build_python_cudf` group - *(sorry I'd missed that in the review on #16299)* I found myself starting to make similar changes in the PR breaking up these packages into more (splitting out a `libcudf` in #15483) and thought they'd be better as a standalone PR. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16597 --- dependencies.yaml | 45 ++++++++++++++------------------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index ca615905a15..a774345fe95 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -96,7 +96,6 @@ files: - build_base - build_python_common - build_python_cudf - - pylibcudf_build_dep py_run_cudf: output: pyproject pyproject_dir: python/cudf @@ -383,12 +382,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - rmm-cu12==24.10.*,>=0.0.0a0 + - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - rmm-cu11==24.10.*,>=0.0.0a0 + - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_unsuffixed]} build_python_cudf: common: @@ -412,34 +411,18 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - rmm-cu12==24.10.*,>=0.0.0a0 - - pylibcudf-cu12==24.10.*,>=0.0.0a0 + - &pylibcudf_cu12 pylibcudf-cu12==24.10.*,>=0.0.0a0 + - *rmm_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - rmm-cu11==24.10.*,>=0.0.0a0 - - pylibcudf-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_unsuffixed]} - pylibcudf_build_dep: - common: - - output_types: conda - packages: - - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0 - specific: - - output_types: [pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - pylibcudf-cu12==24.10.*,>=0.0.0a0 + - &pylibcudf_cu11 pylibcudf-cu11==24.10.*,>=0.0.0a0 + - *rmm_cu11 - matrix: - cuda: "11.*" - cuda_suffixed: "true" packages: - - pylibcudf-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*pylibcudf_unsuffixed]} + - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0 + - *rmm_unsuffixed libarrow_build: common: - output_types: conda @@ -677,12 +660,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - rmm-cu12==24.10.*,>=0.0.0a0 + - *rmm_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - rmm-cu11==24.10.*,>=0.0.0a0 + - *rmm_cu11 - {matrix: null, packages: [*rmm_unsuffixed]} run_cudf: common: @@ -728,7 +711,7 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - rmm-cu12==24.10.*,>=0.0.0a0 + - *rmm_cu12 - pynvjitlink-cu12>=0.0.0a0 - matrix: cuda: "12.*" @@ -740,7 +723,7 @@ dependencies: cuda: "11.*" cuda_suffixed: "true" packages: - - rmm-cu11==24.10.*,>=0.0.0a0 + - *rmm_cu11 - cubinlinker-cu11 - ptxcompiler-cu11 - matrix: @@ -874,12 +857,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - pylibcudf-cu12==24.10.*,>=0.0.0a0 + - *pylibcudf_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - pylibcudf-cu11==24.10.*,>=0.0.0a0 + - *pylibcudf_cu11 - {matrix: null, packages: [*pylibcudf_unsuffixed]} depends_on_cudf: common: From 3f6dd14e26deccc761ed06790cf806edc266d5e4 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 19 Aug 2024 12:29:15 -1000 Subject: [PATCH 091/270] Make StructColumn.__init__ strict (#16467) This PR makes `StructColumn.__init__` strict putting restrictions on data, dtype, size and children so these columns cannot be constructed into to an invalid state. It also aligns the signature with the base class. xref https://github.com/rapidsai/cudf/issues/16469 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16467 --- python/cudf/cudf/core/column/column.py | 13 +++-- python/cudf/cudf/core/column/interval.py | 71 ++++++++++++++++-------- python/cudf/cudf/core/column/struct.py | 50 ++++++++++++++--- python/cudf/cudf/core/index.py | 6 +- 4 files changed, 100 insertions(+), 40 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 0857727d23f..27278120abb 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1635,22 +1635,23 @@ def build_column( ) elif isinstance(dtype, IntervalDtype): return cudf.core.column.IntervalColumn( + data=None, + size=size, # type: ignore[arg-type] dtype=dtype, mask=mask, - size=size, offset=offset, - children=children, null_count=null_count, + children=children, # type: ignore[arg-type] ) elif isinstance(dtype, StructDtype): return cudf.core.column.StructColumn( - data=data, - dtype=dtype, + data=None, size=size, # type: ignore[arg-type] - offset=offset, + dtype=dtype, mask=mask, + offset=offset, null_count=null_count, - children=children, + children=children, # type: ignore[arg-type] ) elif isinstance(dtype, cudf.Decimal64Dtype): return cudf.core.column.Decimal64Column( diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index d9fc96a9f3e..9147270c289 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -11,32 +11,46 @@ from cudf.core.dtypes import IntervalDtype if TYPE_CHECKING: + from typing_extensions import Self + from cudf._typing import ScalarLike + from cudf.core.buffer import Buffer from cudf.core.column import ColumnBase class IntervalColumn(StructColumn): def __init__( self, - dtype, - mask=None, - size=None, - offset=0, - null_count=None, - children=(), + data: None, + size: int, + dtype: IntervalDtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple[ColumnBase, ColumnBase] = (), # type: ignore[assignment] ): + if len(children) != 2: + raise ValueError( + "children must be a tuple of two columns (left edges, right edges)." + ) super().__init__( - data=None, + data=data, + size=size, dtype=dtype, mask=mask, - size=size, offset=offset, null_count=null_count, children=children, ) + @staticmethod + def _validate_dtype_instance(dtype: IntervalDtype) -> IntervalDtype: + if not isinstance(dtype, IntervalDtype): + raise ValueError("dtype must be a IntervalDtype.") + return dtype + @classmethod - def from_arrow(cls, data): + def from_arrow(cls, data: pa.Array) -> Self: new_col = super().from_arrow(data.storage) size = len(data) dtype = IntervalDtype.from_arrow(data.type) @@ -48,16 +62,17 @@ def from_arrow(cls, data): null_count = data.null_count children = new_col.children - return IntervalColumn( + return cls( + data=None, size=size, dtype=dtype, mask=mask, offset=offset, null_count=null_count, - children=children, + children=children, # type: ignore[arg-type] ) - def to_arrow(self): + def to_arrow(self) -> pa.Array: typ = self.dtype.to_arrow() struct_arrow = super().to_arrow() if len(struct_arrow) == 0: @@ -67,9 +82,14 @@ def to_arrow(self): return pa.ExtensionArray.from_storage(typ, struct_arrow) @classmethod - def from_struct_column(cls, struct_column: StructColumn, closed="right"): + def from_struct_column( + cls, + struct_column: StructColumn, + closed: Literal["left", "right", "both", "neither"] = "right", + ) -> Self: first_field_name = next(iter(struct_column.dtype.fields.keys())) - return IntervalColumn( + return cls( + data=None, size=struct_column.size, dtype=IntervalDtype( struct_column.dtype.fields[first_field_name], closed @@ -77,12 +97,13 @@ def from_struct_column(cls, struct_column: StructColumn, closed="right"): mask=struct_column.base_mask, offset=struct_column.offset, null_count=struct_column.null_count, - children=struct_column.base_children, + children=struct_column.base_children, # type: ignore[arg-type] ) - def copy(self, deep=True): + def copy(self, deep: bool = True) -> Self: struct_copy = super().copy(deep=deep) - return IntervalColumn( + return IntervalColumn( # type: ignore[return-value] + data=None, size=struct_copy.size, dtype=IntervalDtype( struct_copy.dtype.fields["left"], self.dtype.closed @@ -90,7 +111,7 @@ def copy(self, deep=True): mask=struct_copy.base_mask, offset=struct_copy.offset, null_count=struct_copy.null_count, - children=struct_copy.base_children, + children=struct_copy.base_children, # type: ignore[arg-type] ) @property @@ -138,25 +159,27 @@ def overlaps(other) -> ColumnBase: def set_closed( self, closed: Literal["left", "right", "both", "neither"] - ) -> IntervalColumn: - return IntervalColumn( + ) -> Self: + return IntervalColumn( # type: ignore[return-value] + data=None, size=self.size, dtype=IntervalDtype(self.dtype.fields["left"], closed), mask=self.base_mask, offset=self.offset, null_count=self.null_count, - children=self.base_children, + children=self.base_children, # type: ignore[arg-type] ) - def as_interval_column(self, dtype): + def as_interval_column(self, dtype: IntervalDtype) -> Self: # type: ignore[override] if isinstance(dtype, IntervalDtype): - return IntervalColumn( + return IntervalColumn( # type: ignore[return-value] + data=None, size=self.size, dtype=dtype, mask=self.mask, offset=self.offset, null_count=self.null_count, - children=tuple( + children=tuple( # type: ignore[arg-type] child.astype(dtype.subtype) for child in self.children ), ) diff --git a/python/cudf/cudf/core/column/struct.py b/python/cudf/cudf/core/column/struct.py index c2ce787eeae..2fda3b2c434 100644 --- a/python/cudf/cudf/core/column/struct.py +++ b/python/cudf/cudf/core/column/struct.py @@ -14,7 +14,10 @@ from cudf.core.missing import NA if TYPE_CHECKING: + from typing_extensions import Self + from cudf._typing import Dtype + from cudf.core.buffer import Buffer class StructColumn(ColumnBase): @@ -23,10 +26,39 @@ class StructColumn(ColumnBase): Every column has n children, where n is the number of fields in the Struct Dtype. - """ - dtype: StructDtype + def __init__( + self, + data: None, + size: int, + dtype: StructDtype, + mask: Buffer | None = None, + offset: int = 0, + null_count: int | None = None, + children: tuple[ColumnBase, ...] = (), + ): + if data is not None: + raise ValueError("data must be None.") + dtype = self._validate_dtype_instance(dtype) + super().__init__( + data=data, + size=size, + dtype=dtype, + mask=mask, + offset=offset, + null_count=null_count, + children=children, + ) + + @staticmethod + def _validate_dtype_instance(dtype: StructDtype) -> StructDtype: + # IntervalDtype is a subclass of StructDtype, so compare types exactly + if type(dtype) is not StructDtype: + raise ValueError( + f"{type(dtype).__name__} must be a StructDtype exactly." + ) + return dtype @property def base_size(self): @@ -35,7 +67,7 @@ def base_size(self): else: return self.size + self.offset - def to_arrow(self): + def to_arrow(self) -> pa.Array: children = [ pa.nulls(len(child)) if len(child) == child.null_count @@ -50,7 +82,7 @@ def to_arrow(self): } ) - if self.nullable: + if self.mask is not None: buffers = (pa.py_buffer(self.mask.memoryview()),) else: buffers = (None,) @@ -73,7 +105,7 @@ def to_pandas( return pd.Index(self.to_arrow().tolist(), dtype="object") @cached_property - def memory_usage(self): + def memory_usage(self) -> int: n = 0 if self.nullable: n += cudf._lib.null_mask.bitmask_allocation_size_bytes(self.size) @@ -99,7 +131,7 @@ def __setitem__(self, key, value): value = cudf.Scalar(value, self.dtype) super().__setitem__(key, value) - def copy(self, deep=True): + def copy(self, deep: bool = True) -> Self: # Since struct columns are immutable, both deep and # shallow copies share the underlying device data and mask. result = super().copy(deep=False) @@ -107,15 +139,15 @@ def copy(self, deep=True): result = result._rename_fields(self.dtype.fields.keys()) return result - def _rename_fields(self, names): + def _rename_fields(self, names) -> Self: """ Return a StructColumn with the same field values as this StructColumn, but with the field names equal to `names`. """ - dtype = cudf.core.dtypes.StructDtype( + dtype = StructDtype( {name: col.dtype for name, col in zip(names, self.children)} ) - return StructColumn( + return StructColumn( # type: ignore[return-value] data=None, size=self.size, dtype=dtype, diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index ee2f0317f8d..6a5e718c2c5 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3354,6 +3354,7 @@ def interval_range( return IntervalIndex(data, closed=closed, name=name) interval_col = IntervalColumn( + data=None, dtype=IntervalDtype(left_col.dtype, closed), size=len(left_col), children=(left_col, right_col), @@ -3425,6 +3426,7 @@ def __init__( elif isinstance(data.dtype, (pd.IntervalDtype, IntervalDtype)): data = np.array([], dtype=data.dtype.subtype) interval_col = IntervalColumn( + None, dtype=IntervalDtype(data.dtype, closed), size=len(data), children=(as_column(data), as_column(data)), @@ -3436,12 +3438,13 @@ def __init__( if copy: col = col.copy() interval_col = IntervalColumn( + data=None, dtype=IntervalDtype(col.dtype.subtype, closed), mask=col.mask, size=col.size, offset=col.offset, null_count=col.null_count, - children=col.children, + children=col.children, # type: ignore[arg-type] ) if dtype: @@ -3517,6 +3520,7 @@ def from_breaks( ) interval_col = IntervalColumn( + data=None, dtype=IntervalDtype(left_col.dtype, closed), size=len(left_col), children=(left_col, right_col), From a45af4a61ba582d6af839702148e9a6e2da69bc9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 19 Aug 2024 19:06:28 -0700 Subject: [PATCH 092/270] Remove arrow_io_source (#16607) The `arrow_io_source` in libcudf only existed to support Python's pyarrow NativeFile integration, which was deprecated and removed in #16589. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Thomas Li (https://github.com/lithomas1) - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/16607 --- cpp/CMakeLists.txt | 1 - cpp/include/cudf/io/arrow_io_source.hpp | 93 ---------------- cpp/src/io/utilities/arrow_io_source.cpp | 87 --------------- cpp/tests/CMakeLists.txt | 4 - cpp/tests/io/arrow_io_source_test.cpp | 103 ------------------ cpp/tests/io/csv_test.cpp | 26 +---- cpp/tests/io/json/json_test.cpp | 26 ----- .../pylibcudf/libcudf/io/arrow_io_source.pxd | 14 --- 8 files changed, 1 insertion(+), 353 deletions(-) delete mode 100644 cpp/include/cudf/io/arrow_io_source.hpp delete mode 100644 cpp/src/io/utilities/arrow_io_source.cpp delete mode 100644 cpp/tests/io/arrow_io_source_test.cpp delete mode 100644 python/pylibcudf/pylibcudf/libcudf/io/arrow_io_source.pxd diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index eeafc411874..ff00c484501 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -438,7 +438,6 @@ add_library( src/io/text/bgzip_data_chunk_source.cu src/io/text/bgzip_utils.cpp src/io/text/multibyte_split.cu - src/io/utilities/arrow_io_source.cpp src/io/utilities/base64_utilities.cpp src/io/utilities/column_buffer.cpp src/io/utilities/column_buffer_strings.cu diff --git a/cpp/include/cudf/io/arrow_io_source.hpp b/cpp/include/cudf/io/arrow_io_source.hpp deleted file mode 100644 index ed5c839cbb4..00000000000 --- a/cpp/include/cudf/io/arrow_io_source.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "datasource.hpp" - -#include - -#include -#include - -#include -#include -#include - -namespace CUDF_EXPORT cudf { -namespace io { -/** - * @addtogroup io_datasources - * @{ - * @file - */ - -/** - * @brief Implementation class for reading from an Apache Arrow file. The file - * could be a memory-mapped file or other implementation supported by Arrow. - */ -class arrow_io_source : public datasource { - public: - /** - * @brief Constructs an object from an Apache Arrow Filesystem URI - * - * @param arrow_uri Apache Arrow Filesystem URI - */ - explicit arrow_io_source(std::string const& arrow_uri); - - /** - * @brief Constructs an object from an `arrow` source object. - * - * @param file The `arrow` object from which the data is read - */ - explicit arrow_io_source(std::shared_ptr file) - : arrow_file(std::move(file)) - { - } - - /** - * @brief Returns a buffer with a subset of data from the `arrow` source. - * - * @param offset The offset in bytes from which to read - * @param size The number of bytes to read - * @return A buffer with the read data - */ - std::unique_ptr host_read(size_t offset, size_t size) override; - - /** - * @brief Reads a selected range from the `arrow` source into a preallocated buffer. - * - * @param[in] offset The offset in bytes from which to read - * @param[in] size The number of bytes to read - * @param[out] dst The preallocated buffer to read into - * @return The number of bytes read - */ - size_t host_read(size_t offset, size_t size, uint8_t* dst) override; - /** - * @brief Returns the size of the data in the `arrow` source. - * - * @return The size of the data in the `arrow` source - */ - [[nodiscard]] size_t size() const override; - - private: - std::shared_ptr filesystem; - std::shared_ptr arrow_file; -}; - -/** @} */ // end of group -} // namespace io -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/io/utilities/arrow_io_source.cpp b/cpp/src/io/utilities/arrow_io_source.cpp deleted file mode 100644 index 157240b8b08..00000000000 --- a/cpp/src/io/utilities/arrow_io_source.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -#include -#include -#include - -namespace cudf::io { - -/** - * @brief Implementation for an owning buffer where `arrow::Buffer` holds the data. - */ -class arrow_io_buffer : public datasource::buffer { - std::shared_ptr arrow_buffer; - - public: - explicit arrow_io_buffer(std::shared_ptr arrow_buffer) - : arrow_buffer(std::move(arrow_buffer)) - { - } - [[nodiscard]] size_t size() const override { return arrow_buffer->size(); } - [[nodiscard]] uint8_t const* data() const override { return arrow_buffer->data(); } -}; - -arrow_io_source::arrow_io_source(std::string const& arrow_uri) -{ - std::string const uri_start_delimiter = "//"; - std::string const uri_end_delimiter = "?"; - - auto const result = arrow::fs::FileSystemFromUri(arrow_uri); - CUDF_EXPECTS(result.ok(), "Failed to generate Arrow Filesystem instance from URI."); - filesystem = result.ValueOrDie(); - - // Parse the path from the URI - auto const start = [&]() { - auto const delim_start = arrow_uri.find(uri_start_delimiter); - return delim_start == std::string::npos ? 0 : delim_start + uri_start_delimiter.size(); - }(); - auto const end = arrow_uri.find(uri_end_delimiter) - start; - auto const path = arrow_uri.substr(start, end); - - auto const in_stream = filesystem->OpenInputFile(path); - CUDF_EXPECTS(in_stream.ok(), "Failed to open Arrow RandomAccessFile"); - arrow_file = in_stream.ValueOrDie(); -} - -std::unique_ptr arrow_io_source::host_read(size_t offset, size_t size) -{ - auto const result = arrow_file->ReadAt(offset, size); - CUDF_EXPECTS(result.ok(), "Cannot read file data"); - return std::make_unique(result.ValueOrDie()); -} - -size_t arrow_io_source::host_read(size_t offset, size_t size, uint8_t* dst) -{ - auto const result = arrow_file->ReadAt(offset, size, dst); - CUDF_EXPECTS(result.ok(), "Cannot read file data"); - return result.ValueOrDie(); -} - -[[nodiscard]] size_t arrow_io_source::size() const -{ - auto const result = arrow_file->GetSize(); - CUDF_EXPECTS(result.ok(), "Cannot get file size"); - return result.ValueOrDie(); -} - -} // namespace cudf::io diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 006b36add0e..ac77a362e1c 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -321,7 +321,6 @@ ConfigureTest( ConfigureTest(JSON_WRITER_TEST io/json/json_writer.cpp) ConfigureTest(JSON_TYPE_CAST_TEST io/json/json_type_cast_test.cu) ConfigureTest(NESTED_JSON_TEST io/json/nested_json_test.cpp io/json/json_tree.cpp) -ConfigureTest(ARROW_IO_SOURCE_TEST io/arrow_io_source_test.cpp) ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp) ConfigureTest(JSON_QUOTE_NORMALIZATION io/json/json_quote_normalization_test.cpp) ConfigureTest(JSON_WHITESPACE_NORMALIZATION io/json/json_whitespace_normalization_test.cu) @@ -334,9 +333,6 @@ target_link_libraries(DATA_CHUNK_SOURCE_TEST PRIVATE ZLIB::ZLIB) ConfigureTest(LOGICAL_STACK_TEST io/fst/logical_stack_test.cu) ConfigureTest(FST_TEST io/fst/fst_test.cu) ConfigureTest(TYPE_INFERENCE_TEST io/type_inference_test.cu) -if(CUDF_ENABLE_ARROW_S3) - target_compile_definitions(ARROW_IO_SOURCE_TEST PRIVATE "S3_ENABLED") -endif() # ################################################################################################## # * sort tests ------------------------------------------------------------------------------------ diff --git a/cpp/tests/io/arrow_io_source_test.cpp b/cpp/tests/io/arrow_io_source_test.cpp deleted file mode 100644 index ffdf2c7e00f..00000000000 --- a/cpp/tests/io/arrow_io_source_test.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -// Global environment for temporary files -auto const temp_env = static_cast( - ::testing::AddGlobalTestEnvironment(new cudf::test::TempDirTestEnvironment)); - -// Base test fixture for tests -struct ArrowIOTest : public cudf::test::BaseFixture {}; - -TEST_F(ArrowIOTest, URIFileSystem) -{ - const std::string file_name = temp_env->get_temp_dir() + "JsonLinesFileTest.json"; - std::ofstream outfile(file_name, std::ofstream::out); - outfile << "{\"a\":11, \"b\":1.1}\n{\"a\":22, \"b\":2.2}"; - outfile.close(); - - std::string file_uri = "file://" + file_name; - auto datasource = std::make_unique(file_uri); - - // Populate the JSON Reader Options - cudf::io::json_reader_options options = - cudf::io::json_reader_options::builder(cudf::io::source_info(datasource.get())).lines(true); - - // Read the JSON file from the LocalFileSystem - cudf::io::table_with_metadata tbl = cudf::io::read_json(options); - - ASSERT_EQ(2, tbl.tbl->num_columns()); - ASSERT_EQ(2, tbl.tbl->num_rows()); -} - -TEST_F(ArrowIOTest, S3FileSystem) -{ - std::string s3_uri = "s3://rapidsai-data/cudf/test/tips.parquet?region=us-east-2"; - - // Check to see if Arrow was built with support for S3. If not, ensure this - // test throws. If so, validate the S3 file contents. - auto const s3_unsupported = arrow::fs::FileSystemFromUri(s3_uri).status().IsNotImplemented(); - if (s3_unsupported) { - EXPECT_THROW(std::make_unique(s3_uri), cudf::logic_error); - } else { - auto datasource = std::make_unique(s3_uri); - - // Populate the Parquet Reader Options - cudf::io::source_info src(datasource.get()); - std::vector single_column; - single_column.insert(single_column.begin(), "total_bill"); - cudf::io::parquet_reader_options_builder builder(src); - cudf::io::parquet_reader_options options = builder.columns(single_column).build(); - - // Read the Parquet file from S3 - cudf::io::table_with_metadata tbl = cudf::io::read_parquet(options); - - ASSERT_EQ(1, tbl.tbl->num_columns()); // Only single column specified in reader_options - ASSERT_EQ(244, tbl.tbl->num_rows()); // known number of rows from the S3 file - } - -#ifdef ARROW_S3 - if (!s3_unsupported) { - // Verify that we are using Arrow with S3, and call finalize - // https://github.com/apache/arrow/issues/36974 - // This needs to be in a separate conditional to ensure we call - // finalize after all arrow_io_source instances have been deleted. - [[maybe_unused]] auto _ = arrow::fs::EnsureS3Finalized(); - } -#endif -} - -CUDF_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/io/csv_test.cpp b/cpp/tests/io/csv_test.cpp index ff433264446..dc14824d834 100644 --- a/cpp/tests/io/csv_test.cpp +++ b/cpp/tests/io/csv_test.cpp @@ -25,8 +25,8 @@ #include #include -#include #include +#include #include #include #include @@ -1197,30 +1197,6 @@ TEST_F(CsvReaderTest, HeaderOnlyFile) EXPECT_EQ(3, view.num_columns()); } -TEST_F(CsvReaderTest, ArrowFileSource) -{ - auto filepath = temp_env->get_temp_dir() + "ArrowFileSource.csv"; - { - std::ofstream outfile(filepath, std::ofstream::out); - outfile << "A\n9\n8\n7\n6\n5\n4\n3\n2\n"; - } - - std::shared_ptr infile; - ASSERT_TRUE(arrow::io::ReadableFile::Open(filepath).Value(&infile).ok()); - - auto arrow_source = cudf::io::arrow_io_source{infile}; - cudf::io::csv_reader_options in_opts = - cudf::io::csv_reader_options::builder(cudf::io::source_info{&arrow_source}) - .dtypes({dtype()}); - auto result = cudf::io::read_csv(in_opts); - - auto const view = result.tbl->view(); - EXPECT_EQ(1, view.num_columns()); - ASSERT_EQ(type_id::INT8, view.column(0).type().id()); - - expect_column_data_equal(std::vector{9, 8, 7, 6, 5, 4, 3, 2}, view.column(0)); -} - TEST_F(CsvReaderTest, InvalidFloatingPoint) { auto const filepath = temp_env->get_temp_dir() + "InvalidFloatingPoint.csv"; diff --git a/cpp/tests/io/json/json_test.cpp b/cpp/tests/io/json/json_test.cpp index 0a485e26b71..576a698ba31 100644 --- a/cpp/tests/io/json/json_test.cpp +++ b/cpp/tests/io/json/json_test.cpp @@ -26,7 +26,6 @@ #include #include -#include #include #include #include @@ -958,31 +957,6 @@ TEST_F(JsonReaderTest, NoDataFileValues) EXPECT_EQ(0, view.num_columns()); } -TEST_F(JsonReaderTest, ArrowFileSource) -{ - const std::string fname = temp_env->get_temp_dir() + "ArrowFileSource.csv"; - - std::ofstream outfile(fname, std::ofstream::out); - outfile << "[9]\n[8]\n[7]\n[6]\n[5]\n[4]\n[3]\n[2]\n"; - outfile.close(); - - std::shared_ptr infile; - ASSERT_TRUE(arrow::io::ReadableFile::Open(fname).Value(&infile).ok()); - - auto arrow_source = cudf::io::arrow_io_source{infile}; - cudf::io::json_reader_options in_options = - cudf::io::json_reader_options::builder(cudf::io::source_info{&arrow_source}) - .dtypes({dtype()}) - .lines(true); - - cudf::io::table_with_metadata result = cudf::io::read_json(in_options); - - EXPECT_EQ(result.tbl->num_columns(), 1); - EXPECT_EQ(result.tbl->get_column(0).type().id(), cudf::type_id::INT8); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->get_column(0), int8_wrapper{{9, 8, 7, 6, 5, 4, 3, 2}}); -} - TEST_P(JsonReaderParamTest, InvalidFloatingPoint) { auto const test_opt = GetParam(); diff --git a/python/pylibcudf/pylibcudf/libcudf/io/arrow_io_source.pxd b/python/pylibcudf/pylibcudf/libcudf/io/arrow_io_source.pxd deleted file mode 100644 index 54a913a9ce3..00000000000 --- a/python/pylibcudf/pylibcudf/libcudf/io/arrow_io_source.pxd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -cimport pylibcudf.libcudf.io.datasource as cudf_io_datasource -from libcpp.memory cimport shared_ptr -from libcpp.string cimport string -from pyarrow.includes.libarrow cimport CRandomAccessFile - - -cdef extern from "cudf/io/arrow_io_source.hpp" \ - namespace "cudf::io" nogil: - - cdef cppclass arrow_io_source(cudf_io_datasource.datasource): - arrow_io_source(const string& arrow_uri) except + - arrow_io_source(shared_ptr[CRandomAccessFile]) except + From 3ac409dc26437deb77d30f64ec148121394878e4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 19 Aug 2024 21:18:11 -0700 Subject: [PATCH 093/270] Fix C++ and Cython io types (#16610) The C++ I/O types were previously not specifying a base type despite the fact that the Cython code was relying on the base being an int32. This has apparently never bitten us before, but in theory this could go very wrong since it leaves the underlying type up to the compiler and if the C++ binary used something other than an int32 that would result in an ABI incompatibility with the Python build that would produce spurious results. While fixing this, I also noticed that the Cython contained a number of erroneous (likely outdated) declarations. Since Cython extern declarations are simply an indicate to Cython of how to resolve a function call _if_ it appears in compiled Cython code, these were not causing any build failures because these were all unused APIs, so I removed them from the Cython with no further changes needed. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16610 --- cpp/include/cudf/io/types.hpp | 12 ++--- .../pylibcudf/pylibcudf/libcudf/io/types.pxd | 50 ++++++++----------- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index 3df737413fa..a34881942ce 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -54,7 +54,7 @@ namespace io { /** * @brief Compression algorithms */ -enum class compression_type { +enum class compression_type : int32_t { NONE, ///< No compression AUTO, ///< Automatically detect or select compression format SNAPPY, ///< Snappy format, using byte-oriented LZ77 @@ -72,7 +72,7 @@ enum class compression_type { /** * @brief Data source or destination types */ -enum class io_type { +enum class io_type : int32_t { FILEPATH, ///< Input/output is a file path HOST_BUFFER, ///< Input/output is a buffer in host memory DEVICE_BUFFER, ///< Input/output is a buffer in device memory @@ -83,7 +83,7 @@ enum class io_type { /** * @brief Behavior when handling quotations in field data */ -enum class quote_style { +enum class quote_style : int32_t { MINIMAL, ///< Quote only fields which contain special characters ALL, ///< Quote all fields NONNUMERIC, ///< Quote all non-numeric fields @@ -93,7 +93,7 @@ enum class quote_style { /** * @brief Column statistics granularity type for parquet/orc writers */ -enum statistics_freq { +enum statistics_freq : int32_t { STATISTICS_NONE = 0, ///< No column statistics STATISTICS_ROWGROUP = 1, ///< Per-Rowgroup column statistics STATISTICS_PAGE = 2, ///< Per-page column statistics @@ -103,7 +103,7 @@ enum statistics_freq { /** * @brief Valid encodings for use with `column_in_metadata::set_encoding()` */ -enum class column_encoding { +enum class column_encoding : int32_t { // Common encodings: USE_DEFAULT = -1, ///< No encoding has been requested, use default encoding DICTIONARY, ///< Use dictionary encoding @@ -222,7 +222,7 @@ class writer_compression_statistics { /** * @brief Control use of dictionary encoding for parquet writer */ -enum dictionary_policy { +enum dictionary_policy : int32_t { NEVER = 0, ///< Never use dictionary encoding ADAPTIVE = 1, ///< Use dictionary when it will not impact compression ALWAYS = 2 ///< Use dictionary regardless of impact on compression diff --git a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd index a3d99807876..5f3be2f0727 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd @@ -6,12 +6,10 @@ cimport pylibcudf.libcudf.table.table_view as cudf_table_view from libc.stdint cimport int32_t, uint8_t from libcpp cimport bool from libcpp.map cimport map -from libcpp.memory cimport shared_ptr, unique_ptr -from libcpp.pair cimport pair +from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.unordered_map cimport unordered_map from libcpp.vector cimport vector -from pyarrow.includes.libarrow cimport CRandomAccessFile from pylibcudf.libcudf.table.table cimport table from pylibcudf.libcudf.types cimport size_type @@ -42,32 +40,32 @@ cdef extern from "cudf/io/types.hpp" \ cpdef enum class io_type(int32_t): FILEPATH HOST_BUFFER + DEVICE_BUFFER VOID USER_IMPLEMENTED cpdef enum class statistics_freq(int32_t): - STATISTICS_NONE = 0, - STATISTICS_ROWGROUP = 1, - STATISTICS_PAGE = 2, - STATISTICS_COLUMN = 3, + STATISTICS_NONE, + STATISTICS_ROWGROUP, + STATISTICS_PAGE, + STATISTICS_COLUMN, cpdef enum class dictionary_policy(int32_t): - NEVER = 0, - ADAPTIVE = 1, - ALWAYS = 2, - - cdef extern from "cudf/io/types.hpp" namespace "cudf::io" nogil: - cpdef enum class column_encoding(int32_t): - USE_DEFAULT = -1 - DICTIONARY = 0 - PLAIN = 1 - DELTA_BINARY_PACKED = 2 - DELTA_LENGTH_BYTE_ARRAY =3 - DELTA_BYTE_ARRAY = 4 - BYTE_STREAM_SPLIT = 5 - DIRECT = 6 - DIRECT_V2 = 7 - DICTIONARY_V2 = 8 + NEVER, + ADAPTIVE, + ALWAYS, + + cpdef enum class column_encoding(int32_t): + USE_DEFAULT + DICTIONARY + PLAIN + DELTA_BINARY_PACKED + DELTA_LENGTH_BYTE_ARRAY + DELTA_BYTE_ARRAY + BYTE_STREAM_SPLIT + DIRECT + DIRECT_V2 + DICTIONARY_V2 cdef cppclass column_name_info: string name @@ -76,7 +74,6 @@ cdef extern from "cudf/io/types.hpp" \ cdef cppclass table_metadata: table_metadata() except + - vector[string] column_names map[string, string] user_data vector[unordered_map[string, string]] per_file_user_data vector[column_name_info] schema_info @@ -120,10 +117,7 @@ cdef extern from "cudf/io/types.hpp" \ host_buffer(const char* data, size_t size) cdef cppclass source_info: - io_type type const vector[string]& filepaths() except + - const vector[host_buffer]& buffers() except + - vector[shared_ptr[CRandomAccessFile]] files source_info() except + source_info(const vector[string] &filepaths) except + @@ -132,9 +126,7 @@ cdef extern from "cudf/io/types.hpp" \ source_info(const vector[cudf_io_datasource.datasource*] &datasources) except + cdef cppclass sink_info: - io_type type const vector[string]& filepaths() - const vector[vector[char] *]& buffers() const vector[cudf_io_data_sink.data_sink *]& user_sinks() sink_info() except + From 2f7d35435db2b5ed9ead96cf43e2a710db5e5e6d Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 20 Aug 2024 03:52:34 -0500 Subject: [PATCH 094/270] bug-fix: cudf/io/json.hpp use after move (#16609) This PR fixes a use after move in json header. The fix simply shifts the attributes to access the object value before moving it. Closes https://github.com/rapidsai/cudf/issues/16608 Authors: - Nicolas (https://github.com/NicolasDenoyelle) Approvers: - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/16609 --- cpp/include/cudf/io/json.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/include/cudf/io/json.hpp b/cpp/include/cudf/io/json.hpp index 0cb39d15cd5..fde1857cb7f 100644 --- a/cpp/include/cudf/io/json.hpp +++ b/cpp/include/cudf/io/json.hpp @@ -696,6 +696,8 @@ class json_writer_options_builder; class json_writer_options { // Specify the sink to use for writer output sink_info _sink; + // maximum number of rows to write in each chunk (limits memory use) + size_type _rows_per_chunk = std::numeric_limits::max(); // Set of columns to output table_view _table; // string to use for null entries @@ -704,8 +706,6 @@ class json_writer_options { bool _include_nulls = false; // Indicates whether to use JSON lines for records format bool _lines = false; - // maximum number of rows to write in each chunk (limits memory use) - size_type _rows_per_chunk = std::numeric_limits::max(); // string to use for values != 0 in INT8 types (default 'true') std::string _true_value = std::string{"true"}; // string to use for values == 0 in INT8 types (default 'false') @@ -720,7 +720,7 @@ class json_writer_options { * @param table Table to be written to output */ explicit json_writer_options(sink_info sink, table_view table) - : _sink(std::move(sink)), _table(std::move(table)), _rows_per_chunk(table.num_rows()) + : _sink(std::move(sink)), _rows_per_chunk(table.num_rows()), _table(std::move(table)) { } From 1cccf3eeaee50cd69107b3c54ee349720233d8c6 Mon Sep 17 00:00:00 2001 From: Michael Schellenberger Costa Date: Tue, 20 Aug 2024 16:32:29 +0200 Subject: [PATCH 095/270] Replace usages of `thrust::optional` with `std::optional` (#15091) We want to get rid of thrust types in API boundaries so replace them by the better suited std types Authors: - Michael Schellenberger Costa (https://github.com/miscco) - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - https://github.com/nvdbaranec - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/15091 --- .../cudf/column/column_device_view.cuh | 28 ++++----- cpp/include/cudf/detail/copy_if_else.cuh | 6 +- cpp/include/cudf/detail/indexalator.cuh | 12 ++-- cpp/include/cudf/detail/iterator.cuh | 26 ++++---- cpp/include/cudf/json/json.hpp | 2 - .../strings/detail/convert/fixed_point.cuh | 8 +-- .../cudf/strings/detail/copy_if_else.cuh | 6 +- .../cudf/table/experimental/row_operators.cuh | 6 +- cpp/src/binaryop/binaryop.cpp | 4 +- cpp/src/io/orc/orc.hpp | 7 ++- cpp/src/io/orc/writer_impl.cu | 6 +- .../io/parquet/compact_protocol_reader.cpp | 8 +-- cpp/src/io/parquet/parquet.hpp | 62 +++++++++---------- cpp/src/io/parquet/parquet_gpu.hpp | 14 ++--- cpp/src/io/parquet/predicate_pushdown.cpp | 6 +- cpp/src/io/parquet/reader_impl.cpp | 2 +- cpp/src/io/parquet/reader_impl_chunking.cu | 6 +- cpp/src/io/parquet/reader_impl_helpers.cpp | 6 +- cpp/src/io/parquet/writer_impl.cu | 8 +-- cpp/src/json/json_path.cu | 22 +++---- cpp/src/lists/contains.cu | 1 - cpp/src/lists/explode.cu | 14 ++--- cpp/src/strings/convert/convert_datetime.cu | 10 +-- cpp/src/strings/regex/regex.cuh | 4 +- cpp/src/strings/regex/regex.inl | 6 +- cpp/src/strings/replace/multi_re.cu | 2 +- cpp/src/transform/row_bit_count.cu | 18 +++--- cpp/tests/io/parquet_common.cpp | 2 +- cpp/tests/io/parquet_common.hpp | 2 +- cpp/tests/iterator/indexalator_test.cu | 11 ++-- cpp/tests/iterator/offsetalator_test.cu | 3 +- cpp/tests/iterator/optional_iterator_test.cuh | 25 ++++---- .../optional_iterator_test_numeric.cu | 10 +-- 33 files changed, 176 insertions(+), 177 deletions(-) diff --git a/cpp/include/cudf/column/column_device_view.cuh b/cpp/include/cudf/column/column_device_view.cuh index 89fe59bfeaa..c3238cb94fd 100644 --- a/cpp/include/cudf/column/column_device_view.cuh +++ b/cpp/include/cudf/column/column_device_view.cuh @@ -32,9 +32,9 @@ #include +#include #include #include -#include #include #include @@ -614,7 +614,7 @@ class alignas(16) column_device_view : public detail::column_device_view_base { /** * @brief Return an optional iterator to the first element of the column. * - * Dereferencing the returned iterator returns a `thrust::optional`. + * Dereferencing the returned iterator returns a `cuda::std::optional`. * * The element of this iterator contextually converts to bool. The conversion returns true * if the object contains a value and false if it does not contain a value. @@ -739,7 +739,7 @@ class alignas(16) column_device_view : public detail::column_device_view_base { /** * @brief Return an optional iterator to the element following the last element of the column. * - * The returned iterator represents a `thrust::optional` element. + * The returned iterator represents a `cuda::std::optional` element. * * This function does not participate in overload resolution if * `column_device_view::has_element_accessor()` is false. @@ -1272,21 +1272,21 @@ struct value_accessor { * @brief optional accessor of a column * * - * The optional_accessor always returns a `thrust::optional` of `column[i]`. The validity + * The optional_accessor always returns a `cuda::std::optional` of `column[i]`. The validity * of the optional is determined by the `Nullate` parameter which may be one of the following: * * - `nullate::YES` means that the column supports nulls and the optional returned * might be valid or invalid. * * - `nullate::NO` means the caller attests that the column has no null values, - * no checks will occur and `thrust::optional{column[i]}` will be + * no checks will occur and `cuda::std::optional{column[i]}` will be * return for each `i`. * * - `nullate::DYNAMIC` defers the assumption of nullability to runtime and the caller * specifies if the column has nulls at runtime. - * For `DYNAMIC{true}` the return value will be `thrust::optional{column[i]}` if - * element `i` is not null and `thrust::optional{}` if element `i` is null. - * For `DYNAMIC{false}` the return value will always be `thrust::optional{column[i]}`. + * For `DYNAMIC{true}` the return value will be `cuda::std::optional{column[i]}` if + * element `i` is not null and `cuda::std::optional{}` if element `i` is null. + * For `DYNAMIC{false}` the return value will always be `cuda::std::optional{column[i]}`. * * @throws cudf::logic_error if column datatype and template T type mismatch. * @throws cudf::logic_error if the column is not nullable and `with_nulls` evaluates to true @@ -1312,19 +1312,19 @@ struct optional_accessor { } /** - * @brief Returns a `thrust::optional` of `column[i]`. + * @brief Returns a `cuda::std::optional` of `column[i]`. * * @param i The index of the element to return - * @return A `thrust::optional` that contains the value of `column[i]` is not null. If that + * @return A `cuda::std::optional` that contains the value of `column[i]` is not null. If that * element is null, the resulting optional will not contain a value. */ - __device__ inline thrust::optional operator()(cudf::size_type i) const + __device__ inline cuda::std::optional operator()(cudf::size_type i) const { if (has_nulls) { - return (col.is_valid_nocheck(i)) ? thrust::optional{col.element(i)} - : thrust::optional{thrust::nullopt}; + return (col.is_valid_nocheck(i)) ? cuda::std::optional{col.element(i)} + : cuda::std::optional{cuda::std::nullopt}; } - return thrust::optional{col.element(i)}; + return cuda::std::optional{col.element(i)}; } Nullate has_nulls{}; ///< Indicates if the `col` should be checked for nulls. diff --git a/cpp/include/cudf/detail/copy_if_else.cuh b/cpp/include/cudf/detail/copy_if_else.cuh index 8418e279ce7..d260a4591b7 100644 --- a/cpp/include/cudf/detail/copy_if_else.cuh +++ b/cpp/include/cudf/detail/copy_if_else.cuh @@ -25,8 +25,8 @@ #include #include +#include #include -#include namespace cudf { namespace detail { @@ -70,7 +70,7 @@ __launch_bounds__(block_size) CUDF_KERNEL while (warp_cur <= warp_end) { auto const index = static_cast(tidx); auto const opt_value = - (index < end) ? (filter(index) ? lhs[index] : rhs[index]) : thrust::nullopt; + (index < end) ? (filter(index) ? lhs[index] : rhs[index]) : cuda::std::nullopt; if (opt_value) { out.element(index) = static_cast(*opt_value); } // update validity @@ -156,7 +156,7 @@ std::unique_ptr copy_if_else(bool nullable, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - // This is the type of the thrust::optional element in the passed iterators + // This is the type of the cuda::std::optional element in the passed iterators using Element = typename thrust::iterator_traits::value_type::value_type; size_type size = std::distance(lhs_begin, lhs_end); diff --git a/cpp/include/cudf/detail/indexalator.cuh b/cpp/include/cudf/detail/indexalator.cuh index b5d57da6cd5..ec7b1c3e6b6 100644 --- a/cpp/include/cudf/detail/indexalator.cuh +++ b/cpp/include/cudf/detail/indexalator.cuh @@ -22,9 +22,9 @@ #include #include +#include #include #include -#include #include namespace cudf { @@ -376,10 +376,10 @@ struct indexalator_factory { iter = make_input_iterator(col); } - __device__ thrust::optional operator()(size_type i) const + __device__ cuda::std::optional operator()(size_type i) const { - return has_nulls && !bit_is_set(null_mask, i + offset) ? thrust::nullopt - : thrust::make_optional(iter[i]); + return has_nulls && !bit_is_set(null_mask, i + offset) ? cuda::std::nullopt + : cuda::std::make_optional(iter[i]); } }; @@ -400,9 +400,9 @@ struct indexalator_factory { iter = indexalator_factory::make_input_iterator(input); } - __device__ thrust::optional operator()(size_type) const + __device__ cuda::std::optional operator()(size_type) const { - return is_null ? thrust::nullopt : thrust::make_optional(*iter); + return is_null ? cuda::std::nullopt : cuda::std::make_optional(*iter); } }; diff --git a/cpp/include/cudf/detail/iterator.cuh b/cpp/include/cudf/detail/iterator.cuh index 9e6227ec19b..4349e1b70fd 100644 --- a/cpp/include/cudf/detail/iterator.cuh +++ b/cpp/include/cudf/detail/iterator.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,10 +37,10 @@ #include #include +#include #include #include #include -#include #include #include @@ -186,7 +186,7 @@ auto make_null_replacement_iterator(column_device_view const& column, /** * @brief Constructs an optional iterator over a column's values and its validity. * - * Dereferencing the returned iterator returns a `thrust::optional`. + * Dereferencing the returned iterator returns a `cuda::std::optional`. * * The element of this iterator contextually converts to bool. The conversion returns true * if the object contains a value and false if it does not contain a value. @@ -237,7 +237,7 @@ auto make_null_replacement_iterator(column_device_view const& column, * @param column The column to iterate * @param has_nulls Indicates whether `column` is checked for nulls. * @return Iterator that returns valid column elements and the validity of the - * element in a `thrust::optional` + * element in a `cuda::std::optional` */ template auto make_optional_iterator(column_device_view const& column, Nullate has_nulls) @@ -393,7 +393,7 @@ auto inline make_scalar_iterator(scalar const& scalar_value) /** * @brief Optional accessor for a scalar * - * The `scalar_optional_accessor` always returns a `thrust::optional` of the scalar. + * The `scalar_optional_accessor` always returns a `cuda::std::optional` of the scalar. * The validity of the optional is determined by the `Nullate` parameter which may * be one of the following: * @@ -401,14 +401,14 @@ auto inline make_scalar_iterator(scalar const& scalar_value) * will contain a value only if the scalar is valid. * * - `nullate::NO` means the caller attests that the scalar will always be valid, - * no checks will occur and `thrust::optional{column[i]}` will return a value + * no checks will occur and `cuda::std::optional{column[i]}` will return a value * for each `i`. * * - `nullate::DYNAMIC` defers the assumption of nullability to runtime and the caller * specifies if the scalar may be valid or invalid. - * For `DYNAMIC{true}` the return value will be a `thrust::optional{scalar}` when the - * scalar is valid and a `thrust::optional{}` when the scalar is invalid. - * For `DYNAMIC{false}` the return value will always be a `thrust::optional{scalar}`. + * For `DYNAMIC{true}` the return value will be a `cuda::std::optional{scalar}` when the + * scalar is valid and a `cuda::std::optional{}` when the scalar is invalid. + * For `DYNAMIC{false}` the return value will always be a `cuda::std::optional{scalar}`. * * @throws `cudf::logic_error` if scalar datatype and Element type mismatch. * @@ -418,7 +418,7 @@ auto inline make_scalar_iterator(scalar const& scalar_value) template struct scalar_optional_accessor : public scalar_value_accessor { using super_t = scalar_value_accessor; - using value_type = thrust::optional; + using value_type = cuda::std::optional; scalar_optional_accessor(scalar const& scalar_value, Nullate with_nulls) : scalar_value_accessor(scalar_value), has_nulls{with_nulls} @@ -427,7 +427,7 @@ struct scalar_optional_accessor : public scalar_value_accessor { __device__ inline value_type const operator()(size_type) const { - if (has_nulls && !super_t::dscalar.is_valid()) { return value_type{thrust::nullopt}; } + if (has_nulls && !super_t::dscalar.is_valid()) { return value_type{cuda::std::nullopt}; } if constexpr (cudf::is_fixed_point()) { using namespace numeric; @@ -519,7 +519,7 @@ struct scalar_representation_pair_accessor : public scalar_value_accessor`. + * Dereferencing the returned iterator returns a `cuda::std::optional`. * * The element of this iterator contextually converts to bool. The conversion returns true * if the object contains a value and false if it does not contain a value. @@ -575,7 +575,7 @@ struct scalar_representation_pair_accessor : public scalar_value_accessor auto inline make_optional_iterator(scalar const& scalar_value, Nullate has_nulls) diff --git a/cpp/include/cudf/json/json.hpp b/cpp/include/cudf/json/json.hpp index 48d5dcf7727..403374c536d 100644 --- a/cpp/include/cudf/json/json.hpp +++ b/cpp/include/cudf/json/json.hpp @@ -22,8 +22,6 @@ #include #include -#include - namespace CUDF_EXPORT cudf { /** diff --git a/cpp/include/cudf/strings/detail/convert/fixed_point.cuh b/cpp/include/cudf/strings/detail/convert/fixed_point.cuh index 5f51da967d3..8440805960e 100644 --- a/cpp/include/cudf/strings/detail/convert/fixed_point.cuh +++ b/cpp/include/cudf/strings/detail/convert/fixed_point.cuh @@ -17,8 +17,8 @@ #include +#include #include -#include #include namespace cudf { @@ -88,7 +88,7 @@ __device__ inline thrust::pair parse_integer( * @return Integer value of the exponent */ template -__device__ thrust::optional parse_exponent(char const* iter, char const* iter_end) +__device__ cuda::std::optional parse_exponent(char const* iter, char const* iter_end) { constexpr uint32_t exponent_max = static_cast(std::numeric_limits::max()); @@ -105,12 +105,12 @@ __device__ thrust::optional parse_exponent(char const* iter, char const while (iter < iter_end) { auto const ch = *iter++; if (ch < '0' || ch > '9') { - if (check_only) { return thrust::nullopt; } + if (check_only) { return cuda::std::nullopt; } break; } uint32_t exp_check = static_cast(exp_ten * 10) + static_cast(ch - '0'); - if (check_only && (exp_check > exponent_max)) { return thrust::nullopt; } // check overflow + if (check_only && (exp_check > exponent_max)) { return cuda::std::nullopt; } // check overflow exp_ten = static_cast(exp_check); } diff --git a/cpp/include/cudf/strings/detail/copy_if_else.cuh b/cpp/include/cudf/strings/detail/copy_if_else.cuh index 4db7651330b..213a41ca596 100644 --- a/cpp/include/cudf/strings/detail/copy_if_else.cuh +++ b/cpp/include/cudf/strings/detail/copy_if_else.cuh @@ -25,8 +25,8 @@ #include #include +#include #include -#include #include namespace cudf { @@ -41,9 +41,9 @@ namespace detail { * ``` * * @tparam StringIterLeft A random access iterator whose value_type is - * `thrust::optional` where the `optional` has a value iff the element is valid. + * `cuda::std::optional` where the `optional` has a value iff the element is valid. * @tparam StringIterRight A random access iterator whose value_type is - * `thrust::optional` where the `optional` has a value iff the element is valid. + * `cuda::std::optional` where the `optional` has a value iff the element is valid. * @tparam Filter Functor that takes an index and returns a boolean. * * @param lhs_begin Start of first set of data. Used when `filter_fn` returns true. diff --git a/cpp/include/cudf/table/experimental/row_operators.cuh b/cpp/include/cudf/table/experimental/row_operators.cuh index f05e5f4ca5c..3f33c70c29a 100644 --- a/cpp/include/cudf/table/experimental/row_operators.cuh +++ b/cpp/include/cudf/table/experimental/row_operators.cuh @@ -211,7 +211,7 @@ struct sorting_physical_element_comparator { } }; -using optional_dremel_view = thrust::optional; +using optional_dremel_view = cuda::std::optional; // The has_nested_columns template parameter of the device_row_comparator is // necessary to help the compiler optimize our code. Without it, the list and @@ -223,12 +223,12 @@ using optional_dremel_view = thrust::optional; // std::optional> in the // preprocessed_table/device_row_comparator (which is always valid when // has_nested_columns and is otherwise invalid) that is then unpacked to a -// thrust::optional at the element_comparator level (which +// cuda::std::optional at the element_comparator level (which // is always valid for a list column and otherwise invalid). We cannot use an // additional template parameter for the element_comparator on a per-column // basis because we cannot conditionally define dremel_device_view member // variables without jumping through extra hoops with inheritance, so the -// thrust::optional member must be an optional rather than +// cuda::std::optional member must be an optional rather than // a raw dremel_device_view. /** * @brief Computes the lexicographic comparison between 2 rows. diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 3ac8547baad..25b0f68aaa8 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -41,7 +41,7 @@ #include #include -#include +#include #include @@ -173,7 +173,7 @@ template void fixed_point_binary_operation_validation(binary_operator op, Lhs lhs, Rhs rhs, - thrust::optional output_type = {}) + cuda::std::optional output_type = {}) { CUDF_EXPECTS((is_fixed_point(lhs) or is_fixed_point(rhs)), "One of the inputs must have fixed_point data_type."); diff --git a/cpp/src/io/orc/orc.hpp b/cpp/src/io/orc/orc.hpp index e1403acd455..790532c9d54 100644 --- a/cpp/src/io/orc/orc.hpp +++ b/cpp/src/io/orc/orc.hpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include @@ -692,11 +692,12 @@ class metadata { * @brief `column_device_view` and additional, ORC specific, information on the column. */ struct orc_column_device_view : public column_device_view { - __device__ orc_column_device_view(column_device_view col, thrust::optional parent_idx) + __device__ orc_column_device_view(column_device_view col, + cuda::std::optional parent_idx) : column_device_view{col}, parent_index{parent_idx} { } - thrust::optional parent_index; + cuda::std::optional parent_index; bitmask_type const* pushdown_mask = nullptr; }; diff --git a/cpp/src/io/orc/writer_impl.cu b/cpp/src/io/orc/writer_impl.cu index f3b8cfbc836..04eee68e757 100644 --- a/cpp/src/io/orc/writer_impl.cu +++ b/cpp/src/io/orc/writer_impl.cu @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -50,7 +51,6 @@ #include #include #include -#include #include #include #include @@ -1831,7 +1831,7 @@ orc_table_view make_orc_table_view(table_view const& table, type_kinds, stream, rmm::mr::get_current_device_resource()); rmm::device_uvector d_orc_columns(orc_columns.size(), stream); - using stack_value_type = thrust::pair>; + using stack_value_type = thrust::pair>; rmm::device_uvector stack_storage(orc_columns.size(), stream); // pre-order append ORC device columns @@ -1847,7 +1847,7 @@ orc_table_view make_orc_table_view(table_view const& table, thrust::make_reverse_iterator(d_table.end()), thrust::make_reverse_iterator(d_table.begin()), [&stack](column_device_view const& c) { - stack.push({&c, thrust::nullopt}); + stack.push({&c, cuda::std::nullopt}); }); uint32_t idx = 0; diff --git a/cpp/src/io/parquet/compact_protocol_reader.cpp b/cpp/src/io/parquet/compact_protocol_reader.cpp index e13ed5e85e5..afcf6b373a9 100644 --- a/cpp/src/io/parquet/compact_protocol_reader.cpp +++ b/cpp/src/io/parquet/compact_protocol_reader.cpp @@ -304,10 +304,10 @@ class parquet_field_struct : public parquet_field { template class parquet_field_union_struct : public parquet_field { E& enum_val; - thrust::optional& val; // union structs are always wrapped in std::optional + cuda::std::optional& val; // union structs are always wrapped in std::optional public: - parquet_field_union_struct(int f, E& ev, thrust::optional& v) + parquet_field_union_struct(int f, E& ev, cuda::std::optional& v) : parquet_field(f), enum_val(ev), val(v) { } @@ -431,10 +431,10 @@ class parquet_field_struct_blob : public parquet_field { */ template class parquet_field_optional : public parquet_field { - thrust::optional& val; + cuda::std::optional& val; public: - parquet_field_optional(int f, thrust::optional& v) : parquet_field(f), val(v) {} + parquet_field_optional(int f, cuda::std::optional& v) : parquet_field(f), val(v) {} inline void operator()(CompactProtocolReader* cpr, int field_type) { diff --git a/cpp/src/io/parquet/parquet.hpp b/cpp/src/io/parquet/parquet.hpp index 8ee4c175e09..5d10472b0ae 100644 --- a/cpp/src/io/parquet/parquet.hpp +++ b/cpp/src/io/parquet/parquet.hpp @@ -20,7 +20,7 @@ #include -#include +#include #include #include @@ -94,10 +94,10 @@ struct LogicalType { BSON }; Type type; - thrust::optional decimal_type; - thrust::optional time_type; - thrust::optional timestamp_type; - thrust::optional int_type; + cuda::std::optional decimal_type; + cuda::std::optional time_type; + cuda::std::optional timestamp_type; + cuda::std::optional int_type; LogicalType(Type tp = UNDEFINED) : type(tp) {} LogicalType(DecimalType&& dt) : type(DECIMAL), decimal_type(dt) {} @@ -178,21 +178,21 @@ struct SchemaElement { // 5: nested fields int32_t num_children = 0; // 6: DEPRECATED: record the original type before conversion to parquet type - thrust::optional converted_type; + cuda::std::optional converted_type; // 7: DEPRECATED: record the scale for DECIMAL converted type int32_t decimal_scale = 0; // 8: DEPRECATED: record the precision for DECIMAL converted type int32_t decimal_precision = 0; // 9: save field_id from original schema - thrust::optional field_id; + cuda::std::optional field_id; // 10: replaces converted type - thrust::optional logical_type; + cuda::std::optional logical_type; // extra cudf specific fields bool output_as_byte_array = false; // cudf type determined from arrow:schema - thrust::optional arrow_type; + cuda::std::optional arrow_type; // The following fields are filled in later during schema initialization int max_definition_level = 0; @@ -259,21 +259,21 @@ struct SchemaElement { */ struct Statistics { // deprecated max value in signed comparison order - thrust::optional> max; + cuda::std::optional> max; // deprecated min value in signed comparison order - thrust::optional> min; + cuda::std::optional> min; // count of null values in the column - thrust::optional null_count; + cuda::std::optional null_count; // count of distinct values occurring - thrust::optional distinct_count; + cuda::std::optional distinct_count; // max value for column determined by ColumnOrder - thrust::optional> max_value; + cuda::std::optional> max_value; // min value for column determined by ColumnOrder - thrust::optional> min_value; + cuda::std::optional> min_value; // If true, max_value is the actual maximum value for a column - thrust::optional is_max_value_exact; + cuda::std::optional is_max_value_exact; // If true, min_value is the actual minimum value for a column - thrust::optional is_min_value_exact; + cuda::std::optional is_min_value_exact; }; /** @@ -282,7 +282,7 @@ struct Statistics { struct SizeStatistics { // Number of variable-width bytes stored for the page/chunk. Should not be set for anything // but the BYTE_ARRAY physical type. - thrust::optional unencoded_byte_array_data_bytes; + cuda::std::optional unencoded_byte_array_data_bytes; /** * When present, there is expected to be one element corresponding to each * repetition (i.e. size=max repetition_level+1) where each element @@ -291,14 +291,14 @@ struct SizeStatistics { * * This value should not be written if max_repetition_level is 0. */ - thrust::optional> repetition_level_histogram; + cuda::std::optional> repetition_level_histogram; /** * Same as repetition_level_histogram except for definition levels. * * This value should not be written if max_definition_level is 0 or 1. */ - thrust::optional> definition_level_histogram; + cuda::std::optional> definition_level_histogram; }; /** @@ -319,7 +319,7 @@ struct OffsetIndex { std::vector page_locations; // per-page size info. see description of the same field in SizeStatistics. only present for // columns with a BYTE_ARRAY physical type. - thrust::optional> unencoded_byte_array_data_bytes; + cuda::std::optional> unencoded_byte_array_data_bytes; }; /** @@ -331,10 +331,10 @@ struct ColumnIndex { std::vector> max_values; // upper bound for values in each page BoundaryOrder boundary_order = BoundaryOrder::UNORDERED; // Indicates if min and max values are ordered - thrust::optional> null_counts; // Optional count of null values per page + cuda::std::optional> null_counts; // Optional count of null values per page // Repetition/definition level histograms for the column chunk - thrust::optional> repetition_level_histogram; - thrust::optional> definition_level_histogram; + cuda::std::optional> repetition_level_histogram; + cuda::std::optional> definition_level_histogram; }; /** @@ -384,11 +384,11 @@ struct ColumnChunkMetaData { Statistics statistics; // Set of all encodings used for pages in this column chunk. This information can be used to // determine if all data pages are dictionary encoded for example. - thrust::optional> encoding_stats; + cuda::std::optional> encoding_stats; // Optional statistics to help estimate total memory when converted to in-memory representations. // The histograms contained in these statistics can also be useful in some cases for more // fine-grained nullability/list length filter pushdown. - thrust::optional size_statistics; + cuda::std::optional size_statistics; }; /** @@ -430,13 +430,13 @@ struct RowGroup { int64_t num_rows = 0; // If set, specifies a sort ordering of the rows in this RowGroup. // The sorting columns can be a subset of all the columns. - thrust::optional> sorting_columns; + cuda::std::optional> sorting_columns; // Byte offset from beginning of file to first page (data or dictionary) in this row group - thrust::optional file_offset; + cuda::std::optional file_offset; // Total byte size of all compressed (and potentially encrypted) column data in this row group - thrust::optional total_compressed_size; + cuda::std::optional total_compressed_size; // Row group ordinal in the file - thrust::optional ordinal; + cuda::std::optional ordinal; }; /** @@ -461,7 +461,7 @@ struct FileMetaData { std::vector row_groups; std::vector key_value_metadata; std::string created_by = ""; - thrust::optional> column_orders; + cuda::std::optional> column_orders; }; /** diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index efc1f5ebab1..8f52f073833 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -394,7 +394,7 @@ struct ColumnChunkDesc { uint8_t def_level_bits_, uint8_t rep_level_bits_, Compression codec_, - thrust::optional logical_type_, + cuda::std::optional logical_type_, int32_t ts_clock_rate_, int32_t src_col_index_, int32_t src_col_schema_, @@ -438,12 +438,12 @@ struct ColumnChunkDesc { int32_t num_data_pages{}; // number of data pages int32_t num_dict_pages{}; // number of dictionary pages PageInfo const* dict_page{}; - string_index_pair* str_dict_index{}; // index for string dictionary - bitmask_type** valid_map_base{}; // base pointers of valid bit map for this column - void** column_data_base{}; // base pointers of column data - void** column_string_base{}; // base pointers of column string data - Compression codec{}; // compressed codec enum - thrust::optional logical_type{}; // logical type + string_index_pair* str_dict_index{}; // index for string dictionary + bitmask_type** valid_map_base{}; // base pointers of valid bit map for this column + void** column_data_base{}; // base pointers of column data + void** column_string_base{}; // base pointers of column string data + Compression codec{}; // compressed codec enum + cuda::std::optional logical_type{}; // logical type int32_t ts_clock_rate{}; // output timestamp clock frequency (0=default, 1000=ms, 1000000000=ns) int32_t src_col_index{}; // my input column index diff --git a/cpp/src/io/parquet/predicate_pushdown.cpp b/cpp/src/io/parquet/predicate_pushdown.cpp index 481c1e9fcdd..5ca090b05b3 100644 --- a/cpp/src/io/parquet/predicate_pushdown.cpp +++ b/cpp/src/io/parquet/predicate_pushdown.cpp @@ -154,7 +154,7 @@ struct stats_caster { } void set_index(size_type index, - thrust::optional> const& binary_value, + cuda::std::optional> const& binary_value, Type const type) { if (binary_value.has_value()) { @@ -236,8 +236,8 @@ struct stats_caster { max.set_index(stats_idx, max_value, colchunk.meta_data.type); } else { // Marking it null, if column present in row group - min.set_index(stats_idx, thrust::nullopt, {}); - max.set_index(stats_idx, thrust::nullopt, {}); + min.set_index(stats_idx, cuda::std::nullopt, {}); + max.set_index(stats_idx, cuda::std::nullopt, {}); } stats_idx++; } diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 68ec61ead0a..2648a1f41ab 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -39,7 +39,7 @@ namespace { // be treated as a string. Currently the only logical type that has special handling is DECIMAL. // Other valid types in the future would be UUID (still treated as string) and FLOAT16 (which // for now would also be treated as a string). -inline bool is_treat_fixed_length_as_string(thrust::optional const& logical_type) +inline bool is_treat_fixed_length_as_string(cuda::std::optional const& logical_type) { if (!logical_type.has_value()) { return true; } return logical_type->type != LogicalType::DECIMAL; diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index 794750ab6d2..54ba898b058 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -370,11 +370,11 @@ int64_t find_next_split(int64_t cur_pos, * * @return A tuple of Parquet clock rate and Parquet decimal type. */ -[[nodiscard]] std::tuple> conversion_info( +[[nodiscard]] std::tuple> conversion_info( type_id column_type_id, type_id timestamp_type_id, Type physical, - thrust::optional logical_type) + cuda::std::optional logical_type) { int32_t const clock_rate = is_chrono(data_type{column_type_id}) ? to_clockrate(timestamp_type_id) : 0; @@ -385,7 +385,7 @@ int64_t find_next_split(int64_t cur_pos, // if decimal but not outputting as float or decimal, then convert to no logical type if (column_type_id != type_id::FLOAT64 and not cudf::is_fixed_point(data_type{column_type_id})) { - return std::make_tuple(clock_rate, thrust::nullopt); + return std::make_tuple(clock_rate, cuda::std::nullopt); } } diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 581c44d024b..00f75e4e828 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -38,7 +38,7 @@ namespace flatbuf = cudf::io::parquet::flatbuf; namespace { -thrust::optional converted_to_logical_type(SchemaElement const& schema) +cuda::std::optional converted_to_logical_type(SchemaElement const& schema) { if (schema.converted_type.has_value()) { switch (schema.converted_type.value()) { @@ -66,7 +66,7 @@ thrust::optional converted_to_logical_type(SchemaElement const& sch default: return LogicalType{LogicalType::UNDEFINED}; } } - return thrust::nullopt; + return cuda::std::nullopt; } } // namespace @@ -246,7 +246,7 @@ void metadata::sanitize_schema() struct_elem.repetition_type = REQUIRED; struct_elem.num_children = schema_elem.num_children; struct_elem.type = UNDEFINED_TYPE; - struct_elem.converted_type = thrust::nullopt; + struct_elem.converted_type = cuda::std::nullopt; // swap children struct_elem.children_idx = std::move(schema_elem.children_idx); diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 36a1d8377bf..c2c5dbb4a56 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -185,7 +185,7 @@ struct aggregate_writer_metadata { std::vector> column_indexes; }; std::vector files; - thrust::optional> column_orders = thrust::nullopt; + cuda::std::optional> column_orders = cuda::std::nullopt; }; namespace { @@ -471,7 +471,7 @@ struct leaf_schema_fn { std::enable_if_t, void> operator()() { col_schema.type = (timestamp_is_int96) ? Type::INT96 : Type::INT64; - col_schema.converted_type = thrust::nullopt; + col_schema.converted_type = cuda::std::nullopt; col_schema.stats_dtype = statistics_dtype::dtype_timestamp64; if (timestamp_is_int96) { col_schema.ts_scale = -1000; // negative value indicates division by absolute value @@ -749,7 +749,7 @@ std::vector construct_parquet_schema_tree( col_schema.type = Type::BYTE_ARRAY; } - col_schema.converted_type = thrust::nullopt; + col_schema.converted_type = cuda::std::nullopt; col_schema.stats_dtype = statistics_dtype::dtype_byte_array; col_schema.repetition_type = col_nullable ? OPTIONAL : REQUIRED; col_schema.name = (schema[parent_idx].name == "list") ? "element" : col_meta.get_name(); @@ -2776,7 +2776,7 @@ std::unique_ptr> writer::merge_row_group_metadata( // See https://github.com/rapidsai/cudf/pull/14264#issuecomment-1778311615 for (auto& se : md.schema) { if (se.logical_type.has_value() && se.logical_type.value().type == LogicalType::UNKNOWN) { - se.logical_type = thrust::nullopt; + se.logical_type = cuda::std::nullopt; } } diff --git a/cpp/src/json/json_path.cu b/cpp/src/json/json_path.cu index d1a1097de35..1bf4bf3b153 100644 --- a/cpp/src/json/json_path.cu +++ b/cpp/src/json/json_path.cu @@ -39,7 +39,7 @@ #include #include -#include +#include #include #include #include @@ -207,7 +207,7 @@ class parser { struct json_output { size_t output_max_len; char* output; - thrust::optional output_len; + cuda::std::optional output_len; __device__ void add_output(char const* str, size_t len) { @@ -656,7 +656,7 @@ class path_state : private parser { * @param stream Cuda stream to perform any gpu actions on * @returns A pair containing the command buffer, and maximum stack depth required. */ -std::pair>, int> build_command_buffer( +std::pair>, int> build_command_buffer( cudf::string_scalar const& json_path, rmm::cuda_stream_view stream) { std::string h_json_path = json_path.to_string(stream); @@ -690,8 +690,8 @@ std::pair>, int> build_comma } while (op.type != path_operator_type::END); auto const is_empty = h_operators.size() == 1 && h_operators[0].type == path_operator_type::END; - return is_empty ? std::pair(thrust::nullopt, 0) - : std::pair(thrust::make_optional(cudf::detail::make_device_uvector_sync( + return is_empty ? std::pair(cuda::std::nullopt, 0) + : std::pair(cuda::std::make_optional(cudf::detail::make_device_uvector_sync( h_operators, stream, rmm::mr::get_current_device_resource())), max_stack_depth); } @@ -920,9 +920,9 @@ __launch_bounds__(block_size) CUDF_KERNEL path_operator const* const commands, size_type* d_sizes, cudf::detail::input_offsetalator output_offsets, - thrust::optional out_buf, - thrust::optional out_validity, - thrust::optional out_valid_count, + cuda::std::optional out_buf, + cuda::std::optional out_validity, + cuda::std::optional out_valid_count, get_json_object_options options) { auto tid = cudf::detail::grid_1d::global_thread_id(); @@ -1012,9 +1012,9 @@ std::unique_ptr get_json_object(cudf::strings_column_view const& c std::get<0>(preprocess).value().data(), sizes.data(), d_offsets, - thrust::nullopt, - thrust::nullopt, - thrust::nullopt, + cuda::std::nullopt, + cuda::std::nullopt, + cuda::std::nullopt, options); // convert sizes to offsets diff --git a/cpp/src/lists/contains.cu b/cpp/src/lists/contains.cu index 30c03a8cd68..11703527d26 100644 --- a/cpp/src/lists/contains.cu +++ b/cpp/src/lists/contains.cu @@ -40,7 +40,6 @@ #include #include #include -#include #include #include #include diff --git a/cpp/src/lists/explode.cu b/cpp/src/lists/explode.cu index 46c4fc78a6f..74a0d842aad 100644 --- a/cpp/src/lists/explode.cu +++ b/cpp/src/lists/explode.cu @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -36,7 +37,6 @@ #include #include #include -#include #include #include @@ -57,8 +57,8 @@ std::unique_ptr
build_table( size_type const explode_column_idx, column_view const& sliced_child, cudf::device_span gather_map, - thrust::optional> explode_col_gather_map, - thrust::optional> position_array, + cuda::std::optional> explode_col_gather_map, + cuda::std::optional> position_array, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -143,8 +143,8 @@ std::unique_ptr
explode(table_view const& input_table, explode_column_idx, sliced_child, gather_map, - thrust::nullopt, - thrust::nullopt, + cuda::std::nullopt, + cuda::std::nullopt, stream, mr); } @@ -193,7 +193,7 @@ std::unique_ptr
explode_position(table_view const& input_table, explode_column_idx, sliced_child, gather_map, - thrust::nullopt, + cuda::std::nullopt, std::move(pos), stream, mr); @@ -292,7 +292,7 @@ std::unique_ptr
explode_outer(table_view const& input_table, sliced_child, gather_map, explode_col_gather_map, - include_position ? std::move(pos) : thrust::optional>{}, + include_position ? std::move(pos) : cuda::std::optional>{}, stream, mr); } diff --git a/cpp/src/strings/convert/convert_datetime.cu b/cpp/src/strings/convert/convert_datetime.cu index 64a2107e17a..99c40f00b00 100644 --- a/cpp/src/strings/convert/convert_datetime.cu +++ b/cpp/src/strings/convert/convert_datetime.cu @@ -36,11 +36,11 @@ #include #include +#include #include #include #include #include -#include #include #include @@ -519,7 +519,7 @@ struct check_datetime_format { * The checking here is a little more strict than the actual * parser used for conversion. */ - __device__ thrust::optional check_string(string_view const& d_string) + __device__ cuda::std::optional check_string(string_view const& d_string) { timestamp_components dateparts = {1970, 1, 1, 0}; // init to epoch time @@ -529,7 +529,7 @@ struct check_datetime_format { // eliminate static character values first if (item.item_type == format_char_type::literal) { // check static character matches - if (*ptr != item.value) return thrust::nullopt; + if (*ptr != item.value) return cuda::std::nullopt; ptr += item.length; length -= item.length; continue; @@ -645,7 +645,7 @@ struct check_datetime_format { case 'Z': result = true; // skip default: break; } - if (!result) return thrust::nullopt; + if (!result) return cuda::std::nullopt; ptr += bytes_read; length -= bytes_read; } @@ -821,7 +821,7 @@ struct datetime_formatter_fn { // We only dissect the timestamp into components if needed // by a specifier. And then we only do it once and reuse it. // This can improve performance when not using uncommon specifiers. - thrust::optional days; + cuda::std::optional days; auto days_from_timestamp = [tstamp]() { auto const count = tstamp.time_since_epoch().count(); diff --git a/cpp/src/strings/regex/regex.cuh b/cpp/src/strings/regex/regex.cuh index e6134296e45..2df404048f7 100644 --- a/cpp/src/strings/regex/regex.cuh +++ b/cpp/src/strings/regex/regex.cuh @@ -23,8 +23,8 @@ #include +#include #include -#include #include #include @@ -36,7 +36,7 @@ namespace detail { struct relist; using match_pair = thrust::pair; -using match_result = thrust::optional; +using match_result = cuda::std::optional; constexpr int32_t MAX_SHARED_MEM = 2048; ///< Memory size for storing prog instruction data constexpr std::size_t MAX_WORKING_MEM = 0x01'FFFF'FFFF; ///< Memory size for state data diff --git a/cpp/src/strings/regex/regex.inl b/cpp/src/strings/regex/regex.inl index 23e1944cda4..3b899e4edc1 100644 --- a/cpp/src/strings/regex/regex.inl +++ b/cpp/src/strings/regex/regex.inl @@ -260,12 +260,12 @@ __device__ __forceinline__ match_result reprog_device::regexec(string_view const switch (jnk.starttype) { case BOL: if (pos == 0) break; - if (jnk.startchar != '^') { return thrust::nullopt; } + if (jnk.startchar != '^') { return cuda::std::nullopt; } --itr; startchar = static_cast('\n'); case CHAR: { auto const find_itr = find_char(startchar, dstr, itr); - if (find_itr.byte_offset() >= dstr.size_bytes()) { return thrust::nullopt; } + if (find_itr.byte_offset() >= dstr.size_bytes()) { return cuda::std::nullopt; } itr = find_itr + (jnk.starttype == BOL); pos = itr.position(); break; @@ -396,7 +396,7 @@ __device__ __forceinline__ match_result reprog_device::regexec(string_view const checkstart = jnk.list1->get_size() == 0; } while (!last_character && (!checkstart || !match)); - return match ? match_result({begin, end}) : thrust::nullopt; + return match ? match_result({begin, end}) : cuda::std::nullopt; } __device__ __forceinline__ match_result reprog_device::find(int32_t const thread_idx, diff --git a/cpp/src/strings/replace/multi_re.cu b/cpp/src/strings/replace/multi_re.cu index 31234ea42ec..0ad3ab2305c 100644 --- a/cpp/src/strings/replace/multi_re.cu +++ b/cpp/src/strings/replace/multi_re.cu @@ -92,7 +92,7 @@ struct replace_multi_regex_fn { } reprog_device prog = progs[ptn_idx]; - auto const result = !prog.is_empty() ? prog.find(idx, d_str, itr) : thrust::nullopt; + auto const result = !prog.is_empty() ? prog.find(idx, d_str, itr) : cuda::std::nullopt; d_ranges[ptn_idx] = result ? found_range{result->first, result->second} : found_range{nchars, nchars}; } diff --git a/cpp/src/transform/row_bit_count.cu b/cpp/src/transform/row_bit_count.cu index 4530fabf889..6a965d10184 100644 --- a/cpp/src/transform/row_bit_count.cu +++ b/cpp/src/transform/row_bit_count.cu @@ -35,8 +35,8 @@ #include #include +#include #include -#include #include namespace cudf { @@ -159,9 +159,9 @@ void flatten_hierarchy(ColIter begin, std::vector& info, hierarchy_info& h_info, rmm::cuda_stream_view stream, - size_type cur_depth = 0, - size_type cur_branch_depth = 0, - thrust::optional parent_index = {}); + size_type cur_depth = 0, + size_type cur_branch_depth = 0, + cuda::std::optional parent_index = {}); /** * @brief Type-dispatched functor called by flatten_hierarchy. @@ -177,7 +177,7 @@ struct flatten_functor { rmm::cuda_stream_view, size_type cur_depth, size_type cur_branch_depth, - thrust::optional) + cuda::std::optional) { out.push_back(col); info.push_back({cur_depth, cur_branch_depth, cur_branch_depth}); @@ -194,7 +194,7 @@ struct flatten_functor { rmm::cuda_stream_view, size_type cur_depth, size_type cur_branch_depth, - thrust::optional) + cuda::std::optional) { out.push_back(col); info.push_back({cur_depth, cur_branch_depth, cur_branch_depth}); @@ -210,7 +210,7 @@ struct flatten_functor { rmm::cuda_stream_view stream, size_type cur_depth, size_type cur_branch_depth, - thrust::optional parent_index) + cuda::std::optional parent_index) { // track branch depth as we reach this list and after we pass it auto const branch_depth_start = cur_branch_depth; @@ -243,7 +243,7 @@ struct flatten_functor { rmm::cuda_stream_view stream, size_type cur_depth, size_type cur_branch_depth, - thrust::optional) + cuda::std::optional) { out.push_back(col); info.push_back({cur_depth, cur_branch_depth, cur_branch_depth}); @@ -284,7 +284,7 @@ void flatten_hierarchy(ColIter begin, rmm::cuda_stream_view stream, size_type cur_depth, size_type cur_branch_depth, - thrust::optional parent_index) + cuda::std::optional parent_index) { std::for_each(begin, end, [&](column_view const& col) { cudf::type_dispatcher(col.type(), diff --git a/cpp/tests/io/parquet_common.cpp b/cpp/tests/io/parquet_common.cpp index c1211869bcc..3dd5ad145ea 100644 --- a/cpp/tests/io/parquet_common.cpp +++ b/cpp/tests/io/parquet_common.cpp @@ -744,7 +744,7 @@ int32_t compare(T& v1, T& v2) int32_t compare_binary(std::vector const& v1, std::vector const& v2, cudf::io::parquet::detail::Type ptype, - thrust::optional const& ctype) + cuda::std::optional const& ctype) { auto ctype_val = ctype.value_or(cudf::io::parquet::detail::UNKNOWN); switch (ptype) { diff --git a/cpp/tests/io/parquet_common.hpp b/cpp/tests/io/parquet_common.hpp index 59ee85444f2..bc6145d77da 100644 --- a/cpp/tests/io/parquet_common.hpp +++ b/cpp/tests/io/parquet_common.hpp @@ -172,7 +172,7 @@ std::pair create_parquet_typed_with_stats(std::string int32_t compare_binary(std::vector const& v1, std::vector const& v2, cudf::io::parquet::detail::Type ptype, - thrust::optional const& ctype); + cuda::std::optional const& ctype); void expect_compression_stats_empty(std::shared_ptr stats); diff --git a/cpp/tests/iterator/indexalator_test.cu b/cpp/tests/iterator/indexalator_test.cu index 0c10853ec02..dac2356dcb0 100644 --- a/cpp/tests/iterator/indexalator_test.cu +++ b/cpp/tests/iterator/indexalator_test.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ #include +#include #include #include #include -#include #include #include #include @@ -84,15 +84,16 @@ TYPED_TEST(IndexalatorTest, optional_iterator) auto d_col = cudf::test::fixed_width_column_wrapper( host_values.begin(), host_values.end(), validity.begin()); - auto expected_values = thrust::host_vector>(host_values.size()); + auto expected_values = + thrust::host_vector>(host_values.size()); std::transform(host_values.begin(), host_values.end(), validity.begin(), expected_values.begin(), [](T v, bool b) { - return (b) ? thrust::make_optional(static_cast(v)) - : thrust::nullopt; + return (b) ? cuda::std::make_optional(static_cast(v)) + : cuda::std::nullopt; }); auto it_dev = cudf::detail::indexalator_factory::make_input_optional_iterator(d_col); diff --git a/cpp/tests/iterator/offsetalator_test.cu b/cpp/tests/iterator/offsetalator_test.cu index e569e58f42a..b206ff947bb 100644 --- a/cpp/tests/iterator/offsetalator_test.cu +++ b/cpp/tests/iterator/offsetalator_test.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include diff --git a/cpp/tests/iterator/optional_iterator_test.cuh b/cpp/tests/iterator/optional_iterator_test.cuh index 6a264cee9a8..04f5410a44f 100644 --- a/cpp/tests/iterator/optional_iterator_test.cuh +++ b/cpp/tests/iterator/optional_iterator_test.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,8 @@ #include +#include #include -#include template void nonull_optional_iterator(IteratorTest& testFixture) @@ -32,9 +32,9 @@ void nonull_optional_iterator(IteratorTest& testFixture) auto d_col = cudf::column_device_view::create(w_col); // calculate the expected value by CPU. - thrust::host_vector> replaced_array(host_values.size()); + thrust::host_vector> replaced_array(host_values.size()); std::transform(host_values.begin(), host_values.end(), replaced_array.begin(), [](auto s) { - return thrust::optional{s}; + return cuda::std::optional{s}; }); // GPU test @@ -61,19 +61,20 @@ void null_optional_iterator(IteratorTest& testFixture) auto d_col = cudf::column_device_view::create(w_col); // calculate the expected value by CPU. - thrust::host_vector> optional_values(host_values.size()); - std::transform(host_values.begin(), - host_values.end(), - host_bools.begin(), - optional_values.begin(), - [](auto s, bool b) { return b ? thrust::optional{s} : thrust::optional{}; }); + thrust::host_vector> optional_values(host_values.size()); + std::transform( + host_values.begin(), + host_values.end(), + host_bools.begin(), + optional_values.begin(), + [](auto s, bool b) { return b ? cuda::std::optional{s} : cuda::std::optional{}; }); - thrust::host_vector> value_all_valid(host_values.size()); + thrust::host_vector> value_all_valid(host_values.size()); std::transform(host_values.begin(), host_values.end(), host_bools.begin(), value_all_valid.begin(), - [](auto s, bool b) { return thrust::optional{s}; }); + [](auto s, bool b) { return cuda::std::optional{s}; }); // GPU test for correct null mapping testFixture.iterator_test_thrust( diff --git a/cpp/tests/iterator/optional_iterator_test_numeric.cu b/cpp/tests/iterator/optional_iterator_test_numeric.cu index 98befb0a3ee..257c0979017 100644 --- a/cpp/tests/iterator/optional_iterator_test_numeric.cu +++ b/cpp/tests/iterator/optional_iterator_test_numeric.cu @@ -18,9 +18,9 @@ #include +#include #include #include -#include #include #include @@ -49,21 +49,21 @@ TYPED_TEST(NumericOptionalIteratorTest, null_optional_iterator) { null_optional_ // Transformers and Operators for optional_iterator test template struct transformer_optional_meanvar { - using ResultType = thrust::optional>; + using ResultType = cuda::std::optional>; - CUDF_HOST_DEVICE inline ResultType operator()(thrust::optional const& optional) + CUDF_HOST_DEVICE inline ResultType operator()(cuda::std::optional const& optional) { if (optional.has_value()) { auto v = *optional; return cudf::meanvar{v, static_cast(v * v), 1}; } - return thrust::nullopt; + return cuda::std::nullopt; } }; template struct optional_to_meanvar { - CUDF_HOST_DEVICE inline T operator()(thrust::optional const& v) { return v.value_or(T{0}); } + CUDF_HOST_DEVICE inline T operator()(cuda::std::optional const& v) { return v.value_or(T{0}); } }; // TODO: enable this test also at __CUDACC_DEBUG__ From 555734dee7a8fb10f50c8609a8e4fb2c025e6305 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 20 Aug 2024 09:32:59 -0500 Subject: [PATCH 096/270] Remove thrust::optional from expression evaluator (#16604) This PR follows up on a request from @davidwendt in https://github.com/rapidsai/cudf/pull/15091#discussion_r1722183142. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/16604 --- cpp/include/cudf/ast/detail/expression_evaluator.cuh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cpp/include/cudf/ast/detail/expression_evaluator.cuh b/cpp/include/cudf/ast/detail/expression_evaluator.cuh index 105d87ff96f..9d8762555d7 100644 --- a/cpp/include/cudf/ast/detail/expression_evaluator.cuh +++ b/cpp/include/cudf/ast/detail/expression_evaluator.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,8 +29,6 @@ #include -#include - namespace cudf { namespace ast { @@ -278,7 +276,7 @@ struct expression_evaluator { detail::device_data_reference const& input_reference, IntermediateDataType* thread_intermediate_storage, cudf::size_type left_row_index, - thrust::optional right_row_index = {}) const + cudf::size_type right_row_index = {}) const { // TODO: Everywhere in the code assumes that the table reference is either // left or right. Should we error-check somewhere to prevent @@ -291,7 +289,7 @@ struct expression_evaluator { // any case where input_reference.table_source == table_reference::RIGHT. // Otherwise, behavior is undefined. auto const row_index = - (input_reference.table_source == table_reference::LEFT) ? left_row_index : *right_row_index; + (input_reference.table_source == table_reference::LEFT) ? left_row_index : right_row_index; if constexpr (has_nulls) { return table.column(input_reference.data_index).is_valid(row_index) ? ReturnType(table.column(input_reference.data_index).element(row_index)) @@ -329,7 +327,7 @@ struct expression_evaluator { detail::device_data_reference const& device_data_reference, IntermediateDataType* thread_intermediate_storage, cudf::size_type left_row_index, - thrust::optional right_row_index = {}) const + cudf::size_type right_row_index = {}) const { CUDF_UNREACHABLE("Unsupported type in resolve_input."); } From b32bc10ee9795ba94df9a79d6fa5bfd2a53455d6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 12:43:51 -0500 Subject: [PATCH 097/270] do not install cudf in cudf_polars wheel tests (#16612) Removes unnecessary installation of `cudf` wheels in wheel testing for `cudf_polars`. `cudf_polars` doesn't depend on `cudf`, and neither do its tests. However, right now it's downloading `cudf` during it's wheel tests. I mistakenly introduced that in #16575. This introduced a race condition that could lead to CI failures whenever the `cudf` wheels aren't published yet by the time the `cudf_polars` tests. Because the `cudf_polars` wheel tests (rightly) do not wait for `cudf` wheels to be available: https://github.com/rapidsai/cudf/blob/555734dee7a8fb10f50c8609a8e4fb2c025e6305/.github/workflows/pr.yaml#L154-L155 https://github.com/rapidsai/cudf/blob/555734dee7a8fb10f50c8609a8e4fb2c025e6305/.github/workflows/pr.yaml#L145-L146 Noticed this in #16611 ```text [rapids-download-from-s3] Downloading and decompressing s3://rapids-downloads/ci/cudf/pull-request/16611/a6b7eff/cudf_wheel_python_cudf_cu12_py310_x86_64.tar.gz into ./dist download failed: s3://rapids-downloads/ci/cudf/pull-request/16611/a6b7eff/cudf_wheel_python_cudf_cu12_py310_x86_64.tar.gz to - An error occurred (404) when calling the HeadObject operation: Not Found ``` ([build link](https://github.com/rapidsai/cudf/actions/runs/10472939821/job/29004728278?pr=16611)) Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16612 --- ci/test_wheel_cudf_polars.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 6438d13c4b7..e9c6188502c 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -20,15 +20,13 @@ fi RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist -# Download the cudf and pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# Download pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist rapids-logger "Installing cudf_polars and its dependencies" # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" From e450baf1d748a4a361797ee18a1372095212b816 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 13:59:58 -0500 Subject: [PATCH 098/270] remove streamz git dependency, standardize build dependency names, consolidate some dependency lists (#16611) Proposes some additional cleanup in `dependencies.yaml`, for things I noticed while working through #15483. * standardizes the naming of keys in the `files:` section for build dependencies - *`py_build_{project}` = dependencies for the `[build-system]` table* - *`py_rapids_build_{project}` = dependencies for the `[tool.rapids-build-backend]` table* - *this is how it was done over most of the other repos in https://github.com/rapidsai/build-planning/issues/31, it was just missed because `cudf` was one of the first repos to add `rapids-build-backend`* * removes the dependency on building `streamz` from latest source on GitHub - *`custreamz` conda packages and wheels depend on packages for those, not this git dependency* - https://github.com/rapidsai/cudf/blob/2f7d35435db2b5ed9ead96cf43e2a710db5e5e6d/dependencies.yaml#L752-L754 - https://github.com/rapidsai/cudf/blob/2f7d35435db2b5ed9ead96cf43e2a710db5e5e6d/conda/recipes/custreamz/meta.yaml#L45-L47 - *if this is really needed, I don't think it belongs in the `build_python_cudf` set* - *the last commit to `streamz` was 2 years ago (https://github.com/python-streamz/streamz), this doesn't seem like a `rapids-dask-dependency`, try-to-always-test-against-latest, situation to me* - *I'm guessing this is left over from a time before `streamz` was regularly publishing wheels... it's been in `dependencies.yaml` since that file was first introduced here in November 2022 (#11674)* - *the last release, v0.6.4, was made on July 27, 2022. There have been around 20 commits to `master` since then ([history link](https://github.com/python-streamz/streamz/commits/master/)) ... but if `custreamz` really needed those, I'd expect `custreamz` to depend on the version built from GitHub sources. I strongly suspect that that isn't the case.* * removes `build_python_cudf` and `build_python_libcudf` lists in `dependencies.yaml`, in favor of re-using the `depends_on_rmm` and `depends_on_pylibcudf` lists Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16611 --- .../all_cuda-118_arch-x86_64.yaml | 3 - .../all_cuda-125_arch-x86_64.yaml | 3 - dependencies.yaml | 137 ++++++------------ 3 files changed, 42 insertions(+), 101 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d0d18e57abc..018162bd848 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -66,7 +66,6 @@ dependencies: - pandas - pandas>=2.0,<2.2.3dev0 - pandoc -- pip - pre-commit - ptxcompiler - pyarrow==16.1.0.* @@ -99,6 +98,4 @@ dependencies: - transformers==4.39.3 - typing_extensions>=4.0.0 - zlib>=1.2.13 -- pip: - - git+https://github.com/python-streamz/streamz.git@master name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index caf39a32d79..c60ffa7aaa5 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -64,7 +64,6 @@ dependencies: - pandas - pandas>=2.0,<2.2.3dev0 - pandoc -- pip - pre-commit - pyarrow==16.1.0.* - pydata-sphinx-theme!=0.14.2 @@ -97,6 +96,4 @@ dependencies: - transformers==4.39.3 - typing_extensions>=4.0.0 - zlib>=1.2.13 -- pip: - - git+https://github.com/python-streamz/streamz.git@master name: all_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index a774345fe95..150d03be021 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -10,10 +10,10 @@ files: - build_all - build_cpp - build_python_common - - build_python_pylibcudf - - build_python_cudf - cuda - cuda_version + - depends_on_cupy + - depends_on_rmm - develop - docs - libarrow_build @@ -31,7 +31,6 @@ files: - test_python_cudf - test_python_dask_cudf - test_python_pylibcudf - - depends_on_cupy test_static_build: output: none includes: @@ -95,7 +94,8 @@ files: includes: - build_base - build_python_common - - build_python_cudf + - depends_on_pylibcudf + - depends_on_rmm py_run_cudf: output: pyproject pyproject_dir: python/cudf @@ -107,6 +107,7 @@ files: - pyarrow_run - depends_on_cupy - depends_on_pylibcudf + - depends_on_rmm py_test_cudf: output: pyproject pyproject_dir: python/cudf @@ -116,14 +117,14 @@ files: includes: - test_python_common - test_python_cudf - py_rapids_build_pylibcudf: + py_build_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf extras: table: build-system includes: - rapids_build_skbuild - py_build_pylibcudf: + py_rapids_build_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf extras: @@ -132,15 +133,16 @@ files: includes: - build_base - build_python_common - - build_python_pylibcudf + - depends_on_rmm py_run_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf extras: table: project includes: - - run_pylibcudf + - depends_on_rmm - pyarrow_run + - run_pylibcudf py_test_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf @@ -215,14 +217,14 @@ files: includes: - test_python_common - test_python_dask_cudf - py_rapids_build_cudf_kafka: + py_build_cudf_kafka: output: pyproject pyproject_dir: python/cudf_kafka extras: table: build-system includes: - rapids_build_skbuild - py_build_cudf_kafka: + py_rapids_build_cudf_kafka: output: pyproject pyproject_dir: python/cudf_kafka extras: @@ -364,65 +366,6 @@ dependencies: # Sync with conda build constraint & wheel run constraint. # TODO: Change to `2.0.*` for NumPy 2 - numpy==1.23.* - build_python_pylibcudf: - common: - - output_types: conda - packages: - - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm-cu{11,12}. - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_unsuffixed]} - build_python_cudf: - common: - - output_types: conda - packages: - - *rmm_unsuffixed - - pip - - pip: - - git+https://github.com/python-streamz/streamz.git@master - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm-cu{11,12}. - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - - git+https://github.com/python-streamz/streamz.git@master - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - &pylibcudf_cu12 pylibcudf-cu12==24.10.*,>=0.0.0a0 - - *rmm_cu12 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - &pylibcudf_cu11 pylibcudf-cu11==24.10.*,>=0.0.0a0 - - *rmm_cu11 - - matrix: - packages: - - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0 - - *rmm_unsuffixed libarrow_build: common: - output_types: conda @@ -635,9 +578,6 @@ dependencies: - nvtx>=0.2.1 - packaging - typing_extensions>=4.0.0 - - output_types: conda - packages: - - *rmm_unsuffixed - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -654,19 +594,6 @@ dependencies: packages: &run_pylibcudf_packages_all_cu11 - cuda-python>=11.7.1,<12.0a0 - {matrix: null, packages: *run_pylibcudf_packages_all_cu11} - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - *rmm_cu12 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - *rmm_cu11 - - {matrix: null, packages: [*rmm_unsuffixed]} run_cudf: common: - output_types: [conda, requirements, pyproject] @@ -677,9 +604,6 @@ dependencies: - packaging - rich - typing_extensions>=4.0.0 - - output_types: conda - packages: - - *rmm_unsuffixed - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -711,19 +635,16 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - *rmm_cu12 - pynvjitlink-cu12>=0.0.0a0 - matrix: cuda: "12.*" cuda_suffixed: "false" packages: - - *rmm_unsuffixed - *pynvjitlink_unsuffixed - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - *rmm_cu11 - cubinlinker-cu11 - ptxcompiler-cu11 - matrix: @@ -732,7 +653,6 @@ dependencies: packages: &run_cudf_cu11_unsuffixed - *cubinlinker_unsuffixed - *ptxcompiler_unsuffixed - - *rmm_unsuffixed - {matrix: null, packages: *run_cudf_cu11_unsuffixed} run_cudf_polars: common: @@ -843,7 +763,7 @@ dependencies: common: - output_types: conda packages: - - *pylibcudf_unsuffixed + - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -857,12 +777,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - *pylibcudf_cu12 + - pylibcudf-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - *pylibcudf_cu11 + - pylibcudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*pylibcudf_unsuffixed]} depends_on_cudf: common: @@ -929,6 +849,33 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + depends_on_rmm: + common: + - output_types: conda + packages: + - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - rmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - rmm-cu11==24.10.*,>=0.0.0a0 + - matrix: + packages: + - *rmm_unsuffixed test_python_pandas_cudf: common: - output_types: [requirements, pyproject] From 28fee97c24bcb5f6c61241058c7c3f824687f654 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 20 Aug 2024 17:02:49 -0400 Subject: [PATCH 099/270] Enable gtests previously disabled for compute-sanitizer bug (#16581) Enables tests disable in https://github.com/rapidsai/cudf/pull/15259 due to a `compute-sanitizer` bug. This has been fixed in the CUDA 12.5 release and the nightly memchecks should pass again with these enabled. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16581 --- .../iterator/value_iterator_test_numeric.cu | 16 ++----------- cpp/tests/reductions/reduction_tests.cpp | 3 --- .../reductions/segmented_reduction_tests.cpp | 24 ------------------- 3 files changed, 2 insertions(+), 41 deletions(-) diff --git a/cpp/tests/iterator/value_iterator_test_numeric.cu b/cpp/tests/iterator/value_iterator_test_numeric.cu index d3d1c12bdc7..39e05ff6832 100644 --- a/cpp/tests/iterator/value_iterator_test_numeric.cu +++ b/cpp/tests/iterator/value_iterator_test_numeric.cu @@ -23,17 +23,5 @@ template struct NumericValueIteratorTest : public IteratorTest {}; TYPED_TEST_SUITE(NumericValueIteratorTest, TestingTypes); -TYPED_TEST(NumericValueIteratorTest, non_null_iterator) -{ - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - non_null_iterator(*this); -} -TYPED_TEST(NumericValueIteratorTest, null_iterator) -{ - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - null_iterator(*this); -} +TYPED_TEST(NumericValueIteratorTest, non_null_iterator) { non_null_iterator(*this); } +TYPED_TEST(NumericValueIteratorTest, null_iterator) { null_iterator(*this); } diff --git a/cpp/tests/reductions/reduction_tests.cpp b/cpp/tests/reductions/reduction_tests.cpp index 0ec4cfa34c4..949ffcc26a6 100644 --- a/cpp/tests/reductions/reduction_tests.cpp +++ b/cpp/tests/reductions/reduction_tests.cpp @@ -300,9 +300,6 @@ TYPED_TEST_SUITE(ReductionTest, cudf::test::NumericTypes); TYPED_TEST(ReductionTest, Product) { using T = TypeParam; - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } std::vector int_values({5, -1, 1, 0, 3, 2, 4}); std::vector host_bools({true, true, false, false, true, true, true}); diff --git a/cpp/tests/reductions/segmented_reduction_tests.cpp b/cpp/tests/reductions/segmented_reduction_tests.cpp index 37efc116d2a..668690639a6 100644 --- a/cpp/tests/reductions/segmented_reduction_tests.cpp +++ b/cpp/tests/reductions/segmented_reduction_tests.cpp @@ -87,10 +87,6 @@ TYPED_TEST(SegmentedReductionTest, SumExcludeNulls) TYPED_TEST(SegmentedReductionTest, ProductExcludeNulls) { - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - // [1, 3, 5], [null, 3, 5], [1], [null], [null, null], [] // values: {1, 3, 5, XXX, 3, 5, 1, XXX, XXX, XXX} // offsets: {0, 3, 6, 7, 8, 10, 10} @@ -141,10 +137,6 @@ TYPED_TEST(SegmentedReductionTest, ProductExcludeNulls) TYPED_TEST(SegmentedReductionTest, MaxExcludeNulls) { - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX} // offsets: {0, 3, 6, 7, 8, 10, 10} @@ -193,10 +185,6 @@ TYPED_TEST(SegmentedReductionTest, MaxExcludeNulls) TYPED_TEST(SegmentedReductionTest, MinExcludeNulls) { - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX} // offsets: {0, 3, 6, 7, 8, 10, 10} @@ -388,10 +376,6 @@ TYPED_TEST(SegmentedReductionTest, SumIncludeNulls) TYPED_TEST(SegmentedReductionTest, ProductIncludeNulls) { - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - // [1, 3, 5], [null, 3, 5], [1], [null], [null, null], [] // values: {1, 3, 5, XXX, 3, 5, 1, XXX, XXX, XXX} // offsets: {0, 3, 6, 7, 8, 10, 10} @@ -445,10 +429,6 @@ TYPED_TEST(SegmentedReductionTest, ProductIncludeNulls) TYPED_TEST(SegmentedReductionTest, MaxIncludeNulls) { - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX} // offsets: {0, 3, 6, 7, 8, 10, 10} @@ -500,10 +480,6 @@ TYPED_TEST(SegmentedReductionTest, MaxIncludeNulls) TYPED_TEST(SegmentedReductionTest, MinIncludeNulls) { - if constexpr (std::is_same_v || std::is_same_v) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return; } - } - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX} // offsets: {0, 3, 6, 7, 8, 10, 10} From 58799d698d861866b5650d368f5195174fc9644e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:29:27 -1000 Subject: [PATCH 100/270] Add stricter typing and validation to ColumnAccessor (#16602) * Added typing annotations that are generally a little stricter on when `Column`s should be passed. Added error handling for these cases * Moved some argument checking that was performed on `DataFrame` to `ColumnAccessor` * Adding more `verify=False` to `ColumnAccessor` calls and preserving `.label_dtype` more when we're just selecting columns from the prior `ColumnAccessor` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16602 --- python/cudf/cudf/_lib/csv.pyx | 2 +- python/cudf/cudf/core/_base_index.py | 2 +- python/cudf/cudf/core/column_accessor.py | 114 ++++++++++-------- python/cudf/cudf/core/dataframe.py | 14 +-- python/cudf/cudf/core/frame.py | 4 +- python/cudf/cudf/core/indexing_utils.py | 4 - python/cudf/cudf/core/join/_join_helpers.py | 8 +- python/cudf/cudf/core/join/join.py | 5 +- .../cudf/cudf/tests/test_column_accessor.py | 2 +- 9 files changed, 80 insertions(+), 75 deletions(-) diff --git a/python/cudf/cudf/_lib/csv.pyx b/python/cudf/cudf/_lib/csv.pyx index a90fe0f9ac6..e0f57df1368 100644 --- a/python/cudf/cudf/_lib/csv.pyx +++ b/python/cudf/cudf/_lib/csv.pyx @@ -282,7 +282,7 @@ def read_csv( # Set index if the index_col parameter is passed if index_col is not None and index_col is not False: if isinstance(index_col, int): - index_col_name = df._data.select_by_index(index_col).names[0] + index_col_name = df._data.get_labels_by_index(index_col)[0] df = df.set_index(index_col_name) if isinstance(index_col_name, str) and \ names is None and orig_header == "infer": diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index d13351c49dd..a224e0ce0d0 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -1698,7 +1698,7 @@ def join( # in case of MultiIndex if isinstance(lhs, cudf.MultiIndex): on = ( - lhs._data.select_by_index(level).names[0] + lhs._data.get_labels_by_index(level)[0] if isinstance(level, int) else level ) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 67c19f11e41..7aa3e5f8163 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -102,7 +102,7 @@ def __init__( rangeindex: bool = False, label_dtype: Dtype | None = None, verify: bool = True, - ): + ) -> None: if isinstance(data, ColumnAccessor): self._data = data._data self._level_names = data.level_names @@ -147,10 +147,10 @@ def __iter__(self): def __getitem__(self, key: Any) -> ColumnBase: return self._data[key] - def __setitem__(self, key: Any, value: Any): + def __setitem__(self, key: Any, value: ColumnBase) -> None: self.set_by_label(key, value) - def __delitem__(self, key: Any): + def __delitem__(self, key: Any) -> None: old_ncols = len(self._data) del self._data[key] new_ncols = len(self._data) @@ -174,7 +174,7 @@ def __repr__(self) -> str: def _from_columns_like_self( self, columns: abc.Iterable[ColumnBase], verify: bool = True - ): + ) -> Self: """ Return a new ColumnAccessor with columns and the properties of self. @@ -250,7 +250,7 @@ def _grouped_data(self) -> abc.MutableMapping: else: return self._data - def _clear_cache(self, old_ncols: int, new_ncols: int): + def _clear_cache(self, old_ncols: int, new_ncols: int) -> None: """ Clear cached attributes. @@ -310,16 +310,14 @@ def to_pandas_index(self) -> pd.Index: ) return result - def insert( - self, name: Any, value: Any, loc: int = -1, validate: bool = True - ): + def insert(self, name: Any, value: ColumnBase, loc: int = -1) -> None: """ Insert column into the ColumnAccessor at the specified location. Parameters ---------- name : Name corresponding to the new column - value : column-like + value : ColumnBase loc : int, optional The location to insert the new value at. Must be (0 <= loc <= ncols). By default, the column is added @@ -330,33 +328,35 @@ def insert( None, this function operates in-place. """ name = self._pad_key(name) + if name in self._data: + raise ValueError(f"Cannot insert '{name}', already exists") old_ncols = len(self._data) if loc == -1: loc = old_ncols - if not (0 <= loc <= old_ncols): + elif not (0 <= loc <= old_ncols): raise ValueError( f"insert: loc out of bounds: must be 0 <= loc <= {old_ncols}" ) + + if not isinstance(value, column.ColumnBase): + raise ValueError("value must be a Column") + elif old_ncols > 0 and len(value) != self.nrows: + raise ValueError("All columns must be of equal length") + # TODO: we should move all insert logic here - if name in self._data: - raise ValueError(f"Cannot insert '{name}', already exists") if loc == old_ncols: - if validate: - value = column.as_column(value) - if old_ncols > 0 and len(value) != self.nrows: - raise ValueError("All columns must be of equal length") self._data[name] = value else: new_keys = self.names[:loc] + (name,) + self.names[loc:] new_values = self.columns[:loc] + (value,) + self.columns[loc:] - self._data = self._data.__class__(zip(new_keys, new_values)) + self._data = dict(zip(new_keys, new_values)) self._clear_cache(old_ncols, old_ncols + 1) if old_ncols == 0: # The type(name) may no longer match the prior label_dtype self.label_dtype = None - def copy(self, deep=False) -> ColumnAccessor: + def copy(self, deep: bool = False) -> Self: """ Make a copy of this ColumnAccessor. """ @@ -373,7 +373,7 @@ def copy(self, deep=False) -> ColumnAccessor: verify=False, ) - def select_by_label(self, key: Any) -> ColumnAccessor: + def select_by_label(self, key: Any) -> Self: """ Return a subset of this column accessor, composed of the keys specified by `key`. @@ -389,7 +389,7 @@ def select_by_label(self, key: Any) -> ColumnAccessor: if isinstance(key, slice): return self._select_by_label_slice(key) elif pd.api.types.is_list_like(key) and not isinstance(key, tuple): - return self._select_by_label_list_like(key) + return self._select_by_label_list_like(tuple(key)) else: if isinstance(key, tuple): if any(isinstance(k, slice) for k in key): @@ -427,9 +427,13 @@ def get_labels_by_index(self, index: Any) -> tuple: # TODO: Doesn't handle on-device columns return tuple(n for n, keep in zip(self.names, index) if keep) else: + if len(set(index)) != len(index): + raise NotImplementedError( + "Selecting duplicate column labels is not supported." + ) return tuple(self.names[i] for i in index) - def select_by_index(self, index: Any) -> ColumnAccessor: + def select_by_index(self, index: Any) -> Self: """ Return a ColumnAccessor composed of the columns specified by index. @@ -445,13 +449,15 @@ def select_by_index(self, index: Any) -> ColumnAccessor: """ keys = self.get_labels_by_index(index) data = {k: self._data[k] for k in keys} - return self.__class__( + return type(self)( data, multiindex=self.multiindex, level_names=self.level_names, + label_dtype=self.label_dtype, + verify=False, ) - def swaplevel(self, i=-2, j=-1): + def swaplevel(self, i=-2, j=-1) -> Self: """ Swap level i with level j. Calling this method does not change the ordering of the values. @@ -467,6 +473,10 @@ def swaplevel(self, i=-2, j=-1): ------- ColumnAccessor """ + if not self.multiindex: + raise ValueError( + "swaplevel is only valid for self.multiindex=True" + ) i = _get_level(i, self.nlevels, self.level_names) j = _get_level(j, self.nlevels, self.level_names) @@ -486,13 +496,16 @@ def swaplevel(self, i=-2, j=-1): new_names = list(self.level_names) new_names[i], new_names[j] = new_names[j], new_names[i] - return self.__class__( + return type(self)( new_data, - multiindex=True, + multiindex=self.multiindex, level_names=new_names, + rangeindex=self.rangeindex, + label_dtype=self.label_dtype, + verify=False, ) - def set_by_label(self, key: Any, value: Any, validate: bool = True): + def set_by_label(self, key: Any, value: ColumnBase) -> None: """ Add (or modify) column by name. @@ -500,26 +513,21 @@ def set_by_label(self, key: Any, value: Any, validate: bool = True): ---------- key name of the column - value : column-like + value : Column The value to insert into the column. - validate : bool - If True, the provided value will be coerced to a column and - validated before setting (Default value = True). """ key = self._pad_key(key) - if validate: - value = column.as_column(value) - if len(self._data) > 0 and len(value) != self.nrows: - raise ValueError("All columns must be of equal length") + if not isinstance(value, column.ColumnBase): + raise ValueError("value must be a Column") + if len(self) > 0 and len(value) != self.nrows: + raise ValueError("All columns must be of equal length") old_ncols = len(self._data) self._data[key] = value new_ncols = len(self._data) self._clear_cache(old_ncols, new_ncols) - def _select_by_label_list_like(self, key: Any) -> ColumnAccessor: - # Might be a generator - key = tuple(key) + def _select_by_label_list_like(self, key: tuple) -> Self: # Special-casing for boolean mask if (bn := len(key)) > 0 and all(map(is_bool, key)): if bn != (n := len(self.names)): @@ -539,19 +547,22 @@ def _select_by_label_list_like(self, key: Any) -> ColumnAccessor: ) if self.multiindex: data = dict(_to_flat_dict_inner(data)) - return self.__class__( + return type(self)( data, multiindex=self.multiindex, level_names=self.level_names, + label_dtype=self.label_dtype, + verify=False, ) - def _select_by_label_grouped(self, key: Any) -> ColumnAccessor: + def _select_by_label_grouped(self, key: Any) -> Self: result = self._grouped_data[key] if isinstance(result, column.ColumnBase): # self._grouped_data[key] = self._data[key] so skip validation - return self.__class__( + return type(self)( data={key: result}, multiindex=self.multiindex, + label_dtype=self.label_dtype, verify=False, ) else: @@ -563,9 +574,10 @@ def _select_by_label_grouped(self, key: Any) -> ColumnAccessor: result, multiindex=self.nlevels - len(key) > 1, level_names=self.level_names[len(key) :], + verify=False, ) - def _select_by_label_slice(self, key: slice) -> ColumnAccessor: + def _select_by_label_slice(self, key: slice) -> Self: start, stop = key.start, key.stop if key.step is not None: raise TypeError("Label slicing with step is not supported") @@ -585,19 +597,22 @@ def _select_by_label_slice(self, key: slice) -> ColumnAccessor: stop_idx = len(self.names) - idx break keys = self.names[start_idx:stop_idx] - return self.__class__( + return type(self)( {k: self._data[k] for k in keys}, multiindex=self.multiindex, level_names=self.level_names, + label_dtype=self.label_dtype, verify=False, ) - def _select_by_label_with_wildcard(self, key: Any) -> ColumnAccessor: + def _select_by_label_with_wildcard(self, key: tuple) -> Self: key = self._pad_key(key, slice(None)) - return self.__class__( - {k: self._data[k] for k in self._data if _keys_equal(k, key)}, + data = {k: self._data[k] for k in self.names if _keys_equal(k, key)} + return type(self)( + data, multiindex=self.multiindex, level_names=self.level_names, + label_dtype=self.label_dtype, verify=False, ) @@ -614,7 +629,7 @@ def _pad_key(self, key: Any, pad_value="") -> Any: def rename_levels( self, mapper: Mapping[Any, Any] | Callable, level: int | None = None - ) -> ColumnAccessor: + ) -> Self: """ Rename the specified levels of the given ColumnAccessor @@ -686,7 +701,7 @@ def rename_column(x): verify=False, ) - def droplevel(self, level): + def droplevel(self, level) -> None: # drop the nth level if level < 0: level += self.nlevels @@ -701,9 +716,8 @@ def droplevel(self, level): self._level_names[:level] + self._level_names[level + 1 :] ) - if ( - len(self._level_names) == 1 - ): # can't use nlevels, as it depends on multiindex + if len(self._level_names) == 1: + # can't use nlevels, as it depends on multiindex self.multiindex = False self._clear_cache(old_ncols, new_ncols) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 97684129203..43693ec20b1 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -382,19 +382,19 @@ def _setitem_tuple_arg(self, key, value): value = as_column(value, length=length) if isinstance(value, ColumnBase): - new_col = cudf.Series._from_column(value, index=idx) + new_ser = cudf.Series._from_column(value, index=idx) else: - new_col = cudf.Series(value, index=idx) + new_ser = cudf.Series(value, index=idx) if len(self._frame.index) != 0: - new_col = new_col._align_to_index( + new_ser = new_ser._align_to_index( self._frame.index, how="right" ) if len(self._frame.index) == 0: self._frame.index = ( - idx if idx is not None else cudf.RangeIndex(len(new_col)) + idx if idx is not None else cudf.RangeIndex(len(new_ser)) ) - self._frame._data.insert(key[1], new_col) + self._frame._data.insert(key[1], new_ser._column) else: if is_scalar(value): for col in columns_df._column_names: @@ -981,6 +981,7 @@ def _init_from_series_list(self, data, columns, index): self._data.rangeindex = isinstance( columns, (range, cudf.RangeIndex, pd.RangeIndex) ) + self._data.label_dtype = pd.Index(columns).dtype else: self._data.rangeindex = True @@ -3272,9 +3273,6 @@ def _insert(self, loc, name, value, nan_as_null=None, ignore_index=True): If False, a reindexing operation is performed if `value.index` is not equal to `self.index`. """ - if name in self._data: - raise NameError(f"duplicated column name {name}") - num_cols = self._num_columns if loc < 0: loc += num_cols + 1 diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index ce23d671a6c..3e1efd7c97a 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1010,9 +1010,7 @@ def _copy_type_metadata(self: Self, other: Self) -> Self: See `ColumnBase._with_type_metadata` for more information. """ for (name, col), (_, dtype) in zip(self._data.items(), other._dtypes): - self._data.set_by_label( - name, col._with_type_metadata(dtype), validate=False - ) + self._data.set_by_label(name, col._with_type_metadata(dtype)) return self diff --git a/python/cudf/cudf/core/indexing_utils.py b/python/cudf/cudf/core/indexing_utils.py index a0089242909..8182e5cede2 100644 --- a/python/cudf/cudf/core/indexing_utils.py +++ b/python/cudf/cudf/core/indexing_utils.py @@ -152,10 +152,6 @@ def destructure_dataframe_iloc_indexer( column_names: ColumnLabels = list( frame._data.get_labels_by_index(cols) ) - if len(set(column_names)) != len(column_names): - raise NotImplementedError( - "cudf DataFrames do not support repeated column names" - ) except TypeError: raise TypeError( "Column indices must be integers, slices, " diff --git a/python/cudf/cudf/core/join/_join_helpers.py b/python/cudf/cudf/core/join/_join_helpers.py index 32c84763401..854c44ff1a1 100644 --- a/python/cudf/cudf/core/join/_join_helpers.py +++ b/python/cudf/cudf/core/join/_join_helpers.py @@ -37,16 +37,16 @@ class _ColumnIndexer(_Indexer): def get(self, obj: cudf.DataFrame) -> ColumnBase: return obj._data[self.name] - def set(self, obj: cudf.DataFrame, value: ColumnBase, validate=False): - obj._data.set_by_label(self.name, value, validate=validate) + def set(self, obj: cudf.DataFrame, value: ColumnBase): + obj._data.set_by_label(self.name, value) class _IndexIndexer(_Indexer): def get(self, obj: cudf.DataFrame) -> ColumnBase: return obj.index._data[self.name] - def set(self, obj: cudf.DataFrame, value: ColumnBase, validate=False): - obj.index._data.set_by_label(self.name, value, validate=validate) + def set(self, obj: cudf.DataFrame, value: ColumnBase): + obj.index._data.set_by_label(self.name, value) def _match_join_keys( diff --git a/python/cudf/cudf/core/join/join.py b/python/cudf/cudf/core/join/join.py index ce81c1fc5b1..b65bc7af832 100644 --- a/python/cudf/cudf/core/join/join.py +++ b/python/cudf/cudf/core/join/join.py @@ -272,8 +272,8 @@ def perform_merge(self) -> cudf.DataFrame: lcol_casted = lcol_casted.astype("category") rcol_casted = rcol_casted.astype("category") - left_key.set(self.lhs, lcol_casted, validate=False) - right_key.set(self.rhs, rcol_casted, validate=False) + left_key.set(self.lhs, lcol_casted) + right_key.set(self.rhs, rcol_casted) left_rows, right_rows = self._gather_maps( left_join_cols, right_join_cols @@ -329,7 +329,6 @@ def _merge_results( lkey.set( left_result, lkey.get(left_result).fillna(rkey.get(right_result)), - validate=False, ) # All columns from the left table make it into the output. Non-key diff --git a/python/cudf/cudf/tests/test_column_accessor.py b/python/cudf/cudf/tests/test_column_accessor.py index 2d7bc809d4d..5cef077c18d 100644 --- a/python/cudf/cudf/tests/test_column_accessor.py +++ b/python/cudf/cudf/tests/test_column_accessor.py @@ -370,7 +370,7 @@ def test_replace_level_values_MultiColumn(): def test_clear_nrows_empty_before(): ca = ColumnAccessor({}) assert ca.nrows == 0 - ca.insert("new", [1]) + ca.insert("new", as_column([1])) assert ca.nrows == 1 From 8ab553c7835b21c2d5fcc76cb24960db03722b15 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 21 Aug 2024 08:47:13 -0400 Subject: [PATCH 101/270] Move libcudf reduction google-benchmarks to nvbench (#16564) Reworks the reduction benchmarks currently coded with google-bench to use nvbench instead. This removes the need to support `row_bit_count` for dictionary column types. #16121 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Mark Harris (https://github.com/harrism) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16564 --- cpp/benchmarks/CMakeLists.txt | 14 ++- cpp/benchmarks/reduction/anyall.cpp | 80 +++++++--------- cpp/benchmarks/reduction/dictionary.cpp | 111 ++++++++++++----------- cpp/benchmarks/reduction/minmax.cpp | 63 +++++-------- cpp/benchmarks/reduction/reduce.cpp | 116 ++++++++++++------------ cpp/benchmarks/reduction/scan.cpp | 65 ++++++------- 6 files changed, 210 insertions(+), 239 deletions(-) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 483b7b0a539..6db282a7728 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -207,12 +207,16 @@ ConfigureBench(TYPE_DISPATCHER_BENCH type_dispatcher/type_dispatcher.cu) # ################################################################################################## # * reduction benchmark --------------------------------------------------------------------------- -ConfigureBench( - REDUCTION_BENCH reduction/anyall.cpp reduction/dictionary.cpp reduction/minmax.cpp - reduction/reduce.cpp reduction/scan.cpp -) ConfigureNVBench( - REDUCTION_NVBENCH reduction/rank.cpp reduction/scan_structs.cpp reduction/segmented_reduce.cpp + REDUCTION_NVBENCH + reduction/anyall.cpp + reduction/dictionary.cpp + reduction/minmax.cpp + reduction/rank.cpp + reduction/reduce.cpp + reduction/scan.cpp + reduction/scan_structs.cpp + reduction/segmented_reduce.cpp ) # ################################################################################################## diff --git a/cpp/benchmarks/reduction/anyall.cpp b/cpp/benchmarks/reduction/anyall.cpp index e9d23881764..1e578fab181 100644 --- a/cpp/benchmarks/reduction/anyall.cpp +++ b/cpp/benchmarks/reduction/anyall.cpp @@ -16,65 +16,51 @@ #include #include -#include -#include -#include +#include #include #include #include -#include +#include -class Reduction : public cudf::benchmark {}; +#include -template -void BM_reduction_anyall(benchmark::State& state, - std::unique_ptr const& agg) +template +static void reduction_anyall(nvbench::state& state, nvbench::type_list) { - cudf::size_type const column_size{static_cast(state.range(0))}; - auto const dtype = cudf::type_to_id(); - data_profile const profile = data_profile_builder().no_validity().distribution( - dtype, distribution_id::UNIFORM, 0, agg->kind == cudf::aggregation::ANY ? 0 : 100); - auto const values = create_random_column(dtype, row_count{column_size}, profile); + auto const size = static_cast(state.get_int64("size")); + auto const kind_str = state.get_string("kind"); - cudf::data_type output_dtype{cudf::type_id::BOOL8}; + auto const input_type = cudf::type_to_id(); + auto const agg = kind_str == "any" ? cudf::make_any_aggregation() + : cudf::make_all_aggregation(); - for (auto _ : state) { - cuda_event_timer timer(state, true); - auto result = cudf::reduce(*values, *agg, output_dtype); - } + data_profile const profile = + data_profile_builder().no_validity().distribution(input_type, + distribution_id::UNIFORM, + (kind_str == "all" ? 1 : 0), + (kind_str == "any" ? 0 : 100)); + auto const values = create_random_column(input_type, row_count{size}, profile); - // The benchmark takes a column and produces one scalar. - set_items_processed(state, column_size + 1); - set_bytes_processed(state, estimate_size(values->view()) + cudf::size_of(output_dtype)); -} + auto const output_type = cudf::data_type{cudf::type_id::BOOL8}; + auto stream = cudf::get_default_stream(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); + state.add_element_count(size); + state.add_global_memory_reads(size); + state.add_global_memory_writes(1); -#define concat(a, b, c) a##b##c -#define get_agg(op) concat(cudf::make_, op, _aggregation()) + state.exec(nvbench::exec_tag::sync, [&values, output_type, &agg](nvbench::launch& launch) { + cudf::reduce(*values, *agg, output_type); + }); -// TYPE, OP -#define RBM_BENCHMARK_DEFINE(name, type, aggregation) \ - BENCHMARK_DEFINE_F(Reduction, name)(::benchmark::State & state) \ - { \ - BM_reduction_anyall(state, get_agg(aggregation)); \ - } \ - BENCHMARK_REGISTER_F(Reduction, name) \ - ->UseManualTime() \ - ->Arg(10000) /* 10k */ \ - ->Arg(100000) /* 100k */ \ - ->Arg(1000000) /* 1M */ \ - ->Arg(10000000) /* 10M */ \ - ->Arg(100000000); /* 100M */ + set_throughputs(state); +} -#define REDUCE_BENCHMARK_DEFINE(type, aggregation) \ - RBM_BENCHMARK_DEFINE(concat(type, _, aggregation), type, aggregation) +using Types = nvbench::type_list; -REDUCE_BENCHMARK_DEFINE(bool, all); -REDUCE_BENCHMARK_DEFINE(int8_t, all); -REDUCE_BENCHMARK_DEFINE(int32_t, all); -REDUCE_BENCHMARK_DEFINE(float, all); -REDUCE_BENCHMARK_DEFINE(bool, any); -REDUCE_BENCHMARK_DEFINE(int8_t, any); -REDUCE_BENCHMARK_DEFINE(int32_t, any); -REDUCE_BENCHMARK_DEFINE(float, any); +NVBENCH_BENCH_TYPES(reduction_anyall, NVBENCH_TYPE_AXES(Types)) + .set_name("anyall") + .set_type_axes_names({"DataType"}) + .add_string_axis("kind", {"any", "all"}) + .add_int64_axis("size", {100'000, 1'000'000, 10'000'000, 100'000'000}); diff --git a/cpp/benchmarks/reduction/dictionary.cpp b/cpp/benchmarks/reduction/dictionary.cpp index 5095337dbb3..1bdb50a539a 100644 --- a/cpp/benchmarks/reduction/dictionary.cpp +++ b/cpp/benchmarks/reduction/dictionary.cpp @@ -16,79 +16,84 @@ #include #include -#include -#include +#include +#include #include #include #include #include -class ReductionDictionary : public cudf::benchmark {}; +#include -template -void BM_reduction_dictionary(benchmark::State& state, - std::unique_ptr const& agg) +template +static std::unique_ptr make_reduce_aggregation() { - cudf::size_type const column_size{static_cast(state.range(0))}; + switch (kind) { + case cudf::reduce_aggregation::ANY: + return cudf::make_any_aggregation(); + case cudf::reduce_aggregation::ALL: + return cudf::make_all_aggregation(); + case cudf::reduce_aggregation::MIN: + return cudf::make_min_aggregation(); + case cudf::reduce_aggregation::MAX: + return cudf::make_max_aggregation(); + case cudf::reduce_aggregation::MEAN: + return cudf::make_mean_aggregation(); + default: CUDF_FAIL("Unsupported reduce aggregation in this benchmark"); + } +} + +template +static void reduction_dictionary(nvbench::state& state, + nvbench::type_list>) +{ + cudf::size_type const size{static_cast(state.get_int64("size"))}; - // int column and encoded dictionary column data_profile const profile = data_profile_builder().cardinality(0).no_validity().distribution( cudf::type_to_id(), distribution_id::UNIFORM, - (agg->kind == cudf::aggregation::ALL ? 1 : 0), - (agg->kind == cudf::aggregation::ANY ? 0 : 100)); - auto int_column = create_random_column(cudf::type_to_id(), row_count{column_size}, profile); - auto number_col = cudf::cast(*int_column, cudf::data_type{cudf::type_to_id()}); + (kind == cudf::aggregation::ALL ? 1 : 0), + (kind == cudf::aggregation::ANY ? 0 : 100)); + auto int_column = create_random_column(cudf::type_to_id(), row_count{size}, profile); + auto number_col = cudf::cast(*int_column, cudf::data_type{cudf::type_to_id()}); auto values = cudf::dictionary::encode(*number_col); - cudf::data_type output_dtype = [&] { - if (agg->kind == cudf::aggregation::ANY || agg->kind == cudf::aggregation::ALL) + cudf::data_type output_type = [&] { + if (kind == cudf::aggregation::ANY || kind == cudf::aggregation::ALL) { return cudf::data_type{cudf::type_id::BOOL8}; - if (agg->kind == cudf::aggregation::MEAN) return cudf::data_type{cudf::type_id::FLOAT64}; - return cudf::data_type{cudf::type_to_id()}; + } + if (kind == cudf::aggregation::MEAN) { return cudf::data_type{cudf::type_id::FLOAT64}; } + return cudf::data_type{cudf::type_to_id()}; }(); - for (auto _ : state) { - cuda_event_timer timer(state, true); - auto result = cudf::reduce(*values, *agg, output_dtype); + auto agg = make_reduce_aggregation(); + + auto stream = cudf::get_default_stream(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); + state.add_element_count(size); + state.add_global_memory_reads(size); + if (kind == cudf::aggregation::ANY || kind == cudf::aggregation::ALL) { + state.add_global_memory_writes(1); // BOOL8s + } else { + state.add_global_memory_writes(1); } - // The benchmark takes a column and produces two scalars. - set_items_processed(state, column_size + 1); + state.exec(nvbench::exec_tag::sync, [&values, output_type, &agg](nvbench::launch& launch) { + cudf::reduce(*values, *agg, output_type); + }); - // We don't set the metrics for the size read/written as row_bit_count() doesn't - // support the dictionary type yet (and so is estimate_size()). - // See https://github.com/rapidsai/cudf/issues/16121 for details. + set_throughputs(state); } -#define concat(a, b, c) a##b##c -#define get_agg(op) concat(cudf::make_, op, _aggregation()) - -// TYPE, OP -#define RBM_BENCHMARK_DEFINE(name, type, aggregation) \ - BENCHMARK_DEFINE_F(ReductionDictionary, name)(::benchmark::State & state) \ - { \ - BM_reduction_dictionary(state, get_agg(aggregation)); \ - } \ - BENCHMARK_REGISTER_F(ReductionDictionary, name) \ - ->UseManualTime() \ - ->Arg(10000) /* 10k */ \ - ->Arg(100000) /* 100k */ \ - ->Arg(1000000) /* 1M */ \ - ->Arg(10000000) /* 10M */ \ - ->Arg(100000000); /* 100M */ - -#define REDUCE_BENCHMARK_DEFINE(type, aggregation) \ - RBM_BENCHMARK_DEFINE(concat(type, _, aggregation), type, aggregation) +using Types = nvbench::type_list; +using AggKinds = nvbench::enum_type_list; -REDUCE_BENCHMARK_DEFINE(int32_t, all); -REDUCE_BENCHMARK_DEFINE(float, all); -REDUCE_BENCHMARK_DEFINE(int32_t, any); -REDUCE_BENCHMARK_DEFINE(float, any); -REDUCE_BENCHMARK_DEFINE(int32_t, min); -REDUCE_BENCHMARK_DEFINE(float, min); -REDUCE_BENCHMARK_DEFINE(int32_t, max); -REDUCE_BENCHMARK_DEFINE(float, max); -REDUCE_BENCHMARK_DEFINE(int32_t, mean); -REDUCE_BENCHMARK_DEFINE(float, mean); +NVBENCH_BENCH_TYPES(reduction_dictionary, NVBENCH_TYPE_AXES(Types, AggKinds)) + .set_name("reduction_dictionary") + .set_type_axes_names({"DataType", "AggKinds"}) + .add_int64_axis("size", {100'000, 1'000'000, 10'000'000, 100'000'000}); diff --git a/cpp/benchmarks/reduction/minmax.cpp b/cpp/benchmarks/reduction/minmax.cpp index 050f2887221..c89e22d3f44 100644 --- a/cpp/benchmarks/reduction/minmax.cpp +++ b/cpp/benchmarks/reduction/minmax.cpp @@ -16,55 +16,40 @@ #include #include -#include -#include -#include +#include #include #include #include -class Reduction : public cudf::benchmark {}; +#include -template -void BM_reduction(benchmark::State& state) +template +static void reduction_minmax(nvbench::state& state, nvbench::type_list) { - cudf::size_type const column_size{(cudf::size_type)state.range(0)}; - auto const dtype_id = cudf::type_to_id(); - auto const input_column = - create_random_column(dtype_id, row_count{column_size}, data_profile_builder().no_validity()); + auto const size = static_cast(state.get_int64("size")); - for (auto _ : state) { - cuda_event_timer timer(state, true); - auto result = cudf::minmax(*input_column); - } + auto const input_type = cudf::type_to_id(); - // The benchmark takes a column and produces two scalars. - set_items_processed(state, column_size + 2); - cudf::data_type dtype = cudf::data_type{dtype_id}; - set_bytes_processed(state, estimate_size(input_column->view()) + 2 * cudf::size_of(dtype)); -} + data_profile const profile = + data_profile_builder().no_validity().distribution(input_type, distribution_id::UNIFORM, 0, 100); + auto const input_column = create_random_column(input_type, row_count{size}, profile); + + auto stream = cudf::get_default_stream(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); + state.add_element_count(size); + state.add_global_memory_reads(size); + state.add_global_memory_writes(2); -#define concat(a, b, c) a##b##c -#define get_agg(op) concat(cudf::make_, op, _aggregation()) + state.exec(nvbench::exec_tag::sync, + [&input_column](nvbench::launch& launch) { cudf::minmax(*input_column); }); -// TYPE, OP -#define RBM_BENCHMARK_DEFINE(name, type, aggregation) \ - BENCHMARK_DEFINE_F(Reduction, name)(::benchmark::State & state) { BM_reduction(state); } \ - BENCHMARK_REGISTER_F(Reduction, name) \ - ->UseManualTime() \ - ->Arg(10000) /* 10k */ \ - ->Arg(100000) /* 100k */ \ - ->Arg(1000000) /* 1M */ \ - ->Arg(10000000) /* 10M */ \ - ->Arg(100000000); /* 100M */ + set_throughputs(state); +} -#define REDUCE_BENCHMARK_DEFINE(type, aggregation) \ - RBM_BENCHMARK_DEFINE(concat(type, _, aggregation), type, aggregation) +using Types = nvbench::type_list; -REDUCE_BENCHMARK_DEFINE(bool, minmax); -REDUCE_BENCHMARK_DEFINE(int8_t, minmax); -REDUCE_BENCHMARK_DEFINE(int32_t, minmax); -using cudf::timestamp_ms; -REDUCE_BENCHMARK_DEFINE(timestamp_ms, minmax); -REDUCE_BENCHMARK_DEFINE(float, minmax); +NVBENCH_BENCH_TYPES(reduction_minmax, NVBENCH_TYPE_AXES(Types)) + .set_name("minmax") + .set_type_axes_names({"DataType"}) + .add_int64_axis("size", {100'000, 1'000'000, 10'000'000, 100'000'000}); diff --git a/cpp/benchmarks/reduction/reduce.cpp b/cpp/benchmarks/reduction/reduce.cpp index 63c96f4fe9e..14bf90c4943 100644 --- a/cpp/benchmarks/reduction/reduce.cpp +++ b/cpp/benchmarks/reduction/reduce.cpp @@ -16,82 +16,80 @@ #include #include -#include -#include -#include +#include #include #include #include #include +#include + #include -class Reduction : public cudf::benchmark {}; +template +static std::unique_ptr make_reduce_aggregation() +{ + switch (kind) { + case cudf::reduce_aggregation::MIN: + return cudf::make_min_aggregation(); + case cudf::reduce_aggregation::SUM: + return cudf::make_sum_aggregation(); + case cudf::reduce_aggregation::MEAN: + return cudf::make_mean_aggregation(); + case cudf::reduce_aggregation::PRODUCT: + return cudf::make_product_aggregation(); + case cudf::reduce_aggregation::VARIANCE: + return cudf::make_variance_aggregation(); + case cudf::reduce_aggregation::STD: + return cudf::make_std_aggregation(); + default: CUDF_FAIL("Unsupported reduce aggregation in this benchmark"); + } +} -template -void BM_reduction(benchmark::State& state, std::unique_ptr const& agg) +template +static void reduction(nvbench::state& state, nvbench::type_list>) { - cudf::size_type const column_size{(cudf::size_type)state.range(0)}; - auto const dtype = cudf::type_to_id(); + auto const size = static_cast(state.get_int64("size")); + if (cudf::is_chrono() && kind != cudf::aggregation::MIN) { + state.skip("Skip chrono types for some aggregations"); + } + + auto const input_type = cudf::type_to_id(); data_profile const profile = - data_profile_builder().no_validity().distribution(dtype, distribution_id::UNIFORM, 0, 100); - auto const input_column = create_random_column(dtype, row_count{column_size}, profile); + data_profile_builder().no_validity().distribution(input_type, distribution_id::UNIFORM, 0, 100); + auto const input_column = create_random_column(input_type, row_count{size}, profile); - cudf::data_type output_dtype = - (agg->kind == cudf::aggregation::MEAN || agg->kind == cudf::aggregation::VARIANCE || - agg->kind == cudf::aggregation::STD) + cudf::data_type output_type = + (kind == cudf::aggregation::MEAN || kind == cudf::aggregation::VARIANCE || + kind == cudf::aggregation::STD) ? cudf::data_type{cudf::type_id::FLOAT64} : input_column->type(); - for (auto _ : state) { - cuda_event_timer timer(state, true); - auto result = cudf::reduce(*input_column, *agg, output_dtype); - } + auto agg = make_reduce_aggregation(); - // The benchmark takes a column and produces two scalars. - set_items_processed(state, column_size + 1); - set_bytes_processed(state, estimate_size(input_column->view()) + cudf::size_of(output_dtype)); -} + auto stream = cudf::get_default_stream(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); + state.add_element_count(size); + state.add_global_memory_reads(size); + state.add_global_memory_writes(1); -#define concat(a, b, c) a##b##c -#define get_agg(op) concat(cudf::make_, op, _aggregation()) + state.exec(nvbench::exec_tag::sync, [&input_column, output_type, &agg](nvbench::launch& launch) { + cudf::reduce(*input_column, *agg, output_type); + }); -// TYPE, OP -#define RBM_BENCHMARK_DEFINE(name, type, aggregation) \ - BENCHMARK_DEFINE_F(Reduction, name)(::benchmark::State & state) \ - { \ - BM_reduction(state, get_agg(aggregation)); \ - } \ - BENCHMARK_REGISTER_F(Reduction, name) \ - ->UseManualTime() \ - ->Arg(10000) /* 10k */ \ - ->Arg(100000) /* 100k */ \ - ->Arg(1000000) /* 1M */ \ - ->Arg(10000000) /* 10M */ \ - ->Arg(100000000); /* 100M */ - -#define REDUCE_BENCHMARK_DEFINE(type, aggregation) \ - RBM_BENCHMARK_DEFINE(concat(type, _, aggregation), type, aggregation) + set_throughputs(state); +} -#define REDUCE_BENCHMARK_NUMERIC(aggregation) \ - REDUCE_BENCHMARK_DEFINE(bool, aggregation); \ - REDUCE_BENCHMARK_DEFINE(int8_t, aggregation); \ - REDUCE_BENCHMARK_DEFINE(int32_t, aggregation); \ - REDUCE_BENCHMARK_DEFINE(int64_t, aggregation); \ - REDUCE_BENCHMARK_DEFINE(float, aggregation); \ - REDUCE_BENCHMARK_DEFINE(double, aggregation); +using Types = nvbench::type_list; +using AggKinds = nvbench::enum_type_list; -REDUCE_BENCHMARK_NUMERIC(sum); -REDUCE_BENCHMARK_DEFINE(int32_t, product); -REDUCE_BENCHMARK_DEFINE(float, product); -REDUCE_BENCHMARK_DEFINE(int64_t, min); -REDUCE_BENCHMARK_DEFINE(double, min); -using cudf::timestamp_ms; -REDUCE_BENCHMARK_DEFINE(timestamp_ms, min); -REDUCE_BENCHMARK_DEFINE(int8_t, mean); -REDUCE_BENCHMARK_DEFINE(float, mean); -REDUCE_BENCHMARK_DEFINE(int32_t, variance); -REDUCE_BENCHMARK_DEFINE(double, variance); -REDUCE_BENCHMARK_DEFINE(int64_t, std); -REDUCE_BENCHMARK_DEFINE(float, std); +NVBENCH_BENCH_TYPES(reduction, NVBENCH_TYPE_AXES(Types, AggKinds)) + .set_name("reduction") + .set_type_axes_names({"DataType", "AggKinds"}) + .add_int64_axis("size", {100'000, 1'000'000, 10'000'000, 100'000'000}); diff --git a/cpp/benchmarks/reduction/scan.cpp b/cpp/benchmarks/reduction/scan.cpp index dc05aad9807..f3d67a79498 100644 --- a/cpp/benchmarks/reduction/scan.cpp +++ b/cpp/benchmarks/reduction/scan.cpp @@ -16,9 +16,7 @@ #include #include -#include -#include -#include +#include #include #include @@ -26,43 +24,38 @@ #include #include -class ReductionScan : public cudf::benchmark {}; +#include -template -static void BM_reduction_scan(benchmark::State& state, bool include_nulls) +template +static void reduction_scan(nvbench::state& state, nvbench::type_list) { - cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; - auto const dtype = cudf::type_to_id(); - auto const column = create_random_column(dtype, row_count{n_rows}); - if (!include_nulls) column->set_null_mask(rmm::device_buffer{}, 0); + auto const size = static_cast(state.get_int64("size")); + auto const nulls = state.get_float64("nulls"); + auto const input_type = cudf::type_to_id(); - std::unique_ptr result = nullptr; - for (auto _ : state) { - cuda_event_timer timer(state, true); - result = cudf::scan( - *column, *cudf::make_min_aggregation(), cudf::scan_type::INCLUSIVE); - } + data_profile const profile = data_profile_builder().null_probability(nulls).distribution( + input_type, distribution_id::UNIFORM, 0, 100); + auto const input_column = create_random_column(input_type, row_count{size}, profile); - // The benchmark takes a column and produces a new column of the same size as input. - set_items_processed(state, n_rows * 2); - set_bytes_processed(state, estimate_size(column->view()) + estimate_size(result->view())); + auto agg = cudf::make_min_aggregation(); + + auto stream = cudf::get_default_stream(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); + state.add_element_count(size); + state.add_global_memory_reads(size); + state.add_global_memory_writes(1); + + state.exec(nvbench::exec_tag::sync, [&input_column, &agg](nvbench::launch& launch) { + cudf::scan(*input_column, *agg, cudf::scan_type::INCLUSIVE); + }); + + set_throughputs(state); } -#define SCAN_BENCHMARK_DEFINE(name, type, nulls) \ - BENCHMARK_DEFINE_F(ReductionScan, name) \ - (::benchmark::State & state) { BM_reduction_scan(state, nulls); } \ - BENCHMARK_REGISTER_F(ReductionScan, name) \ - ->UseManualTime() \ - ->Arg(10000) /* 10k */ \ - ->Arg(100000) /* 100k */ \ - ->Arg(1000000) /* 1M */ \ - ->Arg(10000000) /* 10M */ \ - ->Arg(100000000); /* 100M */ +using Types = nvbench::type_list; -SCAN_BENCHMARK_DEFINE(int8_no_nulls, int8_t, false); -SCAN_BENCHMARK_DEFINE(int32_no_nulls, int32_t, false); -SCAN_BENCHMARK_DEFINE(uint64_no_nulls, uint64_t, false); -SCAN_BENCHMARK_DEFINE(float_no_nulls, float, false); -SCAN_BENCHMARK_DEFINE(int16_nulls, int16_t, true); -SCAN_BENCHMARK_DEFINE(uint32_nulls, uint32_t, true); -SCAN_BENCHMARK_DEFINE(double_nulls, double, true); +NVBENCH_BENCH_TYPES(reduction_scan, NVBENCH_TYPE_AXES(Types)) + .set_name("scan") + .set_type_axes_names({"DataType"}) + .add_float64_axis("nulls", {0.0, 0.1}) + .add_int64_axis("size", {100'000, 1'000'000, 10'000'000, 100'000'000}); From 6a2f323ac2c53b32d8a1d47b36dd0d0786027a7c Mon Sep 17 00:00:00 2001 From: Nghia Truong <7416935+ttnghia@users.noreply.github.com> Date: Wed, 21 Aug 2024 07:35:44 -0700 Subject: [PATCH 102/270] Fix function parameters with common dependency modified during their evaluation (#16620) This fixes an issue in JNI C++ code. In particular, during a function call, the two passing parameters are evaluated using an index value, but that index is modified during evaluating one of the parameters, leading to out-of-bound access when evaluating the other. Authors: - Nghia Truong (https://github.com/ttnghia) Approvers: - Jason Lowe (https://github.com/jlowe) URL: https://github.com/rapidsai/cudf/pull/16620 --- java/src/main/native/src/TableJni.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index 76ca8c533ce..ecc551f1143 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -1037,9 +1037,9 @@ cudf::io::schema_element read_schema_element(int& index, // go to the next entry, so recursion can parse it. index++; for (int i = 0; i < num_children; i++) { + auto const name = std::string{names.get(index).get()}; child_elems.insert( - std::pair{names.get(index).get(), - cudf::jni::read_schema_element(index, children, names, types, scales)}); + std::pair{name, cudf::jni::read_schema_element(index, children, names, types, scales)}); } return cudf::io::schema_element{d_type, std::move(child_elems)}; } else { @@ -1830,9 +1830,9 @@ Java_ai_rapids_cudf_Table_readJSONFromDataSource(JNIEnv* env, std::map data_types; int at = 0; while (at < n_types.size()) { + auto const name = std::string{n_col_names.get(at).get()}; data_types.insert(std::pair{ - n_col_names.get(at).get(), - cudf::jni::read_schema_element(at, n_children, n_col_names, n_types, n_scales)}); + name, cudf::jni::read_schema_element(at, n_children, n_col_names, n_types, n_scales)}); } opts.dtypes(data_types); } else { @@ -1929,9 +1929,9 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON(JNIEnv* env, std::map data_types; int at = 0; while (at < n_types.size()) { + auto const name = std::string{n_col_names.get(at).get()}; data_types.insert(std::pair{ - n_col_names.get(at).get(), - cudf::jni::read_schema_element(at, n_children, n_col_names, n_types, n_scales)}); + name, cudf::jni::read_schema_element(at, n_children, n_col_names, n_types, n_scales)}); } opts.dtypes(data_types); } else { From bf2ee328f99cae51c8bdbc240e0ceedb102c24ca Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:47:11 -0700 Subject: [PATCH 103/270] DOC: Refresh pylibcudf guide (#15856) This PR updates the pylibcudf dev guide with some more recent recommendations. Authors: - Thomas Li (https://github.com/lithomas1) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/15856 --- docs/cudf/source/developer_guide/pylibcudf.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/cudf/source/developer_guide/pylibcudf.md b/docs/cudf/source/developer_guide/pylibcudf.md index 2ae545a4955..4e10459fe2b 100644 --- a/docs/cudf/source/developer_guide/pylibcudf.md +++ b/docs/cudf/source/developer_guide/pylibcudf.md @@ -13,10 +13,8 @@ To satisfy the goals of pylibcudf, we impose the following set of design princip - Every public function or method should be `cpdef`ed. This allows it to be used in both Cython and Python code. This incurs some slight overhead over `cdef` functions, but we assume that this is acceptable because 1) the vast majority of users will be using pure Python rather than Cython, and 2) the overhead of a `cpdef` function over a `cdef` function is on the order of a nanosecond, while CUDA kernel launch overhead is on the order of a microsecond, so these function overheads should be washed out by typical usage of pylibcudf. - Every variable used should be strongly typed and either be a primitive type (int, float, etc) or a cdef class. Any enums in C++ should be mirrored using `cpdef enum`, which will create both a C-style enum in Cython and a PEP 435-style Python enum that will automatically be used in Python. - All typing in code should be written using Cython syntax, not PEP 484 Python typing syntax. Not only does this ensure compatibility with Cython < 3, but even with Cython 3 PEP 484 support remains incomplete as of this writing. -- All cudf code should interact only with pylibcudf, never with libcudf directly. -- All imports should be relative so that pylibcudf can be easily extracted from cudf later - - Exception: All imports of libcudf API bindings in `cudf._lib.cpp` should use absolute imports of `cudf._lib.cpp as libcudf`. We should convert the `cpp` directory into a proper package so that it can be imported as `libcudf` in that fashion. When moving pylibcudf into a separate package, it will be renamed to `libcudf` and only the imports will need to change. -- Ideally, pylibcudf should depend on nothing other than rmm and pyarrow. This will allow it to be extracted into a a largely standalone library and used in environments where the larger dependency tree of cudf may be cumbersome. +- All cudf code should interact only with pylibcudf, never with libcudf directly. This is not currently the case, but is the direction that the library is moving towards. +- Ideally, pylibcudf should depend on no RAPIDS component other than rmm, and should in general have minimal runtime dependencies. ## Relationship to libcudf @@ -112,6 +110,9 @@ Then, a corresponding pylibcudf fixture may be created using a simple `from_arro This approach ensures consistent global coverage across types for various tests. In general, pylibcudf tests should prefer validating against a corresponding pyarrow implementation rather than hardcoding data. +If there is no pyarrow implementation, another alternative is to write a pure Python implementation that loops over the values +of the Table/Column, if a scalar Python equivalent of the pylibcudf implementation exists (this is especially relevant for string methods). + This approach is more resilient to changes to input data, particularly given the fixture strategy outlined above. Standard tools for comparing between pylibcudf and pyarrow types are provided in the utils module. @@ -242,3 +243,8 @@ cpdef ColumnOrTable empty_like(ColumnOrTable input) [Cython supports specializing the contents of fused-type functions based on the argument types](https://cython.readthedocs.io/en/latest/src/userguide/fusedtypes.html#type-checking-specializations), so any type-specific logic may be encoded using the appropriate conditionals. See the pylibcudf source for examples of how to implement such functions. + +In the event that libcudf provides multiple overloads for the same function with differing numbers of arguments, specify the maximum number of arguments in the Cython definition, +and set arguments not shared between overloads to `None`. If a user tries to pass in an unsupported argument for a specific overload type, you should raise `ValueError`. + +Finally, consider making an libcudf issue if you think this inconsistency can be addressed on the libcudf side. From 6c4905da22ad5b3d5007f45f38a3fa8449f7f8e1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 21 Aug 2024 21:03:12 -0700 Subject: [PATCH 104/270] Remove legacy Arrow interop APIs (#16590) Contributes to #15193. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Robert (Bobby) Evans (https://github.com/revans2) - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/16590 --- cpp/CMakeLists.txt | 3 - cpp/include/cudf/detail/interop.hpp | 101 +--- cpp/include/cudf/interop.hpp | 101 ---- cpp/src/interop/detail/arrow_allocator.cpp | 83 --- cpp/src/interop/detail/arrow_allocator.hpp | 31 -- cpp/src/interop/from_arrow.cu | 524 ------------------- cpp/src/interop/to_arrow.cu | 490 ----------------- cpp/tests/interop/arrow_utils.hpp | 64 ++- java/src/main/native/src/ColumnVectorJni.cpp | 76 ++- java/src/main/native/src/TableJni.cpp | 35 +- 10 files changed, 167 insertions(+), 1341 deletions(-) delete mode 100644 cpp/src/interop/detail/arrow_allocator.cpp delete mode 100644 cpp/src/interop/detail/arrow_allocator.hpp delete mode 100644 cpp/src/interop/from_arrow.cu delete mode 100644 cpp/src/interop/to_arrow.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ff00c484501..6b8bb26825b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -363,17 +363,14 @@ add_library( src/hash/sha512_hash.cu src/hash/xxhash_64.cu src/interop/dlpack.cpp - src/interop/from_arrow.cu src/interop/arrow_utilities.cpp src/interop/decimal_conversion_utilities.cu - src/interop/to_arrow.cu src/interop/to_arrow_device.cu src/interop/to_arrow_host.cu src/interop/from_arrow_device.cu src/interop/from_arrow_host.cu src/interop/from_arrow_stream.cu src/interop/to_arrow_schema.cpp - src/interop/detail/arrow_allocator.cpp src/io/avro/avro.cpp src/io/avro/avro_gpu.cu src/io/avro/reader_impl.cu diff --git a/cpp/include/cudf/detail/interop.hpp b/cpp/include/cudf/detail/interop.hpp index 0b9319ba663..0d8f078c9d1 100644 --- a/cpp/include/cudf/detail/interop.hpp +++ b/cpp/include/cudf/detail/interop.hpp @@ -16,29 +16,13 @@ #pragma once -// We disable warning 611 because the `arrow::TableBatchReader` only partially -// override the `ReadNext` method of `arrow::RecordBatchReader::ReadNext` -// triggering warning 611-D from nvcc. -#ifdef __CUDACC__ -#pragma nv_diag_suppress 611 -#pragma nv_diag_suppress 2810 -#endif -#include - -#include -#ifdef __CUDACC__ -#pragma nv_diag_default 611 -#pragma nv_diag_default 2810 -#endif - #include #include #include #include #include - -#include +#include namespace CUDF_EXPORT cudf { namespace detail { @@ -61,89 +45,6 @@ DLManagedTensor* to_dlpack(table_view const& input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); -// Creating arrow as per given type_id and buffer arguments -template -std::shared_ptr to_arrow_array(cudf::type_id id, Ts&&... args) -{ - switch (id) { - case type_id::BOOL8: return std::make_shared(std::forward(args)...); - case type_id::INT8: return std::make_shared(std::forward(args)...); - case type_id::INT16: return std::make_shared(std::forward(args)...); - case type_id::INT32: return std::make_shared(std::forward(args)...); - case type_id::INT64: return std::make_shared(std::forward(args)...); - case type_id::UINT8: return std::make_shared(std::forward(args)...); - case type_id::UINT16: return std::make_shared(std::forward(args)...); - case type_id::UINT32: return std::make_shared(std::forward(args)...); - case type_id::UINT64: return std::make_shared(std::forward(args)...); - case type_id::FLOAT32: return std::make_shared(std::forward(args)...); - case type_id::FLOAT64: return std::make_shared(std::forward(args)...); - case type_id::TIMESTAMP_DAYS: - return std::make_shared(std::make_shared(), - std::forward(args)...); - case type_id::TIMESTAMP_SECONDS: - return std::make_shared(arrow::timestamp(arrow::TimeUnit::SECOND), - std::forward(args)...); - case type_id::TIMESTAMP_MILLISECONDS: - return std::make_shared(arrow::timestamp(arrow::TimeUnit::MILLI), - std::forward(args)...); - case type_id::TIMESTAMP_MICROSECONDS: - return std::make_shared(arrow::timestamp(arrow::TimeUnit::MICRO), - std::forward(args)...); - case type_id::TIMESTAMP_NANOSECONDS: - return std::make_shared(arrow::timestamp(arrow::TimeUnit::NANO), - std::forward(args)...); - case type_id::DURATION_SECONDS: - return std::make_shared(arrow::duration(arrow::TimeUnit::SECOND), - std::forward(args)...); - case type_id::DURATION_MILLISECONDS: - return std::make_shared(arrow::duration(arrow::TimeUnit::MILLI), - std::forward(args)...); - case type_id::DURATION_MICROSECONDS: - return std::make_shared(arrow::duration(arrow::TimeUnit::MICRO), - std::forward(args)...); - case type_id::DURATION_NANOSECONDS: - return std::make_shared(arrow::duration(arrow::TimeUnit::NANO), - std::forward(args)...); - default: CUDF_FAIL("Unsupported type_id conversion to arrow"); - } -} - -// Converting arrow type to cudf type -data_type arrow_to_cudf_type(arrow::DataType const& arrow_type); - -/** - * @copydoc cudf::to_arrow(table_view input, std::vector const& metadata, - * rmm::cuda_stream_view stream, arrow::MemoryPool* ar_mr) - */ -std::shared_ptr to_arrow(table_view input, - std::vector const& metadata, - rmm::cuda_stream_view stream, - arrow::MemoryPool* ar_mr); - -/** - * @copydoc cudf::to_arrow(cudf::scalar const& input, column_metadata const& metadata, - * rmm::cuda_stream_view stream, arrow::MemoryPool* ar_mr) - */ -std::shared_ptr to_arrow(cudf::scalar const& input, - column_metadata const& metadata, - rmm::cuda_stream_view stream, - arrow::MemoryPool* ar_mr); -/** - * @copydoc cudf::from_arrow(arrow::Table const& input_table, rmm::cuda_stream_view stream, - * rmm::device_async_resource_ref mr) - */ -std::unique_ptr
from_arrow(arrow::Table const& input_table, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr); - -/** - * @copydoc cudf::from_arrow(arrow::Scalar const& input, rmm::cuda_stream_view stream, - * rmm::device_async_resource_ref mr) - */ -std::unique_ptr from_arrow(arrow::Scalar const& input, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr); - /** * @brief Return a maximum precision for a given type. * diff --git a/cpp/include/cudf/interop.hpp b/cpp/include/cudf/interop.hpp index 9a8f87b4a46..0f52b0f7b31 100644 --- a/cpp/include/cudf/interop.hpp +++ b/cpp/include/cudf/interop.hpp @@ -16,21 +16,6 @@ #pragma once -// We disable warning 611 because the `arrow::TableBatchReader` only partially -// override the `ReadNext` method of `arrow::RecordBatchReader::ReadNext` -// triggering warning 611-D from nvcc. -#ifdef __CUDACC__ -#pragma nv_diag_suppress 611 -#pragma nv_diag_suppress 2810 -#endif -#include - -#include -#ifdef __CUDACC__ -#pragma nv_diag_default 611 -#pragma nv_diag_default 2810 -#endif - #include #include #include @@ -131,59 +116,6 @@ struct column_metadata { column_metadata() = default; }; -/** - * @brief Create `arrow::Table` from cudf table `input` - * - * Converts the `cudf::table_view` to `arrow::Table` with the provided - * metadata `column_names`. - * - * @deprecated Since 24.08. Use cudf::to_arrow_host instead. - * - * @throws cudf::logic_error if `column_names` size doesn't match with number of columns. - * - * @param input table_view that needs to be converted to arrow Table - * @param metadata Contains hierarchy of names of columns and children - * @param stream CUDA stream used for device memory operations and kernel launches - * @param ar_mr arrow memory pool to allocate memory for arrow Table - * @return arrow Table generated from `input` - * - * @note For decimals, since the precision is not stored for them in libcudf, - * it will be converted to an Arrow decimal128 that has the widest-precision the cudf decimal type - * supports. For example, numeric::decimal32 will be converted to Arrow decimal128 of the precision - * 9 which is the maximum precision for 32-bit types. Similarly, numeric::decimal128 will be - * converted to Arrow decimal128 of the precision 38. - */ -[[deprecated("Use cudf::to_arrow_host")]] std::shared_ptr to_arrow( - table_view input, - std::vector const& metadata = {}, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - arrow::MemoryPool* ar_mr = arrow::default_memory_pool()); - -/** - * @brief Create `arrow::Scalar` from cudf scalar `input` - * - * Converts the `cudf::scalar` to `arrow::Scalar`. - * - * @deprecated Since 24.08. - * - * @param input scalar that needs to be converted to arrow Scalar - * @param metadata Contains hierarchy of names of columns and children - * @param stream CUDA stream used for device memory operations and kernel launches - * @param ar_mr arrow memory pool to allocate memory for arrow Scalar - * @return arrow Scalar generated from `input` - * - * @note For decimals, since the precision is not stored for them in libcudf, - * it will be converted to an Arrow decimal128 that has the widest-precision the cudf decimal type - * supports. For example, numeric::decimal32 will be converted to Arrow decimal128 of the precision - * 9 which is the maximum precision for 32-bit types. Similarly, numeric::decimal128 will be - * converted to Arrow decimal128 of the precision 38. - */ -[[deprecated("Use cudf::to_arrow_host")]] std::shared_ptr to_arrow( - cudf::scalar const& input, - column_metadata const& metadata = {}, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - arrow::MemoryPool* ar_mr = arrow::default_memory_pool()); - /** * @brief typedef for a unique_ptr to an ArrowSchema with custom deleter * @@ -386,39 +318,6 @@ unique_device_array_t to_arrow_host( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); -/** - * @brief Create `cudf::table` from given arrow Table input - * - * @deprecated Since 24.08. Use cudf::from_arrow_host instead. - * - * @param input arrow:Table that needs to be converted to `cudf::table` - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate `cudf::table` - * @return cudf table generated from given arrow Table - */ -[[deprecated("Use cudf::from_arrow_host")]] std::unique_ptr
from_arrow( - arrow::Table const& input, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); - -/** - * @brief Create `cudf::scalar` from given arrow Scalar input - * - * @deprecated Since 24.08. Use arrow's `MakeArrayFromScalar` on the - * input, followed by `ExportArray` to obtain something that can be - * consumed by `from_arrow_host`. Then use `cudf::get_element` to - * extract a device scalar from the column. - * - * @param input `arrow::Scalar` that needs to be converted to `cudf::scalar` - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate `cudf::scalar` - * @return cudf scalar generated from given arrow Scalar - */ -[[deprecated("See docstring for migration strategies")]] std::unique_ptr from_arrow( - arrow::Scalar const& input, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); - /** * @brief Create `cudf::table` from given ArrowArray and ArrowSchema input * diff --git a/cpp/src/interop/detail/arrow_allocator.cpp b/cpp/src/interop/detail/arrow_allocator.cpp deleted file mode 100644 index 2a19a5360fe..00000000000 --- a/cpp/src/interop/detail/arrow_allocator.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -#include - -namespace cudf { -namespace detail { - -/* - Enable Transparent Huge Pages (THP) for large (>4MB) allocations. - `buf` is returned untouched. - Enabling THP can improve performance of device-host memory transfers - significantly, see . -*/ -template -T enable_hugepage(T&& buf) -{ - if (buf->size() < (1u << 22u)) { // Smaller than 4 MB - return std::move(buf); - } - -#ifdef MADV_HUGEPAGE - auto const pagesize = sysconf(_SC_PAGESIZE); - void* addr = const_cast(buf->data()); - if (addr == nullptr) { return std::move(buf); } - auto length{static_cast(buf->size())}; - if (std::align(pagesize, pagesize, addr, length)) { - // Intentionally not checking for errors that may be returned by older kernel versions; - // optimistically tries enabling huge pages. - madvise(addr, length, MADV_HUGEPAGE); - } -#endif - return std::move(buf); -} - -std::unique_ptr allocate_arrow_buffer(int64_t const size, arrow::MemoryPool* ar_mr) -{ - /* - nvcc 11.0 generates Internal Compiler Error during codegen when arrow::AllocateBuffer - and `ValueOrDie` are used inside a CUDA compilation unit. - - To work around this issue we compile an allocation shim in C++ and use - that from our cuda sources - */ - arrow::Result> result = arrow::AllocateBuffer(size, ar_mr); - CUDF_EXPECTS(result.ok(), "Failed to allocate Arrow buffer"); - return enable_hugepage(std::move(result).ValueOrDie()); -} - -std::shared_ptr allocate_arrow_bitmap(int64_t const size, arrow::MemoryPool* ar_mr) -{ - /* - nvcc 11.0 generates Internal Compiler Error during codegen when arrow::AllocateBuffer - and `ValueOrDie` are used inside a CUDA compilation unit. - - To work around this issue we compile an allocation shim in C++ and use - that from our cuda sources - */ - arrow::Result> result = arrow::AllocateBitmap(size, ar_mr); - CUDF_EXPECTS(result.ok(), "Failed to allocate Arrow bitmap"); - return enable_hugepage(std::move(result).ValueOrDie()); -} - -} // namespace detail -} // namespace cudf diff --git a/cpp/src/interop/detail/arrow_allocator.hpp b/cpp/src/interop/detail/arrow_allocator.hpp deleted file mode 100644 index 75c1baa0dca..00000000000 --- a/cpp/src/interop/detail/arrow_allocator.hpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -namespace cudf { -namespace detail { - -// unique_ptr because that is what AllocateBuffer returns -std::unique_ptr allocate_arrow_buffer(int64_t const size, arrow::MemoryPool* ar_mr); - -// shared_ptr because that is what AllocateBitmap returns -std::shared_ptr allocate_arrow_bitmap(int64_t const size, arrow::MemoryPool* ar_mr); - -} // namespace detail -} // namespace cudf diff --git a/cpp/src/interop/from_arrow.cu b/cpp/src/interop/from_arrow.cu deleted file mode 100644 index 579820cbae3..00000000000 --- a/cpp/src/interop/from_arrow.cu +++ /dev/null @@ -1,524 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -namespace cudf { - -namespace detail { -data_type arrow_to_cudf_type(arrow::DataType const& arrow_type) -{ - switch (arrow_type.id()) { - case arrow::Type::NA: return data_type(type_id::EMPTY); - case arrow::Type::BOOL: return data_type(type_id::BOOL8); - case arrow::Type::INT8: return data_type(type_id::INT8); - case arrow::Type::INT16: return data_type(type_id::INT16); - case arrow::Type::INT32: return data_type(type_id::INT32); - case arrow::Type::INT64: return data_type(type_id::INT64); - case arrow::Type::UINT8: return data_type(type_id::UINT8); - case arrow::Type::UINT16: return data_type(type_id::UINT16); - case arrow::Type::UINT32: return data_type(type_id::UINT32); - case arrow::Type::UINT64: return data_type(type_id::UINT64); - case arrow::Type::FLOAT: return data_type(type_id::FLOAT32); - case arrow::Type::DOUBLE: return data_type(type_id::FLOAT64); - case arrow::Type::DATE32: return data_type(type_id::TIMESTAMP_DAYS); - case arrow::Type::TIMESTAMP: { - auto type = static_cast(&arrow_type); - switch (type->unit()) { - case arrow::TimeUnit::type::SECOND: return data_type(type_id::TIMESTAMP_SECONDS); - case arrow::TimeUnit::type::MILLI: return data_type(type_id::TIMESTAMP_MILLISECONDS); - case arrow::TimeUnit::type::MICRO: return data_type(type_id::TIMESTAMP_MICROSECONDS); - case arrow::TimeUnit::type::NANO: return data_type(type_id::TIMESTAMP_NANOSECONDS); - default: CUDF_FAIL("Unsupported timestamp unit in arrow"); - } - } - case arrow::Type::DURATION: { - auto type = static_cast(&arrow_type); - switch (type->unit()) { - case arrow::TimeUnit::type::SECOND: return data_type(type_id::DURATION_SECONDS); - case arrow::TimeUnit::type::MILLI: return data_type(type_id::DURATION_MILLISECONDS); - case arrow::TimeUnit::type::MICRO: return data_type(type_id::DURATION_MICROSECONDS); - case arrow::TimeUnit::type::NANO: return data_type(type_id::DURATION_NANOSECONDS); - default: CUDF_FAIL("Unsupported duration unit in arrow"); - } - } - case arrow::Type::STRING: return data_type(type_id::STRING); - case arrow::Type::LARGE_STRING: return data_type(type_id::STRING); - case arrow::Type::DICTIONARY: return data_type(type_id::DICTIONARY32); - case arrow::Type::LIST: return data_type(type_id::LIST); - case arrow::Type::DECIMAL: { - auto const type = static_cast(&arrow_type); - return data_type{type_id::DECIMAL128, -type->scale()}; - } - case arrow::Type::STRUCT: return data_type(type_id::STRUCT); - default: CUDF_FAIL("Unsupported type_id conversion to cudf"); - } -} - -namespace { -/** - * @brief Functor to return column for a corresponding arrow array. column - * is formed from buffer underneath the arrow array along with any offset and - * change in length that array has. - */ -struct dispatch_to_cudf_column { - /** - * @brief Returns mask from an array without any offsets. - */ - std::unique_ptr get_mask_buffer(arrow::Array const& array, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) - { - if (array.null_bitmap_data() == nullptr) { - return std::make_unique(0, stream, mr); - } - auto const null_bitmap_size = array.null_bitmap()->size(); - auto const allocation_size = - bitmask_allocation_size_bytes(static_cast(null_bitmap_size * CHAR_BIT)); - auto mask = std::make_unique(allocation_size, stream, mr); - auto mask_buffer = array.null_bitmap(); - CUDF_CUDA_TRY(cudaMemcpyAsync(mask->data(), - reinterpret_cast(mask_buffer->address()), - null_bitmap_size, - cudaMemcpyDefault, - stream.value())); - // Zero-initialize trailing padding bytes - auto const num_trailing_bytes = allocation_size - null_bitmap_size; - if (num_trailing_bytes > 0) { - auto trailing_bytes = static_cast(mask->data()) + null_bitmap_size; - CUDF_CUDA_TRY(cudaMemsetAsync(trailing_bytes, 0, num_trailing_bytes, stream.value())); - } - return mask; - } - - template ())> - std::unique_ptr operator()( - arrow::Array const&, data_type, bool, rmm::cuda_stream_view, rmm::device_async_resource_ref) - { - CUDF_FAIL("Unsupported type in from_arrow."); - } - - template ())> - std::unique_ptr operator()(arrow::Array const& array, - data_type type, - bool skip_mask, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) - { - auto data_buffer = array.data()->buffers[1]; - size_type const num_rows = array.length(); - auto const has_nulls = skip_mask ? false : array.null_bitmap_data() != nullptr; - auto col = make_fixed_width_column(type, num_rows, mask_state::UNALLOCATED, stream, mr); - auto mutable_column_view = col->mutable_view(); - CUDF_CUDA_TRY(cudaMemcpyAsync( - mutable_column_view.data(), - reinterpret_cast(data_buffer->address()) + array.offset() * sizeof(T), - sizeof(T) * num_rows, - cudaMemcpyDefault, - stream.value())); - if (has_nulls) { - auto tmp_mask = get_mask_buffer(array, stream, mr); - - // If array is sliced, we have to copy whole mask and then take copy. - auto out_mask = (num_rows == static_cast(data_buffer->size() / sizeof(T))) - ? std::move(*tmp_mask) - : cudf::detail::copy_bitmask(static_cast(tmp_mask->data()), - array.offset(), - array.offset() + num_rows, - stream, - mr); - - col->set_null_mask(std::move(out_mask), array.null_count()); - } - - return col; - } -}; - -std::unique_ptr get_empty_type_column(size_type size) -{ - // this abomination is required by cuDF Python, which needs to handle - // [PyArrow null arrays](https://arrow.apache.org/docs/python/generated/pyarrow.NullArray.html) - // of finite length - return std::make_unique( - data_type(type_id::EMPTY), size, rmm::device_buffer{}, rmm::device_buffer{}, size); -} - -/** - * @brief Returns cudf column formed from given arrow array - * This has been introduced to take care of compiler error "error: explicit specialization of - * function must precede its first use" - */ -std::unique_ptr get_column(arrow::Array const& array, - data_type type, - bool skip_mask, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr); - -template <> -std::unique_ptr dispatch_to_cudf_column::operator()( - arrow::Array const& array, - data_type type, - bool skip_mask, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - using DeviceType = __int128_t; - - auto data_buffer = array.data()->buffers[1]; - auto const num_rows = static_cast(array.length()); - auto col = make_fixed_width_column(type, num_rows, mask_state::UNALLOCATED, stream, mr); - auto mutable_column_view = col->mutable_view(); - - CUDF_CUDA_TRY(cudaMemcpyAsync( - mutable_column_view.data(), - reinterpret_cast(data_buffer->address()) + array.offset() * sizeof(DeviceType), - sizeof(DeviceType) * num_rows, - cudaMemcpyDefault, - stream.value())); - - auto null_mask = [&] { - if (not skip_mask and array.null_bitmap_data()) { - auto temp_mask = get_mask_buffer(array, stream, mr); - // If array is sliced, we have to copy whole mask and then take copy. - return (num_rows == static_cast(data_buffer->size() / sizeof(DeviceType))) - ? std::move(*temp_mask.release()) - : cudf::detail::copy_bitmask(static_cast(temp_mask->data()), - array.offset(), - array.offset() + num_rows, - stream, - mr); - } - return rmm::device_buffer{}; - }(); - - col->set_null_mask(std::move(null_mask), array.null_count()); - return col; -} - -template <> -std::unique_ptr dispatch_to_cudf_column::operator()(arrow::Array const& array, - data_type, - bool skip_mask, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto data_buffer = array.data()->buffers[1]; - // mask-to-bools expects the mask to be bitmask_type aligned/padded - auto data = rmm::device_buffer( - cudf::bitmask_allocation_size_bytes(data_buffer->size() * CHAR_BIT), stream, mr); - CUDF_CUDA_TRY(cudaMemcpyAsync(data.data(), - reinterpret_cast(data_buffer->address()), - data_buffer->size(), - cudaMemcpyDefault, - stream.value())); - auto out_col = mask_to_bools(static_cast(data.data()), - array.offset(), - array.offset() + array.length(), - stream, - mr); - - auto const has_nulls = skip_mask ? false : array.null_bitmap_data() != nullptr; - if (has_nulls) { - auto out_mask = - detail::copy_bitmask(static_cast(get_mask_buffer(array, stream, mr)->data()), - array.offset(), - array.offset() + array.length(), - stream, - mr); - - out_col->set_null_mask(std::move(out_mask), array.null_count()); - } - - return out_col; -} - -template <> -std::unique_ptr dispatch_to_cudf_column::operator()( - arrow::Array const& array, - data_type, - bool, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - if (array.length() == 0) { return make_empty_column(type_id::STRING); } - - std::unique_ptr offsets_column; - std::unique_ptr char_array; - - if (array.type_id() == arrow::Type::LARGE_STRING) { - auto str_array = static_cast(&array); - auto offset_array = std::make_unique( - str_array->value_offsets()->size() / sizeof(int64_t), str_array->value_offsets(), nullptr); - offsets_column = dispatch_to_cudf_column{}.operator()( - *offset_array, data_type(type_id::INT64), true, stream, mr); - char_array = std::make_unique( - str_array->value_data()->size(), str_array->value_data(), nullptr); - } else if (array.type_id() == arrow::Type::STRING) { - auto str_array = static_cast(&array); - auto offset_array = std::make_unique( - str_array->value_offsets()->size() / sizeof(int32_t), str_array->value_offsets(), nullptr); - offsets_column = dispatch_to_cudf_column{}.operator()( - *offset_array, data_type(type_id::INT32), true, stream, mr); - char_array = std::make_unique( - str_array->value_data()->size(), str_array->value_data(), nullptr); - } else { - throw std::runtime_error("Unsupported array type"); - } - - rmm::device_buffer chars(char_array->length(), stream, mr); - auto data_buffer = char_array->data()->buffers[1]; - CUDF_CUDA_TRY(cudaMemcpyAsync(chars.data(), - reinterpret_cast(data_buffer->address()), - chars.size(), - cudaMemcpyDefault, - stream.value())); - - auto const num_rows = offsets_column->size() - 1; - auto out_col = make_strings_column(num_rows, - std::move(offsets_column), - std::move(chars), - array.null_count(), - std::move(*get_mask_buffer(array, stream, mr))); - - return num_rows == array.length() - ? std::move(out_col) - : std::make_unique( - cudf::detail::slice(out_col->view(), - static_cast(array.offset()), - static_cast(array.offset() + array.length()), - stream), - stream, - mr); -} - -template <> -std::unique_ptr dispatch_to_cudf_column::operator()( - arrow::Array const& array, - data_type, - bool, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto dict_array = static_cast(&array); - auto dict_type = arrow_to_cudf_type(*(dict_array->dictionary()->type())); - auto keys_column = get_column(*(dict_array->dictionary()), dict_type, true, stream, mr); - auto ind_type = arrow_to_cudf_type(*(dict_array->indices()->type())); - - auto indices_column = get_column(*(dict_array->indices()), ind_type, false, stream, mr); - // If index type is not of type uint32_t, then cast it to uint32_t - auto const dict_indices_type = data_type{type_id::UINT32}; - if (indices_column->type().id() != dict_indices_type.id()) - indices_column = cudf::detail::cast(indices_column->view(), dict_indices_type, stream, mr); - - // Child columns shouldn't have masks and we need the mask in main column - auto column_contents = indices_column->release(); - indices_column = std::make_unique(dict_indices_type, - static_cast(array.length()), - std::move(*(column_contents.data)), - rmm::device_buffer{}, - 0); - - return make_dictionary_column(std::move(keys_column), - std::move(indices_column), - std::move(*(column_contents.null_mask)), - array.null_count()); -} - -template <> -std::unique_ptr dispatch_to_cudf_column::operator()( - arrow::Array const& array, - data_type, - bool, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto struct_array = static_cast(&array); - std::vector> child_columns; - // Offsets have already been applied to child - arrow::ArrayVector array_children = struct_array->fields(); - std::transform(array_children.cbegin(), - array_children.cend(), - std::back_inserter(child_columns), - [&mr, &stream](auto const& child_array) { - auto type = arrow_to_cudf_type(*(child_array->type())); - return get_column(*child_array, type, false, stream, mr); - }); - - auto out_mask = std::move(*(get_mask_buffer(array, stream, mr))); - if (struct_array->null_bitmap_data() != nullptr) { - out_mask = detail::copy_bitmask(static_cast(out_mask.data()), - array.offset(), - array.offset() + array.length(), - stream, - mr); - } - - return make_structs_column( - array.length(), move(child_columns), array.null_count(), std::move(out_mask), stream, mr); -} - -template <> -std::unique_ptr dispatch_to_cudf_column::operator()( - arrow::Array const& array, - data_type, - bool, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto list_array = static_cast(&array); - auto offset_array = std::make_unique( - list_array->value_offsets()->size() / sizeof(int32_t), list_array->value_offsets(), nullptr); - auto offsets_column = dispatch_to_cudf_column{}.operator()( - *offset_array, data_type(type_id::INT32), true, stream, mr); - - auto child_type = arrow_to_cudf_type(*(list_array->values()->type())); - auto child_column = get_column(*(list_array->values()), child_type, false, stream, mr); - - auto const num_rows = offsets_column->size() - 1; - auto out_col = make_lists_column(num_rows, - std::move(offsets_column), - std::move(child_column), - array.null_count(), - std::move(*get_mask_buffer(array, stream, mr)), - stream, - mr); - - return num_rows == array.length() - ? std::move(out_col) - : std::make_unique( - cudf::detail::slice(out_col->view(), - static_cast(array.offset()), - static_cast(array.offset() + array.length()), - stream), - stream, - mr); -} - -std::unique_ptr get_column(arrow::Array const& array, - data_type type, - bool skip_mask, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - return type.id() != type_id::EMPTY - ? type_dispatcher(type, dispatch_to_cudf_column{}, array, type, skip_mask, stream, mr) - : get_empty_type_column(array.length()); -} - -} // namespace - -std::unique_ptr
from_arrow(arrow::Table const& input_table, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - if (input_table.num_columns() == 0) { return std::make_unique
(); } - std::vector> columns; - auto chunked_arrays = input_table.columns(); - std::transform(chunked_arrays.begin(), - chunked_arrays.end(), - std::back_inserter(columns), - [&mr, &stream](auto const& chunked_array) { - std::vector> concat_columns; - auto cudf_type = arrow_to_cudf_type(*(chunked_array->type())); - auto array_chunks = chunked_array->chunks(); - if (cudf_type.id() == type_id::EMPTY) { - return get_empty_type_column(chunked_array->length()); - } - std::transform(array_chunks.begin(), - array_chunks.end(), - std::back_inserter(concat_columns), - [&cudf_type, &mr, &stream](auto const& array_chunk) { - return get_column(*array_chunk, cudf_type, false, stream, mr); - }); - if (concat_columns.empty()) { - return std::make_unique( - cudf_type, 0, rmm::device_buffer{}, rmm::device_buffer{}, 0); - } else if (concat_columns.size() == 1) { - return std::move(concat_columns[0]); - } - - std::vector column_views; - std::transform(concat_columns.begin(), - concat_columns.end(), - std::back_inserter(column_views), - [](auto const& col) { return col->view(); }); - return cudf::detail::concatenate(column_views, stream, mr); - }); - - return std::make_unique
(std::move(columns)); -} - -std::unique_ptr from_arrow(arrow::Scalar const& input, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto maybe_array = arrow::MakeArrayFromScalar(input, 1); - if (!maybe_array.ok()) { CUDF_FAIL("Failed to create array"); } - auto array = *maybe_array; - - auto field = arrow::field("", input.type); - - auto table = arrow::Table::Make(arrow::schema({field}), {array}); - - auto cudf_table = detail::from_arrow(*table, stream, mr); - - auto cv = cudf_table->view().column(0); - return get_element(cv, 0, stream); -} - -} // namespace detail - -std::unique_ptr
from_arrow(arrow::Table const& input_table, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - CUDF_FUNC_RANGE(); - - return detail::from_arrow(input_table, stream, mr); -} - -std::unique_ptr from_arrow(arrow::Scalar const& input, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - CUDF_FUNC_RANGE(); - - return detail::from_arrow(input, stream, mr); -} -} // namespace cudf diff --git a/cpp/src/interop/to_arrow.cu b/cpp/src/interop/to_arrow.cu deleted file mode 100644 index a867d4adfa1..00000000000 --- a/cpp/src/interop/to_arrow.cu +++ /dev/null @@ -1,490 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "arrow_utilities.hpp" -#include "decimal_conversion_utilities.cuh" -#include "detail/arrow_allocator.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -namespace cudf { -namespace detail { -namespace { - -/** - * @brief Create arrow data buffer from given cudf column - */ -template -std::shared_ptr fetch_data_buffer(device_span input, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - int64_t const data_size_in_bytes = sizeof(T) * input.size(); - - auto data_buffer = allocate_arrow_buffer(data_size_in_bytes, ar_mr); - - CUDF_CUDA_TRY(cudaMemcpyAsync(data_buffer->mutable_data(), - input.data(), - data_size_in_bytes, - cudaMemcpyDefault, - stream.value())); - - return std::move(data_buffer); -} - -/** - * @brief Create arrow buffer of mask from given cudf column - */ -std::shared_ptr fetch_mask_buffer(column_view input_view, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - int64_t const mask_size_in_bytes = cudf::bitmask_allocation_size_bytes(input_view.size()); - - if (input_view.has_nulls()) { - auto mask_buffer = allocate_arrow_bitmap(static_cast(input_view.size()), ar_mr); - CUDF_CUDA_TRY(cudaMemcpyAsync( - mask_buffer->mutable_data(), - (input_view.offset() > 0) - ? cudf::detail::copy_bitmask(input_view, stream, rmm::mr::get_current_device_resource()) - .data() - : input_view.null_mask(), - mask_size_in_bytes, - cudaMemcpyDefault, - stream.value())); - - // Resets all padded bits to 0 - mask_buffer->ZeroPadding(); - - return mask_buffer; - } - - return nullptr; -} - -/** - * @brief Functor to convert cudf column to arrow array - */ -struct dispatch_to_arrow { - /** - * @brief Creates vector Arrays from given cudf column children - */ - std::vector> fetch_child_array( - column_view input_view, - std::vector const& metadata, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) - { - std::vector> child_arrays; - std::transform( - input_view.child_begin(), - input_view.child_end(), - metadata.begin(), - std::back_inserter(child_arrays), - [&ar_mr, &stream](auto const& child, auto const& meta) { - return type_dispatcher( - child.type(), dispatch_to_arrow{}, child, child.type().id(), meta, ar_mr, stream); - }); - return child_arrays; - } - - template ())> - std::shared_ptr operator()( - column_view, cudf::type_id, column_metadata const&, arrow::MemoryPool*, rmm::cuda_stream_view) - { - CUDF_FAIL("Unsupported type for to_arrow."); - } - - template ())> - std::shared_ptr operator()(column_view input_view, - cudf::type_id id, - column_metadata const&, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) - { - return to_arrow_array( - id, - static_cast(input_view.size()), - fetch_data_buffer( - device_span(input_view.data(), input_view.size()), ar_mr, stream), - fetch_mask_buffer(input_view, ar_mr, stream), - static_cast(input_view.null_count())); - } -}; - -// Convert decimal types from libcudf to arrow where those types are not -// directly supported by Arrow. These types must be fit into 128 bits, the -// smallest decimal resolution supported by Arrow. -template -std::shared_ptr unsupported_decimals_to_arrow(column_view input, - int32_t precision, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - auto buf = detail::convert_decimals_to_decimal128( - input, stream, rmm::mr::get_current_device_resource()); - - // Synchronize stream here to ensure the decimal128 buffer is ready. - stream.synchronize(); - - auto const buf_size_in_bytes = buf->size(); - auto data_buffer = allocate_arrow_buffer(buf_size_in_bytes, ar_mr); - - CUDF_CUDA_TRY(cudaMemcpyAsync(data_buffer->mutable_data(), - buf->data(), - buf_size_in_bytes, - cudaMemcpyDefault, - stream.value())); - - auto type = arrow::decimal(precision, -input.type().scale()); - auto mask = fetch_mask_buffer(input, ar_mr, stream); - auto buffers = std::vector>{mask, std::move(data_buffer)}; - auto data = std::make_shared(type, input.size(), buffers); - - return std::make_shared(data); -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()( - column_view input, - cudf::type_id, - column_metadata const&, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - using DeviceType = int32_t; - return unsupported_decimals_to_arrow( - input, cudf::detail::max_precision(), ar_mr, stream); -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()( - column_view input, - cudf::type_id, - column_metadata const&, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - using DeviceType = int64_t; - return unsupported_decimals_to_arrow( - input, cudf::detail::max_precision(), ar_mr, stream); -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()( - column_view input, - cudf::type_id, - column_metadata const&, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - using DeviceType = __int128_t; - auto const max_precision = cudf::detail::max_precision(); - - rmm::device_uvector buf(input.size(), stream); - - thrust::copy(rmm::exec_policy(stream), // - input.begin(), - input.end(), - buf.begin()); - - auto const buf_size_in_bytes = buf.size() * sizeof(DeviceType); - auto data_buffer = allocate_arrow_buffer(buf_size_in_bytes, ar_mr); - - CUDF_CUDA_TRY(cudaMemcpyAsync( - data_buffer->mutable_data(), buf.data(), buf_size_in_bytes, cudaMemcpyDefault, stream.value())); - - auto type = arrow::decimal(max_precision, -input.type().scale()); - auto mask = fetch_mask_buffer(input, ar_mr, stream); - auto buffers = std::vector>{mask, std::move(data_buffer)}; - auto data = std::make_shared(type, input.size(), buffers); - - return std::make_shared(data); -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()(column_view input, - cudf::type_id id, - column_metadata const&, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - auto bitmask = detail::bools_to_mask(input, stream, rmm::mr::get_current_device_resource()); - - auto data_buffer = allocate_arrow_buffer(static_cast(bitmask.first->size()), ar_mr); - - CUDF_CUDA_TRY(cudaMemcpyAsync(data_buffer->mutable_data(), - bitmask.first->data(), - bitmask.first->size(), - cudaMemcpyDefault, - stream.value())); - return to_arrow_array(id, - static_cast(input.size()), - std::move(data_buffer), - fetch_mask_buffer(input, ar_mr, stream), - static_cast(input.null_count())); -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()( - column_view input, - cudf::type_id, - column_metadata const&, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - std::unique_ptr tmp_column = - ((input.offset() != 0) or - ((input.num_children() == 1) and (input.child(0).size() - 1 != input.size()))) - ? std::make_unique(input, stream) - : nullptr; - - column_view input_view = (tmp_column != nullptr) ? tmp_column->view() : input; - auto child_arrays = fetch_child_array(input_view, {{}, {}}, ar_mr, stream); - if (child_arrays.empty()) { - // Empty string will have only one value in offset of 4 bytes - auto tmp_offset_buffer = allocate_arrow_buffer(sizeof(int32_t), ar_mr); - auto tmp_data_buffer = allocate_arrow_buffer(0, ar_mr); - memset(tmp_offset_buffer->mutable_data(), 0, sizeof(int32_t)); - - return std::make_shared( - 0, std::move(tmp_offset_buffer), std::move(tmp_data_buffer)); - } - auto offset_buffer = child_arrays[strings_column_view::offsets_column_index]->data()->buffers[1]; - auto const sview = strings_column_view{input_view}; - auto data_buffer = fetch_data_buffer( - device_span{sview.chars_begin(stream), - static_cast(sview.chars_size(stream))}, - ar_mr, - stream); - if (sview.offsets().type().id() == cudf::type_id::INT64) { - return std::make_shared(static_cast(input_view.size()), - offset_buffer, - data_buffer, - fetch_mask_buffer(input_view, ar_mr, stream), - static_cast(input_view.null_count())); - } else { - return std::make_shared(static_cast(input_view.size()), - offset_buffer, - data_buffer, - fetch_mask_buffer(input_view, ar_mr, stream), - static_cast(input_view.null_count())); - } -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()( - column_view input, - cudf::type_id, - column_metadata const& metadata, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - CUDF_EXPECTS(metadata.children_meta.size() == static_cast(input.num_children()), - "Number of field names and number of children doesn't match\n"); - std::unique_ptr tmp_column = nullptr; - - if (input.offset() != 0) { tmp_column = std::make_unique(input, stream); } - - column_view input_view = (tmp_column != nullptr) ? tmp_column->view() : input; - auto child_arrays = fetch_child_array(input_view, metadata.children_meta, ar_mr, stream); - auto mask = fetch_mask_buffer(input_view, ar_mr, stream); - - std::vector> fields; - std::transform(child_arrays.cbegin(), - child_arrays.cend(), - metadata.children_meta.cbegin(), - std::back_inserter(fields), - [](auto const array, auto const meta) { - return std::make_shared( - meta.name, array->type(), array->null_count() > 0); - }); - auto dtype = std::make_shared(fields); - - return std::make_shared(dtype, - static_cast(input_view.size()), - child_arrays, - mask, - static_cast(input_view.null_count())); -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()( - column_view input, - cudf::type_id, - column_metadata const& metadata, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - CUDF_EXPECTS(metadata.children_meta.empty() || - metadata.children_meta.size() == static_cast(input.num_children()), - "Number of field names and number of children do not match\n"); - std::unique_ptr tmp_column = nullptr; - if ((input.offset() != 0) or - ((input.num_children() == 2) and (input.child(0).size() - 1 != input.size()))) { - tmp_column = std::make_unique(input, stream); - } - - column_view input_view = (tmp_column != nullptr) ? tmp_column->view() : input; - auto children_meta = - metadata.children_meta.empty() ? std::vector{{}, {}} : metadata.children_meta; - auto child_arrays = fetch_child_array(input_view, children_meta, ar_mr, stream); - if (child_arrays.empty() || child_arrays[0]->data()->length == 0) { - auto element_type = child_arrays.empty() ? arrow::null() : child_arrays[1]->type(); - auto result = arrow::MakeEmptyArray(arrow::list(element_type), ar_mr); - CUDF_EXPECTS(result.ok(), "Failed to construct empty arrow list array\n"); - return result.ValueUnsafe(); - } - - auto offset_buffer = child_arrays[0]->data()->buffers[1]; - auto data = child_arrays[1]; - return std::make_shared(arrow::list(data->type()), - static_cast(input_view.size()), - offset_buffer, - data, - fetch_mask_buffer(input_view, ar_mr, stream), - static_cast(input_view.null_count())); -} - -template <> -std::shared_ptr dispatch_to_arrow::operator()( - column_view input, - cudf::type_id, - column_metadata const& metadata, - arrow::MemoryPool* ar_mr, - rmm::cuda_stream_view stream) -{ - // Arrow dictionary requires indices to be signed integer - std::unique_ptr dict_indices = - detail::cast(cudf::dictionary_column_view(input).get_indices_annotated(), - cudf::data_type{type_id::INT32}, - stream, - rmm::mr::get_current_device_resource()); - auto indices = dispatch_to_arrow{}.operator()( - dict_indices->view(), dict_indices->type().id(), {}, ar_mr, stream); - auto dict_keys = cudf::dictionary_column_view(input).keys(); - auto dictionary = - type_dispatcher(dict_keys.type(), - dispatch_to_arrow{}, - dict_keys, - dict_keys.type().id(), - metadata.children_meta.empty() ? column_metadata{} : metadata.children_meta[0], - ar_mr, - stream); - - return std::make_shared( - arrow::dictionary(indices->type(), dictionary->type()), indices, dictionary); -} -} // namespace - -std::shared_ptr to_arrow(table_view input, - std::vector const& metadata, - rmm::cuda_stream_view stream, - arrow::MemoryPool* ar_mr) -{ - CUDF_EXPECTS((metadata.size() == static_cast(input.num_columns())), - "columns' metadata should be equal to number of columns in table"); - - std::vector> arrays; - std::vector> fields; - - std::transform( - input.begin(), - input.end(), - metadata.begin(), - std::back_inserter(arrays), - [&](auto const& c, auto const& meta) { - return c.type().id() != type_id::EMPTY - ? type_dispatcher( - c.type(), detail::dispatch_to_arrow{}, c, c.type().id(), meta, ar_mr, stream) - : std::make_shared(c.size()); - }); - - std::transform( - arrays.begin(), - arrays.end(), - metadata.begin(), - std::back_inserter(fields), - [](auto const& array, auto const& meta) { return arrow::field(meta.name, array->type()); }); - - auto result = arrow::Table::Make(arrow::schema(fields), arrays); - - // synchronize the stream because after the return the data may be accessed from the host before - // the above `cudaMemcpyAsync` calls have completed their copies (especially if pinned host - // memory is used). - stream.synchronize(); - - return result; -} - -std::shared_ptr to_arrow(cudf::scalar const& input, - column_metadata const& metadata, - rmm::cuda_stream_view stream, - arrow::MemoryPool* ar_mr) -{ - auto const column = cudf::make_column_from_scalar(input, 1, stream); - cudf::table_view const tv{{column->view()}}; - auto const arrow_table = detail::to_arrow(tv, {metadata}, stream, ar_mr); - auto const ac = arrow_table->column(0); - auto const maybe_scalar = ac->GetScalar(0); - if (!maybe_scalar.ok()) { CUDF_FAIL("Failed to produce a scalar"); } - return maybe_scalar.ValueOrDie(); -} -} // namespace detail - -std::shared_ptr to_arrow(table_view input, - std::vector const& metadata, - rmm::cuda_stream_view stream, - arrow::MemoryPool* ar_mr) -{ - CUDF_FUNC_RANGE(); - return detail::to_arrow(input, metadata, stream, ar_mr); -} - -std::shared_ptr to_arrow(cudf::scalar const& input, - column_metadata const& metadata, - rmm::cuda_stream_view stream, - arrow::MemoryPool* ar_mr) -{ - CUDF_FUNC_RANGE(); - return detail::to_arrow(input, metadata, stream, ar_mr); -} -} // namespace cudf diff --git a/cpp/tests/interop/arrow_utils.hpp b/cpp/tests/interop/arrow_utils.hpp index 08eada632a5..70a9fe64d70 100644 --- a/cpp/tests/interop/arrow_utils.hpp +++ b/cpp/tests/interop/arrow_utils.hpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#pragma once + #include #include #include @@ -30,11 +32,65 @@ #include #include +#include #include -#include - -#pragma once +// Creating arrow as per given type_id and buffer arguments +template +std::shared_ptr to_arrow_array(cudf::type_id id, Ts&&... args) +{ + switch (id) { + case cudf::type_id::BOOL8: + return std::make_shared(std::forward(args)...); + case cudf::type_id::INT8: return std::make_shared(std::forward(args)...); + case cudf::type_id::INT16: + return std::make_shared(std::forward(args)...); + case cudf::type_id::INT32: + return std::make_shared(std::forward(args)...); + case cudf::type_id::INT64: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT8: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT16: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT32: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT64: + return std::make_shared(std::forward(args)...); + case cudf::type_id::FLOAT32: + return std::make_shared(std::forward(args)...); + case cudf::type_id::FLOAT64: + return std::make_shared(std::forward(args)...); + case cudf::type_id::TIMESTAMP_DAYS: + return std::make_shared(std::make_shared(), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_SECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::SECOND), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_MILLISECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::MILLI), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_MICROSECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::MICRO), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_NANOSECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::NANO), + std::forward(args)...); + case cudf::type_id::DURATION_SECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::SECOND), + std::forward(args)...); + case cudf::type_id::DURATION_MILLISECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::MILLI), + std::forward(args)...); + case cudf::type_id::DURATION_MICROSECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::MICRO), + std::forward(args)...); + case cudf::type_id::DURATION_NANOSECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::NANO), + std::forward(args)...); + default: CUDF_FAIL("Unsupported type_id conversion to arrow"); + } +} template std::enable_if_t() and !std::is_same_v, @@ -50,7 +106,7 @@ get_arrow_array(std::vector const& data, std::vector const& mask = { std::shared_ptr mask_buffer = mask.empty() ? nullptr : arrow::internal::BytesToBits(mask).ValueOrDie(); - return cudf::detail::to_arrow_array(cudf::type_to_id(), data.size(), data_buffer, mask_buffer); + return to_arrow_array(cudf::type_to_id(), data.size(), data_buffer, mask_buffer); } template diff --git a/java/src/main/native/src/ColumnVectorJni.cpp b/java/src/main/native/src/ColumnVectorJni.cpp index cdc5aa41abe..9b718b2ed83 100644 --- a/java/src/main/native/src/ColumnVectorJni.cpp +++ b/java/src/main/native/src/ColumnVectorJni.cpp @@ -38,12 +38,70 @@ #include #include +#include #include using cudf::jni::ptr_as_jlong; using cudf::jni::release_as_jlong; +// Creating arrow as per given type_id and buffer arguments +template +std::shared_ptr to_arrow_array(cudf::type_id id, Ts&&... args) +{ + switch (id) { + case cudf::type_id::BOOL8: + return std::make_shared(std::forward(args)...); + case cudf::type_id::INT8: return std::make_shared(std::forward(args)...); + case cudf::type_id::INT16: + return std::make_shared(std::forward(args)...); + case cudf::type_id::INT32: + return std::make_shared(std::forward(args)...); + case cudf::type_id::INT64: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT8: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT16: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT32: + return std::make_shared(std::forward(args)...); + case cudf::type_id::UINT64: + return std::make_shared(std::forward(args)...); + case cudf::type_id::FLOAT32: + return std::make_shared(std::forward(args)...); + case cudf::type_id::FLOAT64: + return std::make_shared(std::forward(args)...); + case cudf::type_id::TIMESTAMP_DAYS: + return std::make_shared(std::make_shared(), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_SECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::SECOND), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_MILLISECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::MILLI), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_MICROSECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::MICRO), + std::forward(args)...); + case cudf::type_id::TIMESTAMP_NANOSECONDS: + return std::make_shared(arrow::timestamp(arrow::TimeUnit::NANO), + std::forward(args)...); + case cudf::type_id::DURATION_SECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::SECOND), + std::forward(args)...); + case cudf::type_id::DURATION_MILLISECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::MILLI), + std::forward(args)...); + case cudf::type_id::DURATION_MICROSECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::MICRO), + std::forward(args)...); + case cudf::type_id::DURATION_NANOSECONDS: + return std::make_shared(arrow::duration(arrow::TimeUnit::NANO), + std::forward(args)...); + default: CUDF_FAIL("Unsupported type_id conversion to arrow"); + } +} + extern "C" { JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_sequence( @@ -141,15 +199,27 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_fromArrow(JNIEnv* env, break; default: // this handles the primitive types - arrow_array = cudf::detail::to_arrow_array( - n_type, j_col_length, data_buffer, null_buffer, j_null_count); + arrow_array = to_arrow_array(n_type, j_col_length, data_buffer, null_buffer, j_null_count); } auto name_and_type = arrow::field("col", arrow_array->type()); std::vector> fields = {name_and_type}; std::shared_ptr schema = std::make_shared(fields); auto arrow_table = arrow::Table::Make(schema, std::vector>{arrow_array}); - auto retCols = cudf::from_arrow(*(arrow_table))->release(); + + ArrowSchema sch; + if (!arrow::ExportSchema(*arrow_table->schema(), &sch).ok()) { + JNI_THROW_NEW(env, "java/lang/RuntimeException", "Unable to produce an ArrowSchema", 0) + } + auto batch = arrow_table->CombineChunksToBatch().ValueOrDie(); + ArrowArray arr; + if (!arrow::ExportRecordBatch(*batch, &arr).ok()) { + JNI_THROW_NEW(env, "java/lang/RuntimeException", "Unable to produce an ArrowArray", 0) + } + auto retCols = cudf::from_arrow(&sch, &arr)->release(); + arr.release(&arr); + sch.release(&sch); + if (retCols.size() != 1) { JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "Must result in one column", 0); } diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index ecc551f1143..c749c8c84bf 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -54,6 +54,8 @@ #include +#include +#include #include #include @@ -1069,6 +1071,15 @@ void append_flattened_child_names(cudf::io::column_name_info const& info, } } +// Recursively make schema and its children nullable +void set_nullable(ArrowSchema* schema) +{ + schema->flags |= ARROW_FLAG_NULLABLE; + for (int i = 0; i < schema->n_children; ++i) { + set_nullable(schema->children[i]); + } +} + } // namespace } // namespace jni @@ -2635,7 +2646,13 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_convertCudfToArrowTable(JNIEnv // The pointer to the shared_ptr<> is returned as a jlong. using result_t = std::shared_ptr; - auto result = cudf::to_arrow(*tview, state->get_column_metadata(*tview)); + auto got_arrow_schema = cudf::to_arrow_schema(*tview, state->get_column_metadata(*tview)); + cudf::jni::set_nullable(got_arrow_schema.get()); + auto got_arrow_array = cudf::to_arrow_host(*tview); + auto batch = + arrow::ImportRecordBatch(&got_arrow_array->array, got_arrow_schema.get()).ValueOrDie(); + auto result = arrow::Table::FromRecordBatches({batch}).ValueOrDie(); + return ptr_as_jlong(new result_t{result}); } CATCH_STD(env, 0) @@ -2746,7 +2763,21 @@ Java_ai_rapids_cudf_Table_convertArrowTableToCudf(JNIEnv* env, jclass, jlong arr try { cudf::jni::auto_set_device(env); - return convert_table_for_return(env, cudf::from_arrow(*(handle->get()))); + + ArrowSchema sch; + if (!arrow::ExportSchema(*handle->get()->schema(), &sch).ok()) { + JNI_THROW_NEW(env, "java/lang/RuntimeException", "Unable to produce an ArrowSchema", 0) + } + auto batch = handle->get()->CombineChunksToBatch().ValueOrDie(); + ArrowArray arr; + if (!arrow::ExportRecordBatch(*batch, &arr).ok()) { + JNI_THROW_NEW(env, "java/lang/RuntimeException", "Unable to produce an ArrowArray", 0) + } + auto ret = cudf::from_arrow(&sch, &arr); + arr.release(&arr); + sch.release(&sch); + + return convert_table_for_return(env, ret); } CATCH_STD(env, 0) } From 1fd96756daf90b8d2f901fe19a168e9d11974c0b Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Wed, 21 Aug 2024 21:10:20 -0700 Subject: [PATCH 105/270] Fix overflow bug in low-memory JSON reader (#16632) Bug fix for #16627. Changes byte range offsets and sizes from `size_type` to `size_t` in pylibcudf. Authors: - Shruti Shivakumar (https://github.com/shrshi) Approvers: - Bradley Dice (https://github.com/bdice) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16632 --- python/pylibcudf/pylibcudf/io/json.pxd | 4 ++-- python/pylibcudf/pylibcudf/io/json.pyx | 12 ++++++------ python/pylibcudf/pylibcudf/libcudf/io/json.pxd | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/python/pylibcudf/pylibcudf/io/json.pxd b/python/pylibcudf/pylibcudf/io/json.pxd index ab9b5b99ce2..f65c1034598 100644 --- a/python/pylibcudf/pylibcudf/io/json.pxd +++ b/python/pylibcudf/pylibcudf/io/json.pxd @@ -15,8 +15,8 @@ cpdef TableWithMetadata read_json( list dtypes = *, compression_type compression = *, bool lines = *, - size_type byte_range_offset = *, - size_type byte_range_size = *, + size_t byte_range_offset = *, + size_t byte_range_size = *, bool keep_quotes = *, bool mixed_types_as_string = *, bool prune_columns = *, diff --git a/python/pylibcudf/pylibcudf/io/json.pyx b/python/pylibcudf/pylibcudf/io/json.pyx index ce086f4a489..29e49083bc6 100644 --- a/python/pylibcudf/pylibcudf/io/json.pyx +++ b/python/pylibcudf/pylibcudf/io/json.pyx @@ -51,8 +51,8 @@ cdef json_reader_options _setup_json_reader_options( list dtypes, compression_type compression, bool lines, - size_type byte_range_offset, - size_type byte_range_size, + size_t byte_range_offset, + size_t byte_range_size, bool keep_quotes, bool mixed_types_as_string, bool prune_columns, @@ -189,8 +189,8 @@ cpdef TableWithMetadata read_json( list dtypes = None, compression_type compression = compression_type.AUTO, bool lines = False, - size_type byte_range_offset = 0, - size_type byte_range_size = 0, + size_t byte_range_offset = 0, + size_t byte_range_size = 0, bool keep_quotes = False, bool mixed_types_as_string = False, bool prune_columns = False, @@ -212,9 +212,9 @@ cpdef TableWithMetadata read_json( (column_child_name, column_child_type, list of grandchild dtypes). compression: CompressionType, default CompressionType.AUTO The compression format of the JSON source. - byte_range_offset : size_type, default 0 + byte_range_offset : size_t, default 0 Number of bytes to skip from source start. - byte_range_size : size_type, default 0 + byte_range_size : size_t, default 0 Number of bytes to read. By default, will read all bytes. keep_quotes : bool, default False Whether the reader should keep quotes of string values. diff --git a/python/pylibcudf/pylibcudf/libcudf/io/json.pxd b/python/pylibcudf/pylibcudf/libcudf/io/json.pxd index 7514e6c5258..1c74f8ca3ac 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/json.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/json.pxd @@ -27,8 +27,8 @@ cdef extern from "cudf/io/json.hpp" \ cudf_io_types.source_info get_source() except + vector[string] get_dtypes() except + cudf_io_types.compression_type get_compression() except + - size_type get_byte_range_offset() except + - size_type get_byte_range_size() except + + size_t get_byte_range_offset() except + + size_t get_byte_range_size() except + bool is_enabled_lines() except + bool is_enabled_mixed_types_as_string() except + bool is_enabled_prune_columns() except + @@ -41,8 +41,8 @@ cdef extern from "cudf/io/json.hpp" \ void set_compression( cudf_io_types.compression_type compression ) except + - void set_byte_range_offset(size_type offset) except + - void set_byte_range_size(size_type size) except + + void set_byte_range_offset(size_t offset) except + + void set_byte_range_size(size_t size) except + void enable_lines(bool val) except + void enable_mixed_types_as_string(bool val) except + void enable_prune_columns(bool val) except + @@ -73,10 +73,10 @@ cdef extern from "cudf/io/json.hpp" \ cudf_io_types.compression_type compression ) except + json_reader_options_builder& byte_range_offset( - size_type offset + size_t offset ) except + json_reader_options_builder& byte_range_size( - size_type size + size_t size ) except + json_reader_options_builder& lines( bool val From 00ff2ee5ec2fd23c65e759dc2f9d2907a1c9cb00 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Thu, 22 Aug 2024 10:35:27 -0700 Subject: [PATCH 106/270] [FEA] Add filesystem argument to `cudf.read_parquet` (#16577) This PR adds a `filesystem` kwarg to `cudf.read_parquet` (in alignment with [the pandas API](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html)). When a user has already constructed an `fsspec.AbstractFileSystem` object outside of cudf, they can now pass that object in to `read_parquet` to avoid redundant (and possibly inconsistent) filesystem inference. This PR also makes it possible for us to remove [explicit remote-IO handling from dask-cudf](https://github.com/rapidsai/cudf/blob/623dfceb42eb3e73b352b295898ff3e6cfe7c865/python/dask_cudf/dask_cudf/io/parquet.py#L100) (and consolidate the logic in cudf/ioutils). Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Mads R. B. Kristensen (https://github.com/madsbk) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16577 --- python/cudf/cudf/io/parquet.py | 5 ++- python/cudf/cudf/tests/test_s3.py | 22 ++++++++++ python/cudf/cudf/utils/ioutils.py | 54 ++++++++++++++++++------ python/dask_cudf/dask_cudf/io/parquet.py | 23 +++------- 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index fac51a9e471..560f257c115 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -527,6 +527,7 @@ def read_parquet( engine="cudf", columns=None, storage_options=None, + filesystem=None, filters=None, row_groups=None, use_pandas_metadata=True, @@ -567,7 +568,9 @@ def read_parquet( # Start by trying construct a filesystem object, so we # can apply filters on remote file-systems fs, paths = ioutils._get_filesystem_and_paths( - path_or_data=filepath_or_buffer, storage_options=storage_options + path_or_data=filepath_or_buffer, + storage_options=storage_options, + filesystem=filesystem, ) # Normalize and validate filters diff --git a/python/cudf/cudf/tests/test_s3.py b/python/cudf/cudf/tests/test_s3.py index 6579fd23634..3b23a53091e 100644 --- a/python/cudf/cudf/tests/test_s3.py +++ b/python/cudf/cudf/tests/test_s3.py @@ -269,6 +269,28 @@ def test_read_parquet_ext( assert_eq(expect, got1) +def test_read_parquet_filesystem(s3_base, s3so, pdf): + fname = "data.0.parquet" + # NOTE: Need a unique bucket name when a glob pattern + # is used, otherwise fsspec seems to cache the bucket + # contents, and later tests using the same bucket name + # will fail. + bucket = "test_read_parquet_filesystem" + buffer = BytesIO() + pdf.to_parquet(path=buffer) + buffer.seek(0) + fs = get_fs_token_paths("s3://", mode="rb", storage_options=s3so)[0] + with s3_context( + s3_base=s3_base, + bucket=bucket, + files={fname: buffer}, + ): + # Check that a glob pattern works + path = f"s3://{bucket}/{'data.*.parquet'}" + got = cudf.read_parquet(path, filesystem=fs) + assert_eq(pdf, got) + + def test_read_parquet_multi_file(s3_base, s3so, pdf): fname_1 = "test_parquet_reader_multi_file_1.parquet" buffer_1 = BytesIO() diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 4ac9b63985f..18106e7475b 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -12,7 +12,7 @@ import fsspec.implementations.local import numpy as np import pandas as pd -from fsspec.core import get_fs_token_paths +from fsspec.core import expand_paths_if_needed, get_fs_token_paths from cudf.core._compat import PANDAS_LT_300 from cudf.utils.docutils import docfmt_partial @@ -139,6 +139,9 @@ For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more details. +filesystem : fsspec.AbstractFileSystem, default None + Filesystem object to use when reading the parquet data. This argument + should not be used at the same time as `storage_options`. filters : list of tuple, list of lists of tuples, default None If not None, specifies a filter predicate used to filter out row groups using statistics stored for each row group as Parquet metadata. Row groups @@ -1536,11 +1539,18 @@ def is_directory(path_or_data, storage_options=None): return False -def _get_filesystem_and_paths(path_or_data, storage_options): +def _get_filesystem_and_paths( + path_or_data, + storage_options, + *, + filesystem=None, +): # Returns a filesystem object and the filesystem-normalized # paths. If `path_or_data` does not correspond to a path or # list of paths (or if the protocol is not supported), the # return will be `None` for the fs and `[]` for the paths. + # If a filesystem object is already available, it can be + # passed with the `filesystem` argument. fs = None return_paths = path_or_data @@ -1557,16 +1567,36 @@ def _get_filesystem_and_paths(path_or_data, storage_options): else: path_or_data = [path_or_data] - try: - fs, _, fs_paths = get_fs_token_paths( - path_or_data, mode="rb", storage_options=storage_options - ) - return_paths = fs_paths - except ValueError as e: - if str(e).startswith("Protocol not known"): - return None, [] - else: - raise e + if filesystem is None: + try: + fs, _, fs_paths = get_fs_token_paths( + path_or_data, mode="rb", storage_options=storage_options + ) + return_paths = fs_paths + except ValueError as e: + if str(e).startswith("Protocol not known"): + return None, [] + else: + raise e + else: + if not isinstance(filesystem, fsspec.AbstractFileSystem): + raise ValueError( + f"Expected fsspec.AbstractFileSystem. Got {filesystem}" + ) + + if storage_options: + raise ValueError( + f"Cannot specify storage_options when an explicit " + f"filesystem object is specified. Got: {storage_options}" + ) + + fs = filesystem + return_paths = [ + fs._strip_protocol(u) + for u in expand_paths_if_needed( + path_or_data, "rb", 1, fs, None + ) + ] return fs, return_paths diff --git a/python/dask_cudf/dask_cudf/io/parquet.py b/python/dask_cudf/dask_cudf/io/parquet.py index 8f52fce7818..c025280c240 100644 --- a/python/dask_cudf/dask_cudf/io/parquet.py +++ b/python/dask_cudf/dask_cudf/io/parquet.py @@ -23,11 +23,7 @@ from cudf.io import write_to_dataset from cudf.io.parquet import _apply_post_filters, _normalize_filters from cudf.utils.dtypes import cudf_dtype_from_pa_type -from cudf.utils.ioutils import ( - _ROW_GROUP_SIZE_BYTES_DEFAULT, - _fsspec_data_transfer, - _is_local_filesystem, -) +from cudf.utils.ioutils import _ROW_GROUP_SIZE_BYTES_DEFAULT class CudfEngine(ArrowDatasetEngine): @@ -93,40 +89,35 @@ def _read_paths( dataset_kwargs = dataset_kwargs or {} dataset_kwargs["partitioning"] = partitioning or "hive" - # Non-local filesystem handling - paths_or_fobs = paths - if not _is_local_filesystem(fs): - paths_or_fobs = [ - _fsspec_data_transfer(fpath, fs=fs) for fpath in paths - ] - # Use cudf to read in data try: df = cudf.read_parquet( - paths_or_fobs, + paths, engine="cudf", columns=columns, row_groups=row_groups if row_groups else None, dataset_kwargs=dataset_kwargs, categorical_partitions=False, + filesystem=fs, **kwargs, ) except RuntimeError as err: # TODO: Remove try/except after null-schema issue is resolved # (See: https://github.com/rapidsai/cudf/issues/12702) - if len(paths_or_fobs) > 1: + if len(paths) > 1: df = cudf.concat( [ cudf.read_parquet( - pof, + path, engine="cudf", columns=columns, row_groups=row_groups[i] if row_groups else None, dataset_kwargs=dataset_kwargs, categorical_partitions=False, + filesystem=fs, **kwargs, ) - for i, pof in enumerate(paths_or_fobs) + for i, path in enumerate(paths) ] ) else: From 81d71fce73306ae88bee1c78ed1f88e10916ad17 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:56:09 -0500 Subject: [PATCH 107/270] update-version.sh fix (#16629) Updates the `update-version.sh` script to include missed version updates. --- ci/release/update-version.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 132e58249e6..e79a91510b8 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -51,6 +51,7 @@ DEPENDENCIES=( kvikio libkvikio librmm + pylibcudf rapids-dask-dependency rmm ) @@ -77,7 +78,7 @@ for FILE in .github/workflows/*.yaml .github/workflows/*.yml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" "${FILE}" done -sed_runner "s/branch-[0-9]+\.[0-9]+/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cudf_polars.sh +sed_runner "s/branch-[0-9]\+\.[0-9]\+/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cudf_polars.sh # Java files NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}-SNAPSHOT" From e4e867aace96b80fccf030cc02a11f89cbb9c05f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:29:44 -1000 Subject: [PATCH 108/270] Annotate `ColumnAccessor._data` labels as `Hashable` (#16623) The motivating change here is that since we store a dictionary of columns in `ColumnAccessor`, the labels should be `collections.abc.Hashable` and therefore we can type methods that select by key with this annotation. This led to a mypy-typing-validation cascade that made me type the output of `def as_column(...) -> ColumnBase` which also lead to typing validation in several other files. Namely there no logic changes here. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16623 --- python/cudf/cudf/_lib/column.pyi | 2 +- python/cudf/cudf/core/_internals/timezones.py | 2 +- python/cudf/cudf/core/column/categorical.py | 6 +- python/cudf/cudf/core/column/column.py | 22 ++++-- python/cudf/cudf/core/column/lists.py | 7 +- python/cudf/cudf/core/column/numerical.py | 4 +- python/cudf/cudf/core/column/string.py | 8 +- python/cudf/cudf/core/column_accessor.py | 76 +++++++++++-------- python/cudf/cudf/core/copy_types.py | 19 +++-- python/cudf/cudf/core/dataframe.py | 2 +- python/cudf/cudf/core/indexed_frame.py | 26 ++++--- 11 files changed, 105 insertions(+), 69 deletions(-) diff --git a/python/cudf/cudf/_lib/column.pyi b/python/cudf/cudf/_lib/column.pyi index bcab009c102..bb38488eefb 100644 --- a/python/cudf/cudf/_lib/column.pyi +++ b/python/cudf/cudf/_lib/column.pyi @@ -54,7 +54,7 @@ class Column: @property def mask_ptr(self) -> int: ... def set_base_mask(self, value: Buffer | None) -> None: ... - def set_mask(self, value: Buffer | None) -> Self: ... + def set_mask(self, value: ColumnBase | Buffer | None) -> Self: ... @property def null_count(self) -> int: ... @property diff --git a/python/cudf/cudf/core/_internals/timezones.py b/python/cudf/cudf/core/_internals/timezones.py index 29cb9d7bd12..fd89904e766 100644 --- a/python/cudf/cudf/core/_internals/timezones.py +++ b/python/cudf/cudf/core/_internals/timezones.py @@ -120,7 +120,7 @@ def _read_tzfile_as_columns( # this happens for UTC-like zones min_date = np.int64(np.iinfo("int64").min + 1).astype("M8[s]") - return (as_column([min_date]), as_column([np.timedelta64(0, "s")])) + return (as_column([min_date]), as_column([np.timedelta64(0, "s")])) # type: ignore[return-value] return tuple(transition_times_and_offsets) # type: ignore[return-value] diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 1fdaf9f8c07..a7e98e5218f 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -984,9 +984,9 @@ def find_and_replace( ) replacement_col = catmap._data["index"].astype(replaced.codes.dtype) - replaced = column.as_column(replaced.codes) + replaced_codes = column.as_column(replaced.codes) output = libcudf.replace.replace( - replaced, to_replace_col, replacement_col + replaced_codes, to_replace_col, replacement_col ) result = column.build_categorical_column( @@ -1064,7 +1064,7 @@ def _validate_fillna_value( raise TypeError( "Cannot set a categorical with non-categorical data" ) - fill_value = fill_value._set_categories( + fill_value = cast(CategoricalColumn, fill_value)._set_categories( self.categories, ) return fill_value.codes.astype(self.codes.dtype) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 27278120abb..60b4126ddd4 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -553,7 +553,7 @@ def __setitem__(self, key: Any, value: Any): """ # Normalize value to scalar/column - value_normalized = ( + value_normalized: cudf.Scalar | ColumnBase = ( cudf.Scalar(value, dtype=self.dtype) if is_scalar(value) else as_column(value, dtype=self.dtype) @@ -609,9 +609,12 @@ def _scatter_by_slice( ) # step != 1, create a scatter map with arange - scatter_map = as_column( - rng, - dtype=cudf.dtype(np.int32), + scatter_map = cast( + cudf.core.column.NumericalColumn, + as_column( + rng, + dtype=cudf.dtype(np.int32), + ), ) return self._scatter_by_column(scatter_map, value) @@ -1111,11 +1114,16 @@ def argsort( if (ascending and self.is_monotonic_increasing) or ( not ascending and self.is_monotonic_decreasing ): - return as_column(range(len(self))) + return cast( + cudf.core.column.NumericalColumn, as_column(range(len(self))) + ) elif (ascending and self.is_monotonic_decreasing) or ( not ascending and self.is_monotonic_increasing ): - return as_column(range(len(self) - 1, -1, -1)) + return cast( + cudf.core.column.NumericalColumn, + as_column(range(len(self) - 1, -1, -1)), + ) else: return libcudf.sort.order_by( [self], [ascending], na_position, stable=True @@ -1752,7 +1760,7 @@ def as_column( nan_as_null: bool | None = None, dtype: Dtype | None = None, length: int | None = None, -): +) -> ColumnBase: """Create a Column from an arbitrary object Parameters diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index 302f04a0e71..c6a39199e3b 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -256,7 +256,10 @@ def from_sequences( offset += len(data) offset_vals.append(offset) - offset_col = column.as_column(offset_vals, dtype=size_type_dtype) + offset_col = cast( + NumericalColumn, + column.as_column(offset_vals, dtype=size_type_dtype), + ) # Build ListColumn res = cls( @@ -338,7 +341,7 @@ def __init__(self, parent: ParentType): def get( self, - index: int, + index: int | ColumnLike, default: ScalarLike | ColumnLike | None = None, ) -> ParentType: """ diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index a37355dfcda..90bec049831 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -142,7 +142,7 @@ def __setitem__(self, key: Any, value: Any): """ # Normalize value to scalar/column - device_value = ( + device_value: cudf.Scalar | ColumnBase = ( cudf.Scalar( value, dtype=self.dtype @@ -552,7 +552,7 @@ def _validate_fillna_value( ) -> cudf.Scalar | ColumnBase: """Align fill_value for .fillna based on column type.""" if is_scalar(fill_value): - cudf_obj = cudf.Scalar(fill_value) + cudf_obj: cudf.Scalar | ColumnBase = cudf.Scalar(fill_value) if not as_column(cudf_obj).can_cast_safely(self.dtype): raise TypeError( f"Cannot safely cast non-equivalent " diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 6f7508822d4..16e6908f308 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -776,11 +776,13 @@ def contains( # TODO: we silently ignore the `regex=` flag here if case is False: input_column = libstrings.to_lower(self._column) - pat = libstrings.to_lower(column.as_column(pat, dtype="str")) + col_pat = libstrings.to_lower( + column.as_column(pat, dtype="str") + ) else: input_column = self._column - pat = column.as_column(pat, dtype="str") - result_col = libstrings.contains_multiple(input_column, pat) + col_pat = column.as_column(pat, dtype="str") + result_col = libstrings.contains_multiple(input_column, col_pat) return self._return_or_inplace(result_col) def like(self, pat: str, esc: str | None = None) -> SeriesOrIndex: diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 7aa3e5f8163..34076fa0060 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -6,7 +6,7 @@ import sys from collections import abc from functools import cached_property, reduce -from typing import TYPE_CHECKING, Any, Callable, Mapping +from typing import TYPE_CHECKING, Any, Callable, Mapping, cast import numpy as np import pandas as pd @@ -35,7 +35,7 @@ class _NestedGetItemDict(dict): """ @classmethod - def from_zip(cls, data): + def from_zip(cls, data: abc.Iterator): """Create from zip, specialized factory for nesting.""" obj = cls() for key, value in data: @@ -91,12 +91,12 @@ class ColumnAccessor(abc.MutableMapping): column length and data.values() are all Columns """ - _data: dict[Any, ColumnBase] - _level_names: tuple[Any, ...] + _data: dict[abc.Hashable, ColumnBase] + _level_names: tuple[abc.Hashable, ...] def __init__( self, - data: abc.MutableMapping[Any, ColumnBase] | Self, + data: abc.MutableMapping[abc.Hashable, ColumnBase] | Self, multiindex: bool = False, level_names=None, rangeindex: bool = False, @@ -141,16 +141,16 @@ def __init__( f"data must be a ColumnAccessor or MutableMapping, not {type(data).__name__}" ) - def __iter__(self): + def __iter__(self) -> abc.Iterator: return iter(self._data) - def __getitem__(self, key: Any) -> ColumnBase: + def __getitem__(self, key: abc.Hashable) -> ColumnBase: return self._data[key] - def __setitem__(self, key: Any, value: ColumnBase) -> None: + def __setitem__(self, key: abc.Hashable, value: ColumnBase) -> None: self.set_by_label(key, value) - def __delitem__(self, key: Any) -> None: + def __delitem__(self, key: abc.Hashable) -> None: old_ncols = len(self._data) del self._data[key] new_ncols = len(self._data) @@ -186,7 +186,7 @@ def _from_columns_like_self( Whether to verify column length and type. """ if sys.version_info.major >= 3 and sys.version_info.minor >= 10: - data = zip(self.names, columns, strict=True) + data = zip(self.names, columns, strict=True) # type: ignore[call-overload] else: columns = list(columns) if len(columns) != len(self.names): @@ -205,7 +205,7 @@ def _from_columns_like_self( ) @property - def level_names(self) -> tuple[Any, ...]: + def level_names(self) -> tuple[abc.Hashable, ...]: if self._level_names is None or len(self._level_names) == 0: return tuple((None,) * max(1, self.nlevels)) else: @@ -221,7 +221,7 @@ def nlevels(self) -> int: return len(next(iter(self.keys()))) @property - def name(self) -> Any: + def name(self) -> abc.Hashable: return self.level_names[-1] @cached_property @@ -232,7 +232,7 @@ def nrows(self) -> int: return len(next(iter(self.values()))) @cached_property - def names(self) -> tuple[Any, ...]: + def names(self) -> tuple[abc.Hashable, ...]: return tuple(self.keys()) @cached_property @@ -291,7 +291,7 @@ def to_pandas_index(self) -> pd.Index: ) elif cudf.api.types.infer_dtype(self.names) == "integer": if len(self.names) == 1: - start = self.names[0] + start = cast(int, self.names[0]) return pd.RangeIndex( start=start, stop=start + 1, step=1, name=self.name ) @@ -299,7 +299,9 @@ def to_pandas_index(self) -> pd.Index: if len(uniques) == 1 and uniques[0] != 0: diff = uniques[0] new_range = range( - self.names[0], self.names[-1] + diff, diff + cast(int, self.names[0]), + cast(int, self.names[-1]) + diff, + diff, ) return pd.RangeIndex(new_range, name=self.name) result = pd.Index( @@ -310,7 +312,9 @@ def to_pandas_index(self) -> pd.Index: ) return result - def insert(self, name: Any, value: ColumnBase, loc: int = -1) -> None: + def insert( + self, name: abc.Hashable, value: ColumnBase, loc: int = -1 + ) -> None: """ Insert column into the ColumnAccessor at the specified location. @@ -457,7 +461,7 @@ def select_by_index(self, index: Any) -> Self: verify=False, ) - def swaplevel(self, i=-2, j=-1) -> Self: + def swaplevel(self, i: abc.Hashable = -2, j: abc.Hashable = -1) -> Self: """ Swap level i with level j. Calling this method does not change the ordering of the values. @@ -486,7 +490,7 @@ def swaplevel(self, i=-2, j=-1) -> Self: # swap old keys for i and j for n, row in enumerate(self.names): - new_keys[n][i], new_keys[n][j] = row[j], row[i] + new_keys[n][i], new_keys[n][j] = row[j], row[i] # type: ignore[call-overload, index] new_dict.update({row: tuple(new_keys[n])}) # TODO: Change to deep=False when copy-on-write is default @@ -494,10 +498,10 @@ def swaplevel(self, i=-2, j=-1) -> Self: # swap level_names for i and j new_names = list(self.level_names) - new_names[i], new_names[j] = new_names[j], new_names[i] + new_names[i], new_names[j] = new_names[j], new_names[i] # type: ignore[call-overload] return type(self)( - new_data, + new_data, # type: ignore[arg-type] multiindex=self.multiindex, level_names=new_names, rangeindex=self.rangeindex, @@ -505,7 +509,7 @@ def swaplevel(self, i=-2, j=-1) -> Self: verify=False, ) - def set_by_label(self, key: Any, value: ColumnBase) -> None: + def set_by_label(self, key: abc.Hashable, value: ColumnBase) -> None: """ Add (or modify) column by name. @@ -555,7 +559,7 @@ def _select_by_label_list_like(self, key: tuple) -> Self: verify=False, ) - def _select_by_label_grouped(self, key: Any) -> Self: + def _select_by_label_grouped(self, key: abc.Hashable) -> Self: result = self._grouped_data[key] if isinstance(result, column.ColumnBase): # self._grouped_data[key] = self._data[key] so skip validation @@ -606,8 +610,12 @@ def _select_by_label_slice(self, key: slice) -> Self: ) def _select_by_label_with_wildcard(self, key: tuple) -> Self: - key = self._pad_key(key, slice(None)) - data = {k: self._data[k] for k in self.names if _keys_equal(k, key)} + pad_key = self._pad_key(key, slice(None)) + data = { + k: self._data[k] + for k in self.names + if _keys_equal(k, pad_key) # type: ignore[arg-type] + } return type(self)( data, multiindex=self.multiindex, @@ -616,7 +624,9 @@ def _select_by_label_with_wildcard(self, key: tuple) -> Self: verify=False, ) - def _pad_key(self, key: Any, pad_value="") -> Any: + def _pad_key( + self, key: abc.Hashable, pad_value: str | slice = "" + ) -> abc.Hashable: """ Pad the provided key to a length equal to the number of levels. @@ -628,7 +638,9 @@ def _pad_key(self, key: Any, pad_value="") -> Any: return key + (pad_value,) * (self.nlevels - len(key)) def rename_levels( - self, mapper: Mapping[Any, Any] | Callable, level: int | None = None + self, + mapper: Mapping[abc.Hashable, abc.Hashable] | Callable, + level: int | None = None, ) -> Self: """ Rename the specified levels of the given ColumnAccessor @@ -701,14 +713,14 @@ def rename_column(x): verify=False, ) - def droplevel(self, level) -> None: + def droplevel(self, level: int) -> None: # drop the nth level if level < 0: level += self.nlevels old_ncols = len(self._data) self._data = { - _remove_key_level(key, level): value + _remove_key_level(key, level): value # type: ignore[arg-type] for key, value in self._data.items() } new_ncols = len(self._data) @@ -722,7 +734,7 @@ def droplevel(self, level) -> None: self._clear_cache(old_ncols, new_ncols) -def _keys_equal(target: Any, key: Any) -> bool: +def _keys_equal(target: abc.Hashable, key: abc.Iterable) -> bool: """ Compare `key` to `target`. @@ -740,7 +752,7 @@ def _keys_equal(target: Any, key: Any) -> bool: return True -def _remove_key_level(key: Any, level: int) -> Any: +def _remove_key_level(key: tuple, level: int) -> abc.Hashable: """ Remove a level from key. If detupleize is True, and if only a single level remains, convert the tuple to a scalar. @@ -751,7 +763,9 @@ def _remove_key_level(key: Any, level: int) -> Any: return result -def _get_level(x, nlevels, level_names): +def _get_level( + x: abc.Hashable, nlevels: int, level_names: tuple[abc.Hashable, ...] +) -> abc.Hashable: """Get the level index from a level number or name. If given an integer, this function will handle wraparound for diff --git a/python/cudf/cudf/core/copy_types.py b/python/cudf/cudf/core/copy_types.py index 6afbc0bbc65..16d8964f083 100644 --- a/python/cudf/cudf/core/copy_types.py +++ b/python/cudf/cudf/core/copy_types.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast @@ -44,15 +44,17 @@ class GatherMap: If the map is not in bounds. """ - #: The gather map - column: "NumericalColumn" #: The number of rows the gather map has been validated for nrows: int #: Was the validation for nullify=True? nullify: bool def __init__(self, column: Any, nrows: int, *, nullify: bool): - self.column = cudf.core.column.as_column(column) + #: The gather map + self.column = cast( + cudf.core.column.NumericalColumn, + cudf.core.column.as_column(column), + ) self.nrows = nrows self.nullify = nullify if len(self.column) == 0: @@ -135,11 +137,12 @@ class BooleanMask: If the mask has the wrong number of rows """ - #: The boolean mask - column: "NumericalColumn" - def __init__(self, column: Any, nrows: int): - self.column = cudf.core.column.as_column(column) + #: The boolean mask + self.column = cast( + cudf.core.column.NumericalColumn, + cudf.core.column.as_column(column), + ) if self.column.dtype.kind != "b": raise TypeError("Boolean mask must have bool dtype") if len(column) != nrows: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 43693ec20b1..14b63c2b0d7 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5830,7 +5830,7 @@ def from_records( df = cls._from_data( ColumnAccessor( - data=ca_data, + data=ca_data, # type: ignore[arg-type] multiindex=isinstance( columns, (pd.MultiIndex, cudf.MultiIndex) ), diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index e46e24dd0d8..60253b9ae5d 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -40,7 +40,7 @@ from cudf.core._base_index import BaseIndex from cudf.core._compat import PANDAS_LT_300 from cudf.core.buffer import acquire_spill_lock -from cudf.core.column import ColumnBase, as_column +from cudf.core.column import ColumnBase, NumericalColumn, as_column from cudf.core.column_accessor import ColumnAccessor from cudf.core.copy_types import BooleanMask, GatherMap from cudf.core.dtypes import ListDtype @@ -3008,9 +3008,12 @@ def _slice(self, arg: slice, keep_index: bool = True) -> Self: if stride != 1: return self._gather( GatherMap.from_column_unchecked( - as_column( - range(start, stop, stride), - dtype=libcudf.types.size_type_dtype, + cast( + NumericalColumn, + as_column( + range(start, stop, stride), + dtype=libcudf.types.size_type_dtype, + ), ), len(self), nullify=False, @@ -4761,10 +4764,13 @@ def _sample_axis_0( ): try: gather_map = GatherMap.from_column_unchecked( - cudf.core.column.as_column( - random_state.choice( - len(self), size=n, replace=replace, p=weights - ) + cast( + NumericalColumn, + cudf.core.column.as_column( + random_state.choice( + len(self), size=n, replace=replace, p=weights + ) + ), ), len(self), nullify=False, @@ -6599,7 +6605,7 @@ def _drop_rows_by_labels( level = 0 levels_index = obj.index.get_level_values(level) - if errors == "raise" and not labels.isin(levels_index).all(): + if errors == "raise" and not labels.isin(levels_index).all(): # type: ignore[union-attr] raise KeyError("One or more values not found in axis") if isinstance(level, int): @@ -6649,7 +6655,7 @@ def _drop_rows_by_labels( ) else: - if errors == "raise" and not labels.isin(obj.index).all(): + if errors == "raise" and not labels.isin(obj.index).all(): # type: ignore[union-attr] raise KeyError("One or more values not found in axis") if isinstance(labels, ColumnBase): From 8b20298c960387c825cfd1476bcf0bc9119df58e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 22 Aug 2024 16:48:22 -0500 Subject: [PATCH 109/270] Move pragma once in rolling/jit/operation.hpp. (#16636) I noticed from https://github.com/rapidsai/cudf/pull/16590#discussion_r1725842333 that there was one other file where `#pragma once` was not at the top. This PR fixes that. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16636 --- cpp/src/rolling/jit/operation.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/rolling/jit/operation.hpp b/cpp/src/rolling/jit/operation.hpp index f8a52c03d4e..3be739ec5bf 100644 --- a/cpp/src/rolling/jit/operation.hpp +++ b/cpp/src/rolling/jit/operation.hpp @@ -14,12 +14,12 @@ * limitations under the License. */ +#pragma once + #include "rolling/jit/operation-udf.hpp" #include -#pragma once - struct rolling_udf_ptx { template static OutType operate(InType const* in_col, cudf::size_type start, cudf::size_type count) From eaefcb4e9baa587f40bc6daa5452c170b9f9616b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:33:13 -1000 Subject: [PATCH 110/270] Support DecimalDtype meta in dask_cudf (#16634) To enable some tpch benchmarking for dask-cudf Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Richard (Rick) Zamora (https://github.com/rjzamora) URL: https://github.com/rapidsai/cudf/pull/16634 --- python/dask_cudf/dask_cudf/backends.py | 2 ++ python/dask_cudf/dask_cudf/tests/test_join.py | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 16b2c8959e2..5bd3eb5fa7f 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -134,6 +134,8 @@ def _get_non_empty_data( return cudf.core.column.as_column( np.arange(start=0, stop=2, dtype=s.dtype) ) + elif isinstance(s.dtype, cudf.core.dtypes.DecimalDtype): + return cudf.core.column.as_column(range(2), dtype=s.dtype) else: raise TypeError( f"Don't know how to handle column of type {type(s).__name__}" diff --git a/python/dask_cudf/dask_cudf/tests/test_join.py b/python/dask_cudf/dask_cudf/tests/test_join.py index ed291ef31a7..3e078c47cdd 100644 --- a/python/dask_cudf/dask_cudf/tests/test_join.py +++ b/python/dask_cudf/dask_cudf/tests/test_join.py @@ -386,3 +386,14 @@ def test_issue_12773(): expected.to_pandas(), check_index=False, ) + + +@pytest.mark.parametrize( + "typ", [cudf.Decimal32Dtype, cudf.Decimal64Dtype, cudf.Decimal128Dtype] +) +def test_merge_on_decimal(typ): + df = cudf.DataFrame({"a": [1], "b": [2]}, dtype=typ(1)) + ddf = dask_cudf.from_cudf(df, npartitions=1) + result = ddf.merge(ddf, left_on="a", right_on="a") + expected = df.merge(df, left_on="a", right_on="a") + dd.assert_eq(result, expected) From 83f68c920f51f9e69f2a5bf0fddf26babac2483b Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Thu, 22 Aug 2024 18:59:47 -0400 Subject: [PATCH 111/270] Revert "Hide all gtest symbols in cudftestutil (#16546)" (#16644) This reverts commit ac42bc870a65d807784cae63e25b9e9ca788eb23. We need to revert #16546 as it broke the gtest builds for cudf. Therefore gtests that actually fail wouldn't properly report an error but silently continue and report as passed. Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16644 --- cpp/cmake/thirdparty/get_gtest.cmake | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index ec8cbd8c568..10e6b026d9a 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -16,18 +16,9 @@ function(find_and_configure_gtest) include(${rapids-cmake-dir}/cpm/gtest.cmake) - # Mark all the non explicit googletest symbols as hidden. This ensures that libcudftestutil can be - # used by consumers with a different shared gtest. - set(gtest_hide_internal_symbols ON) - # Find or install GoogleTest rapids_cpm_gtest(BUILD_STATIC) - # Mark all the explicit googletest symbols as hidden. This ensures that libcudftestutil can be - # used by consumers with a different shared gtest. - if(TARGET gtest) - target_compile_definitions(gtest PUBLIC "$") - endif() endfunction() find_and_configure_gtest() From 91f304ecb16dbe06c1405df42ada9b66875f61c8 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 23 Aug 2024 07:51:23 -0500 Subject: [PATCH 112/270] Enable testing `cudf.pandas` unit tests for all minor versions of pandas (#16595) Fixes: https://github.com/rapidsai/cudf/issues/16537 This PR enables testing `cudf.pandas` unit tests with all minor versions of pandas-2 Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16595 --- .../fetch_pandas_versions.py | 24 +++++++++++++ ci/cudf_pandas_scripts/run_tests.sh | 36 ++++++++++++++++--- .../cudf_pandas_tests/test_cudf_pandas.py | 18 ++++++++++ .../cudf/cudf_pandas_tests/test_profiler.py | 8 +++++ 4 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 ci/cudf_pandas_scripts/fetch_pandas_versions.py diff --git a/ci/cudf_pandas_scripts/fetch_pandas_versions.py b/ci/cudf_pandas_scripts/fetch_pandas_versions.py new file mode 100644 index 00000000000..b6913f947e8 --- /dev/null +++ b/ci/cudf_pandas_scripts/fetch_pandas_versions.py @@ -0,0 +1,24 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import requests +from packaging.version import Version +from packaging.specifiers import SpecifierSet +import argparse + +def get_pandas_versions(pandas_range): + url = "https://pypi.org/pypi/pandas/json" + response = requests.get(url) + data = response.json() + versions = [Version(v) for v in data['releases']] + specifier = SpecifierSet(pandas_range.lstrip("pandas")) + matching_versions = [v for v in versions if v in specifier] + matching_minors = sorted(set(".".join((str(v.major), str(v.minor))) for v in matching_versions), key=Version) + return matching_minors + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Filter pandas versions by prefix.") + parser.add_argument("pandas_range", type=str, help="The version prefix to filter by.") + args = parser.parse_args() + + versions = get_pandas_versions(args.pandas_range) + print(','.join(versions)) diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 8215ce729b3..5bfc083bcd3 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -9,13 +9,20 @@ RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" +DEPENDENCIES_PATH="dependencies.yaml" +package_name="pandas" + +# Use grep to find the line containing the package name and version constraint +pandas_version_constraint=$(grep -oP "pandas>=\d+\.\d+,\<\d+\.\d+\.\d+dev\d+" $DEPENDENCIES_PATH) + # Function to display script usage function display_usage { - echo "Usage: $0 [--no-cudf]" + echo "Usage: $0 [--no-cudf] [pandas-version]" } # Default value for the --no-cudf option no_cudf=false +PANDAS_VERSION="" # Parse command-line arguments while [[ $# -gt 0 ]]; do @@ -25,9 +32,14 @@ while [[ $# -gt 0 ]]; do shift ;; *) - echo "Error: Unknown option $1" - display_usage - exit 1 + if [[ -z "$PANDAS_VERSION" ]]; then + PANDAS_VERSION=$1 + shift + else + echo "Error: Unknown option $1" + display_usage + exit 1 + fi ;; esac done @@ -53,3 +65,19 @@ python -m pytest -p cudf.pandas \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-pandas-coverage.xml" \ --cov-report=term \ ./python/cudf/cudf_pandas_tests/ + +output=$(python ci/cudf_pandas_scripts/fetch_pandas_versions.py $pandas_version_constraint) + +# Convert the comma-separated list into an array +IFS=',' read -r -a versions <<< "$output" + +for version in "${versions[@]}"; do + echo "Installing pandas version: ${version}" + python -m pip install "pandas==${version}" + python -m pytest -p cudf.pandas \ + --cov-config=./python/cudf/.coveragerc \ + --cov=cudf \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-pandas-coverage.xml" \ + --cov-report=term \ + ./python/cudf/cudf_pandas_tests/ +done diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 6292022d8e4..028f5f173ac 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -42,6 +42,8 @@ get_calendar, ) +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION + # Accelerated pandas has the real pandas and cudf modules as attributes pd = xpd._fsproxy_slow cudf = xpd._fsproxy_fast @@ -607,6 +609,10 @@ def test_array_function_series_fallback(series): tm.assert_equal(expect, got) +@pytest.mark.xfail( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_timedeltaproperties(series): psr, sr = series psr, sr = psr.astype("timedelta64[ns]"), sr.astype("timedelta64[ns]") @@ -666,6 +672,10 @@ def test_maintain_container_subclasses(multiindex): assert isinstance(got, xpd.core.indexes.frozen.FrozenList) +@pytest.mark.xfail( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas due to unsupported boxcar window type", +) def test_rolling_win_type(): pdf = pd.DataFrame(range(5)) df = xpd.DataFrame(range(5)) @@ -1281,6 +1291,10 @@ def max_times_two(self): assert s.max_times_two() == 6 +@pytest.mark.xfail( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="DatetimeArray.__floordiv__ missing in pandas-2.0.0", +) def test_floordiv_array_vs_df(): xarray = xpd.Series([1, 2, 3], dtype="datetime64[ns]").array parray = pd.Series([1, 2, 3], dtype="datetime64[ns]").array @@ -1552,6 +1566,10 @@ def test_numpy_cupy_flatiter(series): assert type(arr.flat._fsproxy_slow) == np.flatiter +@pytest.mark.xfail( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="pyarrow_numpy storage type was not supported in pandas-2.0.0", +) def test_arrow_string_arrays(): cu_s = xpd.Series(["a", "b", "c"]) pd_s = pd.Series(["a", "b", "c"]) diff --git a/python/cudf/cudf_pandas_tests/test_profiler.py b/python/cudf/cudf_pandas_tests/test_profiler.py index 588398265f2..5b7bde06d1d 100644 --- a/python/cudf/cudf_pandas_tests/test_profiler.py +++ b/python/cudf/cudf_pandas_tests/test_profiler.py @@ -5,6 +5,8 @@ import os import subprocess +import pytest + from cudf.pandas import LOADED, Profiler if not LOADED: @@ -13,7 +15,13 @@ import numpy as np import pandas as pd +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION + +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="function names change across versions of pandas, so making sure it only runs on latest version of pandas", +) def test_profiler(): np.random.seed(42) with Profiler() as profiler: From 8d6b2616af8aeec6dfd02d787084c583e2447791 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Fri, 23 Aug 2024 10:47:40 -0500 Subject: [PATCH 113/270] adding wheel build for libcudf (#15483) Contributes to https://github.com/rapidsai/build-planning/issues/33 Adds a standalone `libcudf` wheel, containing the `libcudf` C++ shared library. Fixes #16588 Authors: - Mike Sarahan (https://github.com/msarahan) - Vyas Ramasubramani (https://github.com/vyasr) - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15483 --- .github/workflows/build.yaml | 20 ++++ .github/workflows/pr.yaml | 11 ++- ci/build_wheel_cudf.sh | 26 ++++-- ci/build_wheel_libcudf.sh | 15 +++ ci/build_wheel_pylibcudf.sh | 22 ++++- ci/cudf_pandas_scripts/pandas-tests/run.sh | 8 +- ci/cudf_pandas_scripts/run_tests.sh | 8 +- ci/release/update-version.sh | 1 + ci/test_wheel_cudf.sh | 8 +- ci/test_wheel_cudf_polars.sh | 8 +- ci/test_wheel_dask_cudf.sh | 10 +- dependencies.yaml | 91 ++++++++++++++++++- python/cudf/CMakeLists.txt | 69 ++------------ python/cudf/cudf/__init__.py | 10 ++ python/cudf/cudf/_lib/CMakeLists.txt | 1 + python/cudf/pyproject.toml | 3 + python/libcudf/CMakeLists.txt | 58 ++++++++++++ python/libcudf/LICENSE | 1 + python/libcudf/README.md | 1 + .../cmake/Modules/WheelHelpers.cmake | 0 python/libcudf/libcudf/VERSION | 1 + python/libcudf/libcudf/__init__.py | 16 ++++ python/libcudf/libcudf/_version.py | 33 +++++++ python/libcudf/libcudf/load.py | 51 +++++++++++ python/libcudf/pyproject.toml | 75 +++++++++++++++ python/pylibcudf/CMakeLists.txt | 68 ++------------ python/pylibcudf/pylibcudf/CMakeLists.txt | 2 + python/pylibcudf/pylibcudf/__init__.py | 10 ++ python/pylibcudf/pyproject.toml | 3 + 29 files changed, 476 insertions(+), 154 deletions(-) create mode 100755 ci/build_wheel_libcudf.sh create mode 100644 python/libcudf/CMakeLists.txt create mode 120000 python/libcudf/LICENSE create mode 120000 python/libcudf/README.md rename python/{pylibcudf => libcudf}/cmake/Modules/WheelHelpers.cmake (100%) create mode 120000 python/libcudf/libcudf/VERSION create mode 100644 python/libcudf/libcudf/__init__.py create mode 100644 python/libcudf/libcudf/_version.py create mode 100644 python/libcudf/libcudf/load.py create mode 100644 python/libcudf/pyproject.toml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9943b02a521..0ea4d5c54dc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,27 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-libcudf: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcudf.sh + wheel-publish-libcudf: + needs: wheel-build-libcudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcudf wheel-build-pylibcudf: + needs: [wheel-publish-libcudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 74bdc666c68..2e2a8b6b9bc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,6 +23,7 @@ jobs: - static-configure - conda-notebook-tests - docs-build + - wheel-build-libcudf - wheel-build-pylibcudf - wheel-build-cudf - wheel-tests-cudf @@ -121,10 +122,18 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-pylibcudf: + wheel-build-libcudf: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + build_type: pull-request + script: "ci/build_wheel_libcudf.sh" + wheel-build-pylibcudf: + needs: [checks, wheel-build-libcudf] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_pylibcudf.sh" diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 7c0fb1efebe..cf33703f544 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -5,16 +5,28 @@ set -euo pipefail package_dir="python/cudf" -export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" - -# Download the pylibcudf built in the previous step RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcudf_dist -echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/pylibcudf_dist/pylibcudf_*.whl)" > /tmp/constraints.txt +# Downloads libcudf and pylibcudf wheels from this current build, +# then ensures 'cudf' wheel builds always use the 'libcudf' and 'pylibcudf' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcudf_dist +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt +echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/pylibcudf_dist/pylibcudf_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" + ./ci/build_wheel.sh ${package_dir} -python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* +python -m auditwheel repair \ + --exclude libcudf.so \ + --exclude libarrow.so.1601 \ + --exclude libnvcomp.so \ + --exclude libnvcomp_bitcomp.so \ + --exclude libnvcomp_gdeflate.so \ + -w ${package_dir}/final_dist \ + ${package_dir}/dist/* -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/final_dist diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh new file mode 100755 index 00000000000..9694c3f6144 --- /dev/null +++ b/ci/build_wheel_libcudf.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir="python/libcudf" + +./ci/build_wheel.sh ${package_dir} + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +mkdir -p ${package_dir}/final_dist +python -m auditwheel repair --exclude libarrow.so.1601 -w ${package_dir}/final_dist ${package_dir}/dist/* + +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index b25d118ff81..7181a49d397 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -5,12 +5,26 @@ set -euo pipefail package_dir="python/pylibcudf" -export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -./ci/build_wheel.sh ${package_dir} +# Downloads libcudf wheel from this current build, +# then ensures 'pylibcudf' wheel builds always use the 'libcudf' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" -python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* +./ci/build_wheel.sh ${package_dir} +python -m auditwheel repair \ + --exclude libcudf.so \ + --exclude libarrow.so.1601 \ + --exclude libnvcomp.so \ + --exclude libnvcomp_bitcomp.so \ + --exclude libnvcomp_gdeflate.so \ + -w ${package_dir}/final_dist \ + ${package_dir}/dist/* -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 97c3139080f..e5cd4436a3a 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -12,13 +12,15 @@ rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the cudf and pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# Download the cudf, libcudf, and pylibcudf built in the previous step +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 5bfc083bcd3..90ea1afbe6a 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -49,13 +49,15 @@ if [ "$no_cudf" = true ]; then else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - # Download the cudf and pylibcudf built in the previous step - RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist - RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + # Download the cudf, libcudf, and pylibcudf built in the previous step + RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist + RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist + RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" fi diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index e79a91510b8..be55b49870f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -49,6 +49,7 @@ DEPENDENCIES=( dask-cuda dask-cudf kvikio + libcudf libkvikio librmm pylibcudf diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 19131952098..6861d699695 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -5,13 +5,15 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the cudf and pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# Download the cudf, libcudf, and pylibcudf built in the previous step +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index e9c6188502c..0baf6c9e277 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -18,16 +18,18 @@ else fi RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist -# Download pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# Download libcudf and pylibcudf built in the previous step +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist rapids-logger "Installing cudf_polars and its dependencies" # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" rapids-logger "Run cudf_polars tests" diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index ff893a08e27..fa74b2398f7 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -4,16 +4,18 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist -# Download the cudf and pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# Download the cudf, libcudf, and pylibcudf built in the previous step +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} diff --git a/dependencies.yaml b/dependencies.yaml index 150d03be021..553d01735b2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -13,6 +13,7 @@ files: - cuda - cuda_version - depends_on_cupy + - depends_on_librmm - depends_on_rmm - develop - docs @@ -95,6 +96,8 @@ files: - build_base - build_python_common - depends_on_pylibcudf + - depends_on_libcudf + - depends_on_librmm - depends_on_rmm py_run_cudf: output: pyproject @@ -106,6 +109,7 @@ files: - run_cudf - pyarrow_run - depends_on_cupy + - depends_on_libcudf - depends_on_pylibcudf - depends_on_rmm py_test_cudf: @@ -117,6 +121,31 @@ files: includes: - test_python_common - test_python_cudf + py_build_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: build-system + includes: + - rapids_build_skbuild + py_rapids_build_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_base + - build_cpp + - build_python_libcudf + - depends_on_librmm + py_run_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: project + includes: + - pyarrow_run py_build_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf @@ -133,6 +162,8 @@ files: includes: - build_base - build_python_common + - depends_on_libcudf + - depends_on_librmm - depends_on_rmm py_run_pylibcudf: output: pyproject @@ -140,6 +171,7 @@ files: extras: table: project includes: + - depends_on_libcudf - depends_on_rmm - pyarrow_run - run_pylibcudf @@ -359,13 +391,18 @@ dependencies: - cython>=3.0.3 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==16.1.0.* + - &pyarrow_build pyarrow==16.1.0.* - output_types: pyproject packages: # Hard pin the patch version used during the build. # Sync with conda build constraint & wheel run constraint. # TODO: Change to `2.0.*` for NumPy 2 - numpy==1.23.* + build_python_libcudf: + common: + - output_types: [conda, requirements, pyproject] + packages: + - *pyarrow_build libarrow_build: common: - output_types: conda @@ -759,6 +796,31 @@ dependencies: packages: - dask-cuda==24.10.*,>=0.0.0a0 - *numba + depends_on_libcudf: + common: + - output_types: conda + packages: + - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for libcudf-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcudf-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcudf-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*libcudf_unsuffixed]} depends_on_pylibcudf: common: - output_types: conda @@ -849,6 +911,33 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + depends_on_librmm: + common: + - output_types: conda + packages: + - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for librmm-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==24.10.*,>=0.0.0a0 + - matrix: + packages: + - *librmm_unsuffixed depends_on_rmm: common: - output_types: conda diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index e11d62b3bd5..72f20b30052 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -24,72 +24,15 @@ project( LANGUAGES CXX CUDA ) -option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files" - OFF -) -option(USE_LIBARROW_FROM_PYARROW "Only use the libarrow contained in pyarrow" OFF) -mark_as_advanced(USE_LIBARROW_FROM_PYARROW) - -# Find Python early so that later commands can use it -find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) - -# If the user requested it we attempt to find CUDF. -if(FIND_CUDF_CPP) - include(rapids-cpm) - include(rapids-export) - include(rapids-find) - rapids_cpm_init() +find_package(cudf "${RAPIDS_VERSION}" REQUIRED) - if(USE_LIBARROW_FROM_PYARROW) - # We need to find arrow before libcudf since libcudf requires it but doesn't bundle arrow - # libraries. These variables have no effect because we are always searching for arrow via - # pyarrow, but they must be set as they are required arguments to the function in - # get_arrow.cmake. - set(CUDF_USE_ARROW_STATIC OFF) - set(CUDF_ENABLE_ARROW_S3 OFF) - set(CUDF_ENABLE_ARROW_ORC OFF) - set(CUDF_ENABLE_ARROW_PYTHON OFF) - set(CUDF_ENABLE_ARROW_PARQUET OFF) - include(../../cpp/cmake/thirdparty/get_arrow.cmake) - endif() - - find_package(cudf "${RAPIDS_VERSION}" REQUIRED) - - # an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack - # for the interop.pyx - include(../../cpp/cmake/thirdparty/get_dlpack.cmake) -else() - set(cudf_FOUND OFF) -endif() +# an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack +# for the interop.pyx +include(rapids-cpm) +rapids_cpm_init() +include(../../cpp/cmake/thirdparty/get_dlpack.cmake) include(rapids-cython-core) - -if(NOT cudf_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_BENCHMARKS OFF) - set(CUDF_BUILD_TESTUTIL OFF) - set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp cudf-cpp EXCLUDE_FROM_ALL) - - # libcudf targets are excluded by default above via EXCLUDE_FROM_ALL to remove extraneous - # components like headers from libcudacxx, but we do need the libraries. However, we want to - # control where they are installed to. Since there are multiple subpackages of cudf._lib that - # require access to libcudf, we place the library and all its dependent artifacts in the cudf - # directory as a single source of truth and modify the other rpaths appropriately. - set(cython_lib_dir cudf) - include(../pylibcudf/cmake/Modules/WheelHelpers.cmake) - # TODO: This install is currently overzealous. We should only install the libraries that are - # downloaded by CPM during the build, not libraries that were found on the system. However, in - # practice right this would only be a problem is if libcudf was not found but some of the - # dependencies were, and we have no real use cases where that happens. - install_aliased_imported_targets( - TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} - ) -endif() - rapids_cython_init() include(../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index ccc45413de4..d7da42a1708 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -1,5 +1,15 @@ # Copyright (c) 2018-2024, NVIDIA CORPORATION. +# If libcudf was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libcudf +except ModuleNotFoundError: + pass +else: + libcudf.load_library() + del libcudf + # _setup_numba _must be called before numba.cuda is imported, because # it sets the numba config variable responsible for enabling # Minor Version Compatibility. Setting it after importing numba.cuda has no effect. diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index d6182673308..5ea378fc0e5 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -63,6 +63,7 @@ rapids_cython_create_modules( ) target_link_libraries(strings_udf PUBLIC cudf_strings_udf) +target_include_directories(interop PUBLIC "$") set(targets_using_arrow_headers avro csv orc json parquet) link_to_pyarrow_headers("${targets_using_arrow_headers}") diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 9db52164eca..cb9fa30afab 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "cuda-python>=11.7.1,<12.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", + "libcudf==24.10.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<2.0a0", "nvtx>=0.2.1", @@ -126,6 +127,8 @@ matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", + "libcudf==24.10.*,>=0.0.0a0", + "librmm==24.10.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt new file mode 100644 index 00000000000..09c7ed2e217 --- /dev/null +++ b/python/libcudf/CMakeLists.txt @@ -0,0 +1,58 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libcudf-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +# Check if cudf is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python cudf package. +find_package(cudf "${RAPIDS_VERSION}") + +if(cudf_FOUND) + return() +endif() + +unset(cudf_FOUND) + +# For wheels, this should always be true +set(USE_LIBARROW_FROM_PYARROW ON) + +# Find Python early so that later commands can use it +find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) + +set(BUILD_TESTS OFF) +set(BUILD_BENCHMARKS OFF) +set(CUDF_BUILD_TESTUTIL OFF) +set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) +set(CUDA_STATIC_RUNTIME ON) + +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + +include(../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) + +add_subdirectory(../../cpp cudf-cpp) + +# Ensure other libraries needed by libcudf.so get installed alongside it. +include(cmake/Modules/WheelHelpers.cmake) +install_aliased_imported_targets( + TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp + DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY} +) diff --git a/python/libcudf/LICENSE b/python/libcudf/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/python/libcudf/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcudf/README.md b/python/libcudf/README.md new file mode 120000 index 00000000000..fe840054137 --- /dev/null +++ b/python/libcudf/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/pylibcudf/cmake/Modules/WheelHelpers.cmake b/python/libcudf/cmake/Modules/WheelHelpers.cmake similarity index 100% rename from python/pylibcudf/cmake/Modules/WheelHelpers.cmake rename to python/libcudf/cmake/Modules/WheelHelpers.cmake diff --git a/python/libcudf/libcudf/VERSION b/python/libcudf/libcudf/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/libcudf/libcudf/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libcudf/libcudf/__init__.py b/python/libcudf/libcudf/__init__.py new file mode 100644 index 00000000000..10c476cbe89 --- /dev/null +++ b/python/libcudf/libcudf/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcudf._version import __git_commit__, __version__ +from libcudf.load import load_library diff --git a/python/libcudf/libcudf/_version.py b/python/libcudf/libcudf/_version.py new file mode 100644 index 00000000000..7dd732b4905 --- /dev/null +++ b/python/libcudf/libcudf/_version.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py new file mode 100644 index 00000000000..f6ba0d51bdb --- /dev/null +++ b/python/libcudf/libcudf/load.py @@ -0,0 +1,51 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + + +def load_library(): + # This is loading the libarrow shared library in situations where it comes from the + # pyarrow package (i.e. when installed as a wheel). + import pyarrow # noqa: F401 + + # Dynamically load libcudf.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + libcudf_lib = None + try: + libcudf_lib = ctypes.CDLL("libcudf.so", ctypes.RTLD_GLOBAL) + except OSError: + # If neither of these directories contain the library, we assume we are in an + # environment where the C++ library is already installed somewhere else and the + # CMake build of the libcudf Python package was a no-op. + # + # Note that this approach won't work for real editable installs of the libcudf package. + # scikit-build-core has limited support for importlib.resources so there isn't a clean + # way to support that case yet. + for lib_dir in ("lib", "lib64"): + if os.path.isfile( + lib := os.path.join( + os.path.dirname(__file__), lib_dir, "libcudf.so" + ) + ): + libcudf_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) + break + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcudf was loaded from. + return libcudf_lib diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml new file mode 100644 index 00000000000..fd01f7f6e2f --- /dev/null +++ b/python/libcudf/pyproject.toml @@ -0,0 +1,75 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcudf" +dynamic = ["version"] +description = "cuDF - GPU Dataframe (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.10" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "pyarrow>=16.1.0,<16.2.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/cudf" + +[project.entry-points."cmake.prefix"] +libcudf = "libcudf" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcudf"] +wheel.install-dir = "libcudf" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcudf/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "librmm==24.10.*,>=0.0.0a0", + "ninja", + "pyarrow==16.1.0.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcudf/CMakeLists.txt b/python/pylibcudf/CMakeLists.txt index 424d8372280..340ad120377 100644 --- a/python/pylibcudf/CMakeLists.txt +++ b/python/pylibcudf/CMakeLists.txt @@ -24,72 +24,16 @@ project( LANGUAGES CXX CUDA ) -option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files" - OFF -) -option(USE_LIBARROW_FROM_PYARROW "Only use the libarrow contained in pyarrow" OFF) -mark_as_advanced(USE_LIBARROW_FROM_PYARROW) - -# Find Python early so that later commands can use it -find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) - -# If the user requested it we attempt to find CUDF. -if(FIND_CUDF_CPP) - include(rapids-cpm) - include(rapids-export) - include(rapids-find) - rapids_cpm_init() - - if(USE_LIBARROW_FROM_PYARROW) - # We need to find arrow before libcudf since libcudf requires it but doesn't bundle arrow - # libraries. These variables have no effect because we are always searching for arrow via - # pyarrow, but they must be set as they are required arguments to the function in - # get_arrow.cmake. - set(CUDF_USE_ARROW_STATIC OFF) - set(CUDF_ENABLE_ARROW_S3 OFF) - set(CUDF_ENABLE_ARROW_ORC OFF) - set(CUDF_ENABLE_ARROW_PYTHON OFF) - set(CUDF_ENABLE_ARROW_PARQUET OFF) - include(../../cpp/cmake/thirdparty/get_arrow.cmake) - endif() - - find_package(cudf "${RAPIDS_VERSION}" REQUIRED) +find_package(cudf "${RAPIDS_VERSION}" REQUIRED) - # an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack - # for the interop.pyx - include(../../cpp/cmake/thirdparty/get_dlpack.cmake) -else() - set(cudf_FOUND OFF) -endif() +# an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack +# for the interop.pyx +include(rapids-cpm) +rapids_cpm_init() +include(../../cpp/cmake/thirdparty/get_dlpack.cmake) include(rapids-cython-core) -if(NOT cudf_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_BENCHMARKS OFF) - set(CUDF_BUILD_TESTUTIL OFF) - set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp cudf-cpp EXCLUDE_FROM_ALL) - - # libcudf targets are excluded by default above via EXCLUDE_FROM_ALL to remove extraneous - # components like headers from libcudacxx, but we do need the libraries. However, we want to - # control where they are installed to. Since there are multiple subpackages of pylibcudf that - # require access to libcudf, we place the library and all its dependent artifacts in the cudf - # directory as a single source of truth and modify the other rpaths appropriately. - set(cython_lib_dir pylibcudf) - include(cmake/Modules/WheelHelpers.cmake) - # TODO: This install is currently overzealous. We should only install the libraries that are - # downloaded by CPM during the build, not libraries that were found on the system. However, in - # practice right this would only be a problem is if libcudf was not found but some of the - # dependencies were, and we have no real use cases where that happens. - install_aliased_imported_targets( - TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} - ) -endif() - rapids_cython_init() include(cmake/Modules/LinkPyarrowHeaders.cmake) diff --git a/python/pylibcudf/pylibcudf/CMakeLists.txt b/python/pylibcudf/pylibcudf/CMakeLists.txt index ab21bfe97ab..f81a32e07f9 100644 --- a/python/pylibcudf/pylibcudf/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/CMakeLists.txt @@ -53,6 +53,8 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX pylibcudf_ ASSOCIATED_TARGETS cudf ) +target_include_directories(pylibcudf_interop PUBLIC "$") + include(${rapids-cmake-dir}/export/find_package_root.cmake) include(../../../cpp/cmake/thirdparty/get_nanoarrow.cmake) target_link_libraries(pylibcudf_interop PUBLIC nanoarrow) diff --git a/python/pylibcudf/pylibcudf/__init__.py b/python/pylibcudf/pylibcudf/__init__.py index 677fdaf80d0..e784c6c6dd5 100644 --- a/python/pylibcudf/pylibcudf/__init__.py +++ b/python/pylibcudf/pylibcudf/__init__.py @@ -1,5 +1,15 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. +# If libcudf was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libcudf +except ModuleNotFoundError: + pass +else: + libcudf.load_library() + del libcudf + from . import ( aggregation, binaryop, diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index b037508d03f..63d76e9fd4e 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -19,6 +19,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", + "libcudf==24.10.*,>=0.0.0a0", "nvtx>=0.2.1", "packaging", "pyarrow>=16.1.0,<16.2.0a0", @@ -101,6 +102,8 @@ matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", + "libcudf==24.10.*,>=0.0.0a0", + "librmm==24.10.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", From a7ca3afb251805face3dd3248381f4cc9503e143 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Fri, 23 Aug 2024 12:24:30 -0700 Subject: [PATCH 114/270] Add the missing `num_aggregations` axis for `groupby_max_cardinality` (#16630) This PR fixes a minor bug where the `num_aggregations` axis was missed when working on #16154. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/16630 --- cpp/benchmarks/groupby/group_max.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/benchmarks/groupby/group_max.cpp b/cpp/benchmarks/groupby/group_max.cpp index f41285008c4..b9a701a71f4 100644 --- a/cpp/benchmarks/groupby/group_max.cpp +++ b/cpp/benchmarks/groupby/group_max.cpp @@ -101,4 +101,5 @@ NVBENCH_BENCH_TYPES(bench_groupby_max, NVBENCH_BENCH_TYPES(bench_groupby_max_cardinality, NVBENCH_TYPE_AXES(nvbench::type_list)) .set_name("groupby_max_cardinality") + .add_int64_axis("num_aggregations", {1}) .add_int64_axis("cardinality", {10, 20, 50, 100, 1'000, 10'000, 100'000, 1'000'000, 10'000'000}); From 7bd14a58cd10504c99044a2d33159bc3d59e7139 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 23 Aug 2024 15:27:12 -0500 Subject: [PATCH 115/270] Add pylibcudf build dir in build.sh for `clean` (#16648) This PR adds `pylibcudf` build dir in `build.sh` for `clean` to properly delete the pylibcudf build files and folders. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16648 --- build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 957f41aedac..211e1db9fbf 100755 --- a/build.sh +++ b/build.sh @@ -54,10 +54,11 @@ KAFKA_LIB_BUILD_DIR=${KAFKA_LIB_BUILD_DIR:=${REPODIR}/cpp/libcudf_kafka/build} CUDF_KAFKA_BUILD_DIR=${REPODIR}/python/cudf_kafka/build CUDF_BUILD_DIR=${REPODIR}/python/cudf/build DASK_CUDF_BUILD_DIR=${REPODIR}/python/dask_cudf/build +PYLIBCUDF_BUILD_DIR=${REPODIR}/python/pylibcudf/build CUSTREAMZ_BUILD_DIR=${REPODIR}/python/custreamz/build CUDF_JAR_JAVA_BUILD_DIR="$REPODIR/java/target" -BUILD_DIRS="${LIB_BUILD_DIR} ${CUDF_BUILD_DIR} ${DASK_CUDF_BUILD_DIR} ${KAFKA_LIB_BUILD_DIR} ${CUDF_KAFKA_BUILD_DIR} ${CUSTREAMZ_BUILD_DIR} ${CUDF_JAR_JAVA_BUILD_DIR}" +BUILD_DIRS="${LIB_BUILD_DIR} ${CUDF_BUILD_DIR} ${DASK_CUDF_BUILD_DIR} ${KAFKA_LIB_BUILD_DIR} ${CUDF_KAFKA_BUILD_DIR} ${CUSTREAMZ_BUILD_DIR} ${CUDF_JAR_JAVA_BUILD_DIR} ${PYLIBCUDF_BUILD_DIR}" # Set defaults for vars modified by flags to this script VERBOSE_FLAG="" From 7ca6a8cfb40291d28dbd0a99e00275e1b4fc869b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 23 Aug 2024 16:53:59 -0400 Subject: [PATCH 116/270] fix libcudf wheel publishing, make package-type explicit in wheel publishing (#16650) Follow-up to #15483. Contributes to https://github.com/rapidsai/build-planning/issues/33 Wheel publishing for `libcudf` is failing like this: ```text Error: File "./dist/*.whl" does not exist ``` ([build link](https://github.com/rapidsai/cudf/actions/runs/10528569930/job/29176811683)) Because the `package-type` was not set to `cpp` in the `wheels-publish` CI workflow, and that workflow defaults to `python`. ([shared-workflows code link](https://github.com/rapidsai/shared-workflows/blob/157e9824e6e2181fca9aa5c4bea4defd4cc322b0/.github/workflows/wheels-publish.yaml#L23-L26)). This fixes that, and makes that choice explicit for all wheel publishing jobs. References for this `package-type` argument: * https://github.com/rapidsai/shared-workflows/pull/209 * https://github.com/rapidsai/gha-tools/pull/105 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16650 --- .github/workflows/build.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0ea4d5c54dc..72daff7b66b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -86,6 +86,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: libcudf + package-type: cpp wheel-build-pylibcudf: needs: [wheel-publish-libcudf] secrets: inherit @@ -106,6 +107,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibcudf + package-type: python wheel-build-cudf: needs: wheel-publish-pylibcudf secrets: inherit @@ -126,6 +128,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cudf + package-type: python wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit @@ -148,6 +151,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: dask_cudf + package-type: python wheel-build-cudf-polars: needs: wheel-publish-pylibcudf secrets: inherit @@ -170,6 +174,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cudf_polars + package-type: python trigger-pandas-tests: if: inputs.build_type == 'nightly' needs: wheel-build-cudf From 508bdea0dac581d5a33ceb609766c419ef51bbbb Mon Sep 17 00:00:00 2001 From: jakirkham Date: Fri, 23 Aug 2024 19:15:07 -0700 Subject: [PATCH 117/270] Rebuild for & Support NumPy 2 (#16300) Part of issue: https://github.com/rapidsai/build-planning/issues/38 Start building `cudf` with `numpy` version `2.0`. This remains compatible with `numpy` version `1.x` and `2.x`. Allows us to test building with `numpy` version `2.0` (and make sure we catch any issues that show up). Also relaxes the `numpy` `1.x` pin. Pulls in the RDFG changes that are rolling out for broader RAPIDS NumPy 2 support. Authors: - https://github.com/jakirkham - Sebastian Berg (https://github.com/seberg) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Ray Douglass (https://github.com/raydouglass) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16300 --- ci/cudf_pandas_scripts/run_tests.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 6 ++---- conda/recipes/pylibcudf/meta.yaml | 6 ++---- dependencies.yaml | 8 +++----- python/cudf/pyproject.toml | 4 ++-- python/cudf_kafka/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- python/pylibcudf/pyproject.toml | 2 +- 10 files changed, 15 insertions(+), 21 deletions(-) diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 90ea1afbe6a..39056d58d56 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -75,7 +75,7 @@ IFS=',' read -r -a versions <<< "$output" for version in "${versions[@]}"; do echo "Installing pandas version: ${version}" - python -m pip install "pandas==${version}" + python -m pip install "numpy>=1.23,<2.0a0" "pandas==${version}" python -m pytest -p cudf.pandas \ --cov-config=./python/cudf/.coveragerc \ --cov=cudf \ diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 018162bd848..5cf7508ba51 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -57,7 +57,7 @@ dependencies: - notebook - numba>=0.57 - numpy -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - numpydoc - nvcc_linux-64=11.8 - nvcomp==3.0.6 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index c60ffa7aaa5..28b927254f7 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -56,7 +56,7 @@ dependencies: - notebook - numba>=0.57 - numpy -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - numpydoc - nvcomp==3.0.6 - nvtx>=0.2.1 diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 7e86147732e..b2dad767da4 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -64,8 +64,7 @@ requirements: - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 - dlpack >=0.8,<1.0 - # TODO: Change to `2.0` for NumPy 2 - - numpy 1.23 + - numpy 2.0 - pyarrow ==16.1.0.* - libcudf ={{ version }} - pylibcudf ={{ version }} @@ -84,8 +83,7 @@ requirements: - pandas >=2.0,<2.2.3dev0 - cupy >=12.0.0 - numba >=0.57 - # TODO: Update `numpy` in `host` when dropping `<2.0a0` - - numpy >=1.23,<2.0a0 + - numpy >=1.23,<3.0a0 - {{ pin_compatible('pyarrow', max_pin='x.x') }} - libcudf ={{ version }} - pylibcudf ={{ version }} diff --git a/conda/recipes/pylibcudf/meta.yaml b/conda/recipes/pylibcudf/meta.yaml index f405fd10f5d..fef78467027 100644 --- a/conda/recipes/pylibcudf/meta.yaml +++ b/conda/recipes/pylibcudf/meta.yaml @@ -64,8 +64,7 @@ requirements: - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 - dlpack >=0.8,<1.0 - # TODO: Change to `2.0` for NumPy 2 - - numpy 1.23 + - numpy 2.0 - pyarrow ==16.1.0.* - libcudf ={{ version }} - rmm ={{ minor_version }} @@ -81,8 +80,7 @@ requirements: - python - typing_extensions >=4.0.0 - pandas >=2.0,<2.2.3dev0 - # TODO: Update `numpy` in `host` when dropping `<2.0a0` - - numpy >=1.23,<2.0a0 + - numpy >=1.23,<3.0a0 - {{ pin_compatible('pyarrow', max_pin='x.x') }} - {{ pin_compatible('rmm', max_pin='x.x') }} - fsspec >=0.6.0 diff --git a/dependencies.yaml b/dependencies.yaml index 553d01735b2..194577817db 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -394,10 +394,9 @@ dependencies: - &pyarrow_build pyarrow==16.1.0.* - output_types: pyproject packages: - # Hard pin the patch version used during the build. + # Hard pin the version used during the build. # Sync with conda build constraint & wheel run constraint. - # TODO: Change to `2.0.*` for NumPy 2 - - numpy==1.23.* + - numpy==2.0.* build_python_libcudf: common: - output_types: [conda, requirements, pyproject] @@ -605,8 +604,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - fsspec>=0.6.0 - # TODO: Update `numpy` in `build_python_common` when dropping `<2.0a0` - - numpy>=1.23,<2.0a0 + - numpy>=1.23,<3.0a0 - pandas>=2.0,<2.2.3dev0 run_pylibcudf: common: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index cb9fa30afab..e7bac17f8ba 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "fsspec>=0.6.0", "libcudf==24.10.*,>=0.0.0a0", "numba>=0.57", - "numpy>=1.23,<2.0a0", + "numpy>=1.23,<3.0a0", "nvtx>=0.2.1", "packaging", "pandas>=2.0,<2.2.3dev0", @@ -130,7 +130,7 @@ requires = [ "libcudf==24.10.*,>=0.0.0a0", "librmm==24.10.*,>=0.0.0a0", "ninja", - "numpy==1.23.*", + "numpy==2.0.*", "pyarrow==16.1.0.*", "pylibcudf==24.10.*,>=0.0.0a0", "rmm==24.10.*,>=0.0.0a0", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 63c5b07c5f3..2d0222a3fe9 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -106,6 +106,6 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", "ninja", - "numpy==1.23.*", + "numpy==2.0.*", "pyarrow==16.1.0.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 872ecd35c28..d5da7030a75 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "cudf==24.10.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", - "numpy>=1.23,<2.0a0", + "numpy>=1.23,<3.0a0", "pandas>=2.0,<2.2.3dev0", "rapids-dask-dependency==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index 63d76e9fd4e..5f5594b462b 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -105,7 +105,7 @@ requires = [ "libcudf==24.10.*,>=0.0.0a0", "librmm==24.10.*,>=0.0.0a0", "ninja", - "numpy==1.23.*", + "numpy==2.0.*", "pyarrow==16.1.0.*", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 96f2cc5262e5b6b0f50109d327857e306214b3a4 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 26 Aug 2024 10:21:48 -0400 Subject: [PATCH 118/270] Remove CUDA whole compilation ODR violations (#16603) CUDA whole compilation mode requires that all kernels are only launched from TUs that compile them. Previously libcudf would compile a subset of kernels in separate TUs from where they are launched. To keep compile times ( and library size ) as low as possible I have introduced a single C++ function call between the original call site and the kernel launch. In testing this neglibile differences on compile time and binary size. Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Mark Harris (https://github.com/harrism) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16603 --- cpp/src/join/mixed_join.cu | 191 ++++++++++--------- cpp/src/join/mixed_join_kernel.cu | 10 +- cpp/src/join/mixed_join_kernel.cuh | 64 +++++-- cpp/src/join/mixed_join_kernel.hpp | 80 ++++++++ cpp/src/join/mixed_join_kernel_nulls.cu | 10 +- cpp/src/join/mixed_join_kernels.cuh | 124 ------------ cpp/src/join/mixed_join_kernels_semi.cu | 86 +++++---- cpp/src/join/mixed_join_kernels_semi.cuh | 29 +-- cpp/src/join/mixed_join_semi.cu | 38 ++-- cpp/src/join/mixed_join_size_kernel.cu | 12 +- cpp/src/join/mixed_join_size_kernel.cuh | 64 +++++-- cpp/src/join/mixed_join_size_kernel.hpp | 85 +++++++++ cpp/src/join/mixed_join_size_kernel_nulls.cu | 12 +- 13 files changed, 472 insertions(+), 333 deletions(-) create mode 100644 cpp/src/join/mixed_join_kernel.hpp delete mode 100644 cpp/src/join/mixed_join_kernels.cuh create mode 100644 cpp/src/join/mixed_join_size_kernel.hpp diff --git a/cpp/src/join/mixed_join.cu b/cpp/src/join/mixed_join.cu index 48b94c777de..eb12065c6a9 100644 --- a/cpp/src/join/mixed_join.cu +++ b/cpp/src/join/mixed_join.cu @@ -16,7 +16,8 @@ #include "join_common_utils.cuh" #include "join_common_utils.hpp" -#include "mixed_join_kernels.cuh" +#include "mixed_join_kernel.hpp" +#include "mixed_join_size_kernel.hpp" #include #include @@ -178,9 +179,6 @@ mixed_join( join_size = output_size_data->first; matches_per_row_span = output_size_data->second; } else { - // Allocate storage for the counter used to get the size of the join output - rmm::device_scalar size(0, stream, mr); - matches_per_row = rmm::device_uvector{static_cast(outer_num_rows), stream, mr}; // Note that the view goes out of scope after this else statement, but the @@ -190,37 +188,38 @@ mixed_join( matches_per_row_span = cudf::device_span{ matches_per_row->begin(), static_cast(outer_num_rows)}; if (has_nulls) { - compute_mixed_join_output_size - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - kernel_join_type, - hash_table_view, - parser.device_expression_data, - swap_tables, - size.data(), - mutable_matches_per_row_span); + join_size = launch_compute_mixed_join_output_size(*left_conditional_view, + *right_conditional_view, + *probe_view, + *build_view, + hash_probe, + equality_probe, + kernel_join_type, + hash_table_view, + parser.device_expression_data, + swap_tables, + mutable_matches_per_row_span, + config, + shmem_size_per_block, + stream, + mr); } else { - compute_mixed_join_output_size - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - kernel_join_type, - hash_table_view, - parser.device_expression_data, - swap_tables, - size.data(), - mutable_matches_per_row_span); + join_size = launch_compute_mixed_join_output_size(*left_conditional_view, + *right_conditional_view, + *probe_view, + *build_view, + hash_probe, + equality_probe, + kernel_join_type, + hash_table_view, + parser.device_expression_data, + swap_tables, + mutable_matches_per_row_span, + config, + shmem_size_per_block, + stream, + mr); } - join_size = size.value(stream); } // The initial early exit clauses guarantee that we will not reach this point @@ -249,37 +248,39 @@ mixed_join( auto const& join_output_r = right_indices->data(); if (has_nulls) { - mixed_join - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - kernel_join_type, - hash_table_view, - join_output_l, - join_output_r, - parser.device_expression_data, - join_result_offsets.data(), - swap_tables); + launch_mixed_join(*left_conditional_view, + *right_conditional_view, + *probe_view, + *build_view, + hash_probe, + equality_probe, + kernel_join_type, + hash_table_view, + join_output_l, + join_output_r, + parser.device_expression_data, + join_result_offsets.data(), + swap_tables, + config, + shmem_size_per_block, + stream); } else { - mixed_join - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - kernel_join_type, - hash_table_view, - join_output_l, - join_output_r, - parser.device_expression_data, - join_result_offsets.data(), - swap_tables); + launch_mixed_join(*left_conditional_view, + *right_conditional_view, + *probe_view, + *build_view, + hash_probe, + equality_probe, + kernel_join_type, + hash_table_view, + join_output_l, + join_output_r, + parser.device_expression_data, + join_result_offsets.data(), + swap_tables, + config, + shmem_size_per_block, + stream); } auto join_indices = std::pair(std::move(left_indices), std::move(right_indices)); @@ -423,9 +424,6 @@ compute_mixed_join_output_size(table_view const& left_equality, detail::grid_1d const config(outer_num_rows, DEFAULT_JOIN_BLOCK_SIZE); auto const shmem_size_per_block = parser.shmem_per_thread * config.num_threads_per_block; - // Allocate storage for the counter used to get the size of the join output - rmm::device_scalar size(0, stream, mr); - auto const preprocessed_probe = experimental::row::equality::preprocessed_table::create(probe, stream); auto const row_hash = cudf::experimental::row::hash::row_hasher{preprocessed_probe}; @@ -436,39 +434,42 @@ compute_mixed_join_output_size(table_view const& left_equality, // Determine number of output rows without actually building the output to simply // find what the size of the output will be. + std::size_t size = 0; if (has_nulls) { - compute_mixed_join_output_size - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - join_type, - hash_table_view, - parser.device_expression_data, - swap_tables, - size.data(), - matches_per_row_span); + size = launch_compute_mixed_join_output_size(*left_conditional_view, + *right_conditional_view, + *probe_view, + *build_view, + hash_probe, + equality_probe, + join_type, + hash_table_view, + parser.device_expression_data, + swap_tables, + matches_per_row_span, + config, + shmem_size_per_block, + stream, + mr); } else { - compute_mixed_join_output_size - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - join_type, - hash_table_view, - parser.device_expression_data, - swap_tables, - size.data(), - matches_per_row_span); + size = launch_compute_mixed_join_output_size(*left_conditional_view, + *right_conditional_view, + *probe_view, + *build_view, + hash_probe, + equality_probe, + join_type, + hash_table_view, + parser.device_expression_data, + swap_tables, + matches_per_row_span, + config, + shmem_size_per_block, + stream, + mr); } - return {size.value(stream), std::move(matches_per_row)}; + return {size, std::move(matches_per_row)}; } } // namespace detail diff --git a/cpp/src/join/mixed_join_kernel.cu b/cpp/src/join/mixed_join_kernel.cu index 61cfa168b03..cd4016837cc 100644 --- a/cpp/src/join/mixed_join_kernel.cu +++ b/cpp/src/join/mixed_join_kernel.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #include "mixed_join_kernel.cuh" +#include "mixed_join_kernel.hpp" namespace cudf { namespace detail { -template __global__ void mixed_join( +template void launch_mixed_join( table_device_view left_table, table_device_view right_table, table_device_view probe, @@ -32,7 +33,10 @@ template __global__ void mixed_join( size_type* join_output_r, cudf::ast::detail::expression_device_view device_expression_data, cudf::size_type const* join_result_offsets, - bool const swap_tables); + bool const swap_tables, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream); } // namespace detail diff --git a/cpp/src/join/mixed_join_kernel.cuh b/cpp/src/join/mixed_join_kernel.cuh index ea59f23c77f..9d011d43de6 100644 --- a/cpp/src/join/mixed_join_kernel.cuh +++ b/cpp/src/join/mixed_join_kernel.cuh @@ -19,6 +19,7 @@ #include "join_common_utils.cuh" #include "join_common_utils.hpp" #include "mixed_join_common_utils.cuh" +#include "mixed_join_kernel.hpp" #include #include @@ -39,20 +40,20 @@ namespace cg = cooperative_groups; #pragma GCC diagnostic ignored "-Wattributes" template -CUDF_HIDDEN __launch_bounds__(block_size) __global__ - void mixed_join(table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - join_kind const join_type, - cudf::detail::mixed_multimap_type::device_view hash_table_view, - size_type* join_output_l, - size_type* join_output_r, - cudf::ast::detail::expression_device_view device_expression_data, - cudf::size_type const* join_result_offsets, - bool const swap_tables) +CUDF_KERNEL void __launch_bounds__(block_size) + mixed_join(table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + join_kind const join_type, + cudf::detail::mixed_multimap_type::device_view hash_table_view, + size_type* join_output_l, + size_type* join_output_r, + cudf::ast::detail::expression_device_view device_expression_data, + cudf::size_type const* join_result_offsets, + bool const swap_tables) { // Normally the casting of a shared memory array is used to create multiple // arrays of different types from the shared memory buffer, but here it is @@ -111,6 +112,41 @@ CUDF_HIDDEN __launch_bounds__(block_size) __global__ } } +template +void launch_mixed_join(table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + join_kind const join_type, + cudf::detail::mixed_multimap_type::device_view hash_table_view, + size_type* join_output_l, + size_type* join_output_r, + cudf::ast::detail::expression_device_view device_expression_data, + cudf::size_type const* join_result_offsets, + bool const swap_tables, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream) +{ + mixed_join + <<>>( + left_table, + right_table, + probe, + build, + hash_probe, + equality_probe, + join_type, + hash_table_view, + join_output_l, + join_output_r, + device_expression_data, + join_result_offsets, + swap_tables); +} + } // namespace detail } // namespace cudf diff --git a/cpp/src/join/mixed_join_kernel.hpp b/cpp/src/join/mixed_join_kernel.hpp new file mode 100644 index 00000000000..cc92e9d8ba4 --- /dev/null +++ b/cpp/src/join/mixed_join_kernel.hpp @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "join/join_common_utils.hpp" +#include "join/mixed_join_common_utils.cuh" + +#include +#include +#include + +namespace CUDF_EXPORT cudf { +namespace detail { + +/** + * @brief Performs a join using the combination of a hash lookup to identify + * equal rows between one pair of tables and the evaluation of an expression + * containing an arbitrary expression. + * + * This method probes the hash table with each row in the probe table using a + * custom equality comparator that also checks that the conditional expression + * evaluates to true between the left/right tables when a match is found + * between probe and build rows. + * + * @tparam block_size The number of threads per block for this kernel + * @tparam has_nulls Whether or not the inputs may contain nulls. + * + * @param[in] left_table The left table + * @param[in] right_table The right table + * @param[in] probe The table with which to probe the hash table for matches. + * @param[in] build The table with which the hash table was built. + * @param[in] hash_probe The hasher used for the probe table. + * @param[in] equality_probe The equality comparator used when probing the hash table. + * @param[in] join_type The type of join to be performed + * @param[in] hash_table_view The hash table built from `build`. + * @param[out] join_output_l The left result of the join operation + * @param[out] join_output_r The right result of the join operation + * @param[in] device_expression_data Container of device data required to evaluate the desired + * expression. + * @param[in] join_result_offsets The starting indices in join_output[l|r] + * where the matches for each row begin. Equivalent to a prefix sum of + * matches_per_row. + * @param[in] swap_tables If true, the kernel was launched with one thread per right row and + * the kernel needs to internally loop over left rows. Otherwise, loop over right rows. + */ +template +void launch_mixed_join(table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + join_kind const join_type, + cudf::detail::mixed_multimap_type::device_view hash_table_view, + size_type* join_output_l, + size_type* join_output_r, + cudf::ast::detail::expression_device_view device_expression_data, + cudf::size_type const* join_result_offsets, + bool const swap_tables, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream); + +} // namespace detail + +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/join/mixed_join_kernel_nulls.cu b/cpp/src/join/mixed_join_kernel_nulls.cu index 518f8ed8555..185aa133f2d 100644 --- a/cpp/src/join/mixed_join_kernel_nulls.cu +++ b/cpp/src/join/mixed_join_kernel_nulls.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #include "mixed_join_kernel.cuh" +#include "mixed_join_kernel.hpp" namespace cudf { namespace detail { -template __global__ void mixed_join( +template void launch_mixed_join( table_device_view left_table, table_device_view right_table, table_device_view probe, @@ -32,7 +33,10 @@ template __global__ void mixed_join( size_type* join_output_r, cudf::ast::detail::expression_device_view device_expression_data, cudf::size_type const* join_result_offsets, - bool const swap_tables); + bool const swap_tables, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream); } // namespace detail diff --git a/cpp/src/join/mixed_join_kernels.cuh b/cpp/src/join/mixed_join_kernels.cuh deleted file mode 100644 index 037c02666d4..00000000000 --- a/cpp/src/join/mixed_join_kernels.cuh +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "join/join_common_utils.hpp" -#include "join/mixed_join_common_utils.cuh" - -#include -#include -#include - -namespace cudf { -namespace detail { - -/** - * @brief Computes the output size of joining the left table to the right table. - * - * This method probes the hash table with each row in the probe table using a - * custom equality comparator that also checks that the conditional expression - * evaluates to true between the left/right tables when a match is found - * between probe and build rows. - * - * @tparam block_size The number of threads per block for this kernel - * @tparam has_nulls Whether or not the inputs may contain nulls. - * - * @param[in] left_table The left table - * @param[in] right_table The right table - * @param[in] probe The table with which to probe the hash table for matches. - * @param[in] build The table with which the hash table was built. - * @param[in] hash_probe The hasher used for the probe table. - * @param[in] equality_probe The equality comparator used when probing the hash table. - * @param[in] join_type The type of join to be performed - * @param[in] hash_table_view The hash table built from `build`. - * @param[in] device_expression_data Container of device data required to evaluate the desired - * expression. - * @param[in] swap_tables If true, the kernel was launched with one thread per right row and - * the kernel needs to internally loop over left rows. Otherwise, loop over right rows. - * @param[out] output_size The resulting output size - * @param[out] matches_per_row The number of matches in one pair of - * equality/conditional tables for each row in the other pair of tables. If - * swap_tables is true, matches_per_row corresponds to the right_table, - * otherwise it corresponds to the left_table. Note that corresponding swap of - * left/right tables to determine which is the build table and which is the - * probe table has already happened on the host. - */ - -template -__global__ void compute_mixed_join_output_size( - table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - join_kind const join_type, - cudf::detail::mixed_multimap_type::device_view hash_table_view, - ast::detail::expression_device_view device_expression_data, - bool const swap_tables, - std::size_t* output_size, - cudf::device_span matches_per_row); - -/** - * @brief Performs a join using the combination of a hash lookup to identify - * equal rows between one pair of tables and the evaluation of an expression - * containing an arbitrary expression. - * - * This method probes the hash table with each row in the probe table using a - * custom equality comparator that also checks that the conditional expression - * evaluates to true between the left/right tables when a match is found - * between probe and build rows. - * - * @tparam block_size The number of threads per block for this kernel - * @tparam has_nulls Whether or not the inputs may contain nulls. - * - * @param[in] left_table The left table - * @param[in] right_table The right table - * @param[in] probe The table with which to probe the hash table for matches. - * @param[in] build The table with which the hash table was built. - * @param[in] hash_probe The hasher used for the probe table. - * @param[in] equality_probe The equality comparator used when probing the hash table. - * @param[in] join_type The type of join to be performed - * @param[in] hash_table_view The hash table built from `build`. - * @param[out] join_output_l The left result of the join operation - * @param[out] join_output_r The right result of the join operation - * @param[in] device_expression_data Container of device data required to evaluate the desired - * expression. - * @param[in] join_result_offsets The starting indices in join_output[l|r] - * where the matches for each row begin. Equivalent to a prefix sum of - * matches_per_row. - * @param[in] swap_tables If true, the kernel was launched with one thread per right row and - * the kernel needs to internally loop over left rows. Otherwise, loop over right rows. - */ -template -__global__ void mixed_join(table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - join_kind const join_type, - cudf::detail::mixed_multimap_type::device_view hash_table_view, - size_type* join_output_l, - size_type* join_output_r, - cudf::ast::detail::expression_device_view device_expression_data, - cudf::size_type const* join_result_offsets, - bool const swap_tables); - -} // namespace detail - -} // namespace cudf diff --git a/cpp/src/join/mixed_join_kernels_semi.cu b/cpp/src/join/mixed_join_kernels_semi.cu index 1f31eaa7878..7459ac3e99c 100644 --- a/cpp/src/join/mixed_join_kernels_semi.cu +++ b/cpp/src/join/mixed_join_kernels_semi.cu @@ -14,9 +14,7 @@ * limitations under the License. */ -#include "join/join_common_utils.cuh" -#include "join/join_common_utils.hpp" -#include "join/mixed_join_common_utils.cuh" +#include "join/mixed_join_kernels_semi.cuh" #include #include @@ -35,16 +33,16 @@ namespace cg = cooperative_groups; #pragma GCC diagnostic ignored "-Wattributes" template -CUDF_HIDDEN __launch_bounds__(block_size) __global__ - void mixed_join_semi(table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - cudf::detail::semi_map_type::device_view hash_table_view, - cudf::device_span left_table_keep_mask, - cudf::ast::detail::expression_device_view device_expression_data) +CUDF_KERNEL void __launch_bounds__(block_size) + mixed_join_semi(table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + cudf::detail::semi_map_type::device_view hash_table_view, + cudf::device_span left_table_keep_mask, + cudf::ast::detail::expression_device_view device_expression_data) { // Normally the casting of a shared memory array is used to create multiple // arrays of different types from the shared memory buffer, but here it is @@ -75,28 +73,46 @@ CUDF_HIDDEN __launch_bounds__(block_size) __global__ } } -template __global__ void mixed_join_semi( - table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - cudf::detail::semi_map_type::device_view hash_table_view, - cudf::device_span left_table_keep_mask, - cudf::ast::detail::expression_device_view device_expression_data); - -template __global__ void mixed_join_semi( - table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - cudf::detail::semi_map_type::device_view hash_table_view, - cudf::device_span left_table_keep_mask, - cudf::ast::detail::expression_device_view device_expression_data); +void launch_mixed_join_semi(bool has_nulls, + table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + cudf::detail::semi_map_type::device_view hash_table_view, + cudf::device_span left_table_keep_mask, + cudf::ast::detail::expression_device_view device_expression_data, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream) +{ + if (has_nulls) { + mixed_join_semi + <<>>( + left_table, + right_table, + probe, + build, + hash_probe, + equality_probe, + hash_table_view, + left_table_keep_mask, + device_expression_data); + } else { + mixed_join_semi + <<>>( + left_table, + right_table, + probe, + build, + hash_probe, + equality_probe, + hash_table_view, + left_table_keep_mask, + device_expression_data); + } +} } // namespace detail - } // namespace cudf diff --git a/cpp/src/join/mixed_join_kernels_semi.cuh b/cpp/src/join/mixed_join_kernels_semi.cuh index 4ea404d451c..43714ffb36a 100644 --- a/cpp/src/join/mixed_join_kernels_semi.cuh +++ b/cpp/src/join/mixed_join_kernels_semi.cuh @@ -16,8 +16,9 @@ #pragma once -#include "join/join_common_utils.hpp" -#include "join/mixed_join_common_utils.cuh" +#include "join_common_utils.cuh" +#include "join_common_utils.hpp" +#include "mixed_join_common_utils.cuh" #include #include @@ -39,6 +40,7 @@ namespace detail { * @tparam block_size The number of threads per block for this kernel * @tparam has_nulls Whether or not the inputs may contain nulls. * + * @param[in] has_nulls If the input has nulls * @param[in] left_table The left table * @param[in] right_table The right table * @param[in] probe The table with which to probe the hash table for matches. @@ -51,16 +53,19 @@ namespace detail { * @param[in] device_expression_data Container of device data required to evaluate the desired * expression. */ -template -__global__ void mixed_join_semi(table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - cudf::detail::semi_map_type::device_view hash_table_view, - cudf::device_span left_table_keep_mask, - cudf::ast::detail::expression_device_view device_expression_data); +void launch_mixed_join_semi(bool has_nulls, + table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + cudf::detail::semi_map_type::device_view hash_table_view, + cudf::device_span left_table_keep_mask, + cudf::ast::detail::expression_device_view device_expression_data, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream); } // namespace detail diff --git a/cpp/src/join/mixed_join_semi.cu b/cpp/src/join/mixed_join_semi.cu index 3e4188a0fbd..a79aa6673d6 100644 --- a/cpp/src/join/mixed_join_semi.cu +++ b/cpp/src/join/mixed_join_semi.cu @@ -227,31 +227,19 @@ std::unique_ptr> mixed_join_semi( // Vector used to indicate indices from left/probe table which are present in output auto left_table_keep_mask = rmm::device_uvector(probe.num_rows(), stream); - if (has_nulls) { - mixed_join_semi - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - hash_table_view, - cudf::device_span(left_table_keep_mask), - parser.device_expression_data); - } else { - mixed_join_semi - <<>>( - *left_conditional_view, - *right_conditional_view, - *probe_view, - *build_view, - hash_probe, - equality_probe, - hash_table_view, - cudf::device_span(left_table_keep_mask), - parser.device_expression_data); - } + launch_mixed_join_semi(has_nulls, + *left_conditional_view, + *right_conditional_view, + *probe_view, + *build_view, + hash_probe, + equality_probe, + hash_table_view, + cudf::device_span(left_table_keep_mask), + parser.device_expression_data, + config, + shmem_size_per_block, + stream); auto gather_map = std::make_unique>(probe.num_rows(), stream, mr); diff --git a/cpp/src/join/mixed_join_size_kernel.cu b/cpp/src/join/mixed_join_size_kernel.cu index 4011acb65d6..4882c8769e6 100644 --- a/cpp/src/join/mixed_join_size_kernel.cu +++ b/cpp/src/join/mixed_join_size_kernel.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #include "mixed_join_size_kernel.cuh" +#include "mixed_join_size_kernel.hpp" namespace cudf { namespace detail { -template __global__ void compute_mixed_join_output_size( +template std::size_t launch_compute_mixed_join_output_size( table_device_view left_table, table_device_view right_table, table_device_view probe, @@ -30,8 +31,11 @@ template __global__ void compute_mixed_join_output_size matches_per_row); + cudf::device_span matches_per_row, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); } // namespace detail } // namespace cudf diff --git a/cpp/src/join/mixed_join_size_kernel.cuh b/cpp/src/join/mixed_join_size_kernel.cuh index 00a90f8273f..a1066e32331 100644 --- a/cpp/src/join/mixed_join_size_kernel.cuh +++ b/cpp/src/join/mixed_join_size_kernel.cuh @@ -36,19 +36,19 @@ namespace cg = cooperative_groups; #pragma GCC diagnostic ignored "-Wattributes" template -CUDF_HIDDEN __launch_bounds__(block_size) __global__ void compute_mixed_join_output_size( - table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - join_kind const join_type, - cudf::detail::mixed_multimap_type::device_view hash_table_view, - ast::detail::expression_device_view device_expression_data, - bool const swap_tables, - std::size_t* output_size, - cudf::device_span matches_per_row) +CUDF_KERNEL void __launch_bounds__(block_size) + compute_mixed_join_output_size(table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + join_kind const join_type, + cudf::detail::mixed_multimap_type::device_view hash_table_view, + ast::detail::expression_device_view device_expression_data, + bool const swap_tables, + std::size_t* output_size, + cudf::device_span matches_per_row) { // The (required) extern storage of the shared memory array leads to // conflicting declarations between different templates. The easiest @@ -103,5 +103,43 @@ CUDF_HIDDEN __launch_bounds__(block_size) __global__ void compute_mixed_join_out } } +template +std::size_t launch_compute_mixed_join_output_size( + table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + join_kind const join_type, + cudf::detail::mixed_multimap_type::device_view hash_table_view, + ast::detail::expression_device_view device_expression_data, + bool const swap_tables, + cudf::device_span matches_per_row, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + // Allocate storage for the counter used to get the size of the join output + rmm::device_scalar size(0, stream, mr); + + compute_mixed_join_output_size + <<>>( + left_table, + right_table, + probe, + build, + hash_probe, + equality_probe, + join_type, + hash_table_view, + device_expression_data, + swap_tables, + size.data(), + matches_per_row); + return size.value(stream); +} + } // namespace detail } // namespace cudf diff --git a/cpp/src/join/mixed_join_size_kernel.hpp b/cpp/src/join/mixed_join_size_kernel.hpp new file mode 100644 index 00000000000..b09805c14dc --- /dev/null +++ b/cpp/src/join/mixed_join_size_kernel.hpp @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "join_common_utils.cuh" +#include "join_common_utils.hpp" +#include "mixed_join_common_utils.cuh" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace CUDF_EXPORT cudf { +namespace detail { + +/** + * @brief Computes the output size of joining the left table to the right table. + * + * This method probes the hash table with each row in the probe table using a + * custom equality comparator that also checks that the conditional expression + * evaluates to true between the left/right tables when a match is found + * between probe and build rows. + * + * @tparam block_size The number of threads per block for this kernel + * @tparam has_nulls Whether or not the inputs may contain nulls. + * + * @param[in] left_table The left table + * @param[in] right_table The right table + * @param[in] probe The table with which to probe the hash table for matches. + * @param[in] build The table with which the hash table was built. + * @param[in] hash_probe The hasher used for the probe table. + * @param[in] equality_probe The equality comparator used when probing the hash table. + * @param[in] join_type The type of join to be performed + * @param[in] hash_table_view The hash table built from `build`. + * @param[in] device_expression_data Container of device data required to evaluate the desired + * expression. + * @param[in] swap_tables If true, the kernel was launched with one thread per right row and + * the kernel needs to internally loop over left rows. Otherwise, loop over right rows. + * @param[out] output_size The resulting output size + * @param[out] matches_per_row The number of matches in one pair of + * equality/conditional tables for each row in the other pair of tables. If + * swap_tables is true, matches_per_row corresponds to the right_table, + * otherwise it corresponds to the left_table. Note that corresponding swap of + * left/right tables to determine which is the build table and which is the + * probe table has already happened on the host. + */ + +template +std::size_t launch_compute_mixed_join_output_size( + cudf::table_device_view left_table, + cudf::table_device_view right_table, + cudf::table_device_view probe, + cudf::table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + join_kind const join_type, + cudf::detail::mixed_multimap_type::device_view hash_table_view, + ast::detail::expression_device_view device_expression_data, + bool const swap_tables, + cudf::device_span matches_per_row, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); +} // namespace detail +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/join/mixed_join_size_kernel_nulls.cu b/cpp/src/join/mixed_join_size_kernel_nulls.cu index 2868113bf33..11f9103da4d 100644 --- a/cpp/src/join/mixed_join_size_kernel_nulls.cu +++ b/cpp/src/join/mixed_join_size_kernel_nulls.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ namespace cudf { namespace detail { -template __global__ void compute_mixed_join_output_size( +template std::size_t launch_compute_mixed_join_output_size( table_device_view left_table, table_device_view right_table, table_device_view probe, @@ -30,8 +30,10 @@ template __global__ void compute_mixed_join_output_size matches_per_row); - + cudf::device_span matches_per_row, + detail::grid_1d const config, + int64_t shmem_size_per_block, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); } // namespace detail } // namespace cudf From a2503913bb362e43fa77615748ed4b4e31ac5055 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Mon, 26 Aug 2024 09:26:32 -0700 Subject: [PATCH 119/270] Revise `get_reader_filepath_or_buffer` to handle a list of data sources (#16613) The cudf read APIs (e.g. `cudf.read_parquet`, `cudf.read_json`, etc...) currently iterate over data sources, calling `get_reader_filepath_or_buffer` on each source independently when multiple files are mapped to the same `cudf.DataFrame`. This is suboptimal when the data sources are remote-file paths (e.g. in S3). In this case, we **should** be initiating network transfers for all files in parallel (and as early as possible). This PR makes it easier to optimize multi-file data transfer in follow-up work. It also simplifies and centralizes some of the common logic used by the various read APIs. Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16613 --- python/cudf/cudf/io/avro.py | 17 +-- python/cudf/cudf/io/csv.py | 15 +-- python/cudf/cudf/io/json.py | 58 ++------- python/cudf/cudf/io/orc.py | 41 ++---- python/cudf/cudf/io/parquet.py | 59 ++------- python/cudf/cudf/io/text.py | 6 +- python/cudf/cudf/utils/ioutils.py | 210 ++++++++++++++++-------------- 7 files changed, 161 insertions(+), 245 deletions(-) diff --git a/python/cudf/cudf/io/avro.py b/python/cudf/cudf/io/avro.py index 728b34045bf..964bd02b03e 100644 --- a/python/cudf/cudf/io/avro.py +++ b/python/cudf/cudf/io/avro.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import cudf from cudf import _lib as libcudf @@ -15,22 +15,13 @@ def read_avro( ): """{docstring}""" - is_single_filepath_or_buffer = ioutils.ensure_single_filepath_or_buffer( + filepath_or_buffer = ioutils.get_reader_filepath_or_buffer( path_or_data=filepath_or_buffer, storage_options=storage_options, ) - if not is_single_filepath_or_buffer: - raise NotImplementedError( - "`read_avro` does not yet support reading multiple files" - ) - - filepath_or_buffer, compression = ioutils.get_reader_filepath_or_buffer( - path_or_data=filepath_or_buffer, - compression=None, - storage_options=storage_options, + filepath_or_buffer = ioutils._select_single_source( + filepath_or_buffer, "read_avro" ) - if compression is not None: - ValueError("URL content-encoding decompression is not supported") return cudf.DataFrame._from_data( *libcudf.avro.read_avro( diff --git a/python/cudf/cudf/io/csv.py b/python/cudf/cudf/io/csv.py index e61fc5063dc..a9c20150930 100644 --- a/python/cudf/cudf/io/csv.py +++ b/python/cudf/cudf/io/csv.py @@ -64,22 +64,15 @@ def read_csv( if bytes_per_thread is None: bytes_per_thread = ioutils._BYTES_PER_THREAD_DEFAULT - is_single_filepath_or_buffer = ioutils.ensure_single_filepath_or_buffer( + filepath_or_buffer = ioutils.get_reader_filepath_or_buffer( path_or_data=filepath_or_buffer, - storage_options=storage_options, - ) - if not is_single_filepath_or_buffer: - raise NotImplementedError( - "`read_csv` does not yet support reading multiple files" - ) - - filepath_or_buffer, compression = ioutils.get_reader_filepath_or_buffer( - path_or_data=filepath_or_buffer, - compression=compression, iotypes=(BytesIO, StringIO), storage_options=storage_options, bytes_per_thread=bytes_per_thread, ) + filepath_or_buffer = ioutils._select_single_source( + filepath_or_buffer, "read_csv" + ) if na_values is not None and is_scalar(na_values): na_values = [na_values] diff --git a/python/cudf/cudf/io/json.py b/python/cudf/cudf/io/json.py index fc3387d5117..d86db656fd0 100644 --- a/python/cudf/cudf/io/json.py +++ b/python/cudf/cudf/io/json.py @@ -9,7 +9,6 @@ import cudf from cudf._lib import json as libjson -from cudf.api.types import is_list_like from cudf.utils import ioutils from cudf.utils.dtypes import _maybe_convert_to_default_type @@ -62,37 +61,15 @@ def read_json( f"following positional arguments: {list(args)}" ) - # Multiple sources are passed as a list. If a single source is passed, - # wrap it in a list for unified processing downstream. - if not is_list_like(path_or_buf): - path_or_buf = [path_or_buf] - - filepaths_or_buffers = [] - for source in path_or_buf: - if ioutils.is_directory( - path_or_data=source, storage_options=storage_options - ): - fs = ioutils._ensure_filesystem( - passed_filesystem=None, - path=source, - storage_options=storage_options, - ) - source = ioutils.stringify_pathlike(source) - source = fs.sep.join([source, "*.json"]) - - tmp_source, compression = ioutils.get_reader_filepath_or_buffer( - path_or_data=source, - compression=compression, - iotypes=(BytesIO, StringIO), - allow_raw_text_input=True, - storage_options=storage_options, - warn_on_raw_text_input=True, - warn_meta=("json", "read_json"), - ) - if isinstance(tmp_source, list): - filepaths_or_buffers.extend(tmp_source) - else: - filepaths_or_buffers.append(tmp_source) + filepaths_or_buffers = ioutils.get_reader_filepath_or_buffer( + path_or_buf, + iotypes=(BytesIO, StringIO), + allow_raw_text_input=True, + storage_options=storage_options, + warn_on_raw_text_input=True, + warn_meta=("json", "read_json"), + expand_dir_pattern="*.json", + ) df = libjson.read_json( filepaths_or_buffers=filepaths_or_buffers, @@ -111,25 +88,18 @@ def read_json( "be GPU accelerated in the future" ) - if not ioutils.ensure_single_filepath_or_buffer( - path_or_data=path_or_buf, - storage_options=storage_options, - ): - raise NotImplementedError( - "`read_json` does not yet support reading " - "multiple files via pandas" - ) - - path_or_buf, compression = ioutils.get_reader_filepath_or_buffer( + filepath_or_buffer = ioutils.get_reader_filepath_or_buffer( path_or_data=path_or_buf, - compression=compression, iotypes=(BytesIO, StringIO), allow_raw_text_input=True, storage_options=storage_options, ) + filepath_or_buffer = ioutils._select_single_source( + filepath_or_buffer, "read_json (via pandas)" + ) pd_value = pd.read_json( - path_or_buf, + filepath_or_buffer, lines=lines, dtype=dtype, compression=compression, diff --git a/python/cudf/cudf/io/orc.py b/python/cudf/cudf/io/orc.py index 4f04caafc5d..fd246c6215f 100644 --- a/python/cudf/cudf/io/orc.py +++ b/python/cudf/cudf/io/orc.py @@ -4,7 +4,6 @@ import warnings import pyarrow as pa -from fsspec.utils import stringify_path import cudf from cudf._lib import orc as liborc @@ -170,8 +169,11 @@ def read_orc_statistics( files_statistics = [] stripes_statistics = [] for source in filepaths_or_buffers: - path_or_buf, _ = ioutils.get_reader_filepath_or_buffer( - path_or_data=source, compression=None, **kwargs + path_or_buf = ioutils.get_reader_filepath_or_buffer( + path_or_data=source, **kwargs + ) + path_or_buf = ioutils._select_single_source( + path_or_buf, "read_orc_statistics" ) ( column_names, @@ -318,33 +320,12 @@ def read_orc( "A list of stripes must be provided for each input source" ) - filepaths_or_buffers = [] - for source in filepath_or_buffer: - if ioutils.is_directory( - path_or_data=source, storage_options=storage_options - ): - fs = ioutils._ensure_filesystem( - passed_filesystem=None, - path=source, - storage_options=storage_options, - ) - source = stringify_path(source) - source = fs.sep.join([source, "*.orc"]) - - tmp_source, compression = ioutils.get_reader_filepath_or_buffer( - path_or_data=source, - compression=None, - storage_options=storage_options, - bytes_per_thread=bytes_per_thread, - ) - if compression is not None: - raise ValueError( - "URL content-encoding decompression is not supported" - ) - if isinstance(tmp_source, list): - filepaths_or_buffers.extend(tmp_source) - else: - filepaths_or_buffers.append(tmp_source) + filepaths_or_buffers = ioutils.get_reader_filepath_or_buffer( + path_or_data=filepath_or_buffer, + storage_options=storage_options, + bytes_per_thread=bytes_per_thread, + expand_dir_pattern="*.orc", + ) if filters is not None: selected_stripes = _filter_stripes( diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 560f257c115..6b895abbf66 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -329,39 +329,12 @@ def write_to_dataset( @_performance_tracking def read_parquet_metadata(filepath_or_buffer): """{docstring}""" - # Multiple sources are passed as a list. If a single source is passed, - # wrap it in a list for unified processing downstream. - if not is_list_like(filepath_or_buffer): - filepath_or_buffer = [filepath_or_buffer] - - # Start by trying to construct a filesystem object - fs, paths = ioutils._get_filesystem_and_paths( - path_or_data=filepath_or_buffer, storage_options=None - ) - - # Check if filepath or buffer - filepath_or_buffer = paths if paths else filepath_or_buffer # List of filepaths or buffers - filepaths_or_buffers = [] - - for source in filepath_or_buffer: - tmp_source, compression = ioutils.get_reader_filepath_or_buffer( - path_or_data=source, - compression=None, - fs=fs, - storage_options=None, - bytes_per_thread=None, - ) - - if compression is not None: - raise ValueError( - "URL content-encoding decompression is not supported" - ) - if isinstance(tmp_source, list): - filepath_or_buffer.extend(tmp_source) - else: - filepaths_or_buffers.append(tmp_source) + filepaths_or_buffers = ioutils.get_reader_filepath_or_buffer( + path_or_data=filepath_or_buffer, + bytes_per_thread=None, + ) return libparquet.read_parquet_metadata(filepaths_or_buffers) @@ -598,24 +571,12 @@ def read_parquet( ) filepath_or_buffer = paths if paths else filepath_or_buffer - filepaths_or_buffers = [] - for source in filepath_or_buffer: - tmp_source, compression = ioutils.get_reader_filepath_or_buffer( - path_or_data=source, - compression=None, - fs=fs, - storage_options=storage_options, - bytes_per_thread=bytes_per_thread, - ) - - if compression is not None: - raise ValueError( - "URL content-encoding decompression is not supported" - ) - if isinstance(tmp_source, list): - filepath_or_buffer.extend(tmp_source) - else: - filepaths_or_buffers.append(tmp_source) + filepaths_or_buffers = ioutils.get_reader_filepath_or_buffer( + path_or_data=filepath_or_buffer, + fs=fs, + storage_options=storage_options, + bytes_per_thread=bytes_per_thread, + ) # Warn user if they are not using cudf for IO # (There is a good chance this was not the intention) diff --git a/python/cudf/cudf/io/text.py b/python/cudf/cudf/io/text.py index 4329480bb2c..0043efce1e4 100644 --- a/python/cudf/cudf/io/text.py +++ b/python/cudf/cudf/io/text.py @@ -24,12 +24,14 @@ def read_text( if delimiter is None: raise ValueError("delimiter needs to be provided") - filepath_or_buffer, _ = ioutils.get_reader_filepath_or_buffer( + filepath_or_buffer = ioutils.get_reader_filepath_or_buffer( path_or_data=filepath_or_buffer, - compression=None, iotypes=(BytesIO, StringIO), storage_options=storage_options, ) + filepath_or_buffer = ioutils._select_single_source( + filepath_or_buffer, "read_text" + ) return cudf.Series._from_data( libtext.read_text( diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 18106e7475b..e5944d7093c 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -14,6 +14,7 @@ import pandas as pd from fsspec.core import expand_paths_if_needed, get_fs_token_paths +from cudf.api.types import is_list_like from cudf.core._compat import PANDAS_LT_300 from cudf.utils.docutils import docfmt_partial @@ -799,7 +800,7 @@ k1 k2 0 1.0 [1] """ # noqa: E501 -doc_read_json = docfmt_partial(docstring=_docstring_read_json) +doc_read_json: Callable = docfmt_partial(docstring=_docstring_read_json) _docstring_to_json = """ Convert the cuDF object to a JSON string. @@ -869,7 +870,7 @@ -------- cudf.read_json """ -doc_to_json = docfmt_partial(docstring=_docstring_to_json) +doc_to_json: Callable = docfmt_partial(docstring=_docstring_to_json) _docstring_read_hdf = """ Read from the store, close it if we opened it. @@ -1399,13 +1400,14 @@ Return either a filepath string to data, or a memory buffer of data. If filepath, then the source filepath is expanded to user's environment. If buffer, then data is returned in-memory as bytes or a ByteIO object. +This function is designed to process multiple data sources of the same +type at once. If path_or_data is a list, the output will also be a list. Parameters ---------- -path_or_data : str, file-like object, bytes, ByteIO - Path to data or the data itself. -compression : str - Type of compression algorithm for the content +path_or_data : str, file-like object, bytes, ByteIO, list + Path to data or the data itself. Pass in a list to process multiple + sources of the same type at once. mode : str Mode in which file is opened iotypes : (), default (BytesIO) @@ -1430,14 +1432,15 @@ better throughput by decomposing it and transferring multiple "blocks" in parallel (using a Python thread pool). Default allocation is {bytes_per_thread} bytes. +expand_dir_pattern : str, default None + Glob pattern to use when expanding directories into file paths + (e.g. "*.json"). If this parameter is not specified, directories + will not be expanded. Returns ------- -filepath_or_buffer : str, bytes, BytesIO, list - Filepath string or in-memory buffer of data or a - list of Filepath strings or in-memory buffers of data. -compression : str - Type of compression algorithm for the content +List[str, bytes, BytesIO] + List of filepath strings or in-memory data buffers. """.format(bytes_per_thread=_BYTES_PER_THREAD_DEFAULT) @@ -1494,29 +1497,15 @@ def _is_local_filesystem(fs): return isinstance(fs, fsspec.implementations.local.LocalFileSystem) -def ensure_single_filepath_or_buffer(path_or_data, storage_options=None): - """Return False if `path_or_data` resolves to multiple filepaths or - buffers. +def _select_single_source(sources: list, caller: str): + """Select the first element from a list of sources. + Raise an error if sources contains multiple elements """ - path_or_data = stringify_pathlike(path_or_data) - if isinstance(path_or_data, str): - path_or_data = os.path.expanduser(path_or_data) - try: - fs, _, paths = get_fs_token_paths( - path_or_data, mode="rb", storage_options=storage_options - ) - except ValueError as e: - if str(e).startswith("Protocol not known"): - return True - else: - raise e - - if len(paths) > 1: - return False - elif isinstance(path_or_data, (list, tuple)) and len(path_or_data) > 1: - return False - - return True + if len(sources) > 1: + raise ValueError( + f"{caller} does not support multiple sources, got: {sources}" + ) + return sources[0] def is_directory(path_or_data, storage_options=None): @@ -1601,10 +1590,24 @@ def _get_filesystem_and_paths( return fs, return_paths +def _maybe_expand_directories(paths, glob_pattern, fs): + # Expand directory paths using a glob pattern. + # This is a no-op if either glob_pattern or fs are None + if fs is None or glob_pattern is None: + return paths + expanded_paths = [] + for path in paths: + if fs.isdir(path): + expanded_paths.extend(fs.glob(fs.sep.join([path, glob_pattern]))) + else: + expanded_paths.append(path) + return expanded_paths + + @doc_get_reader_filepath_or_buffer() def get_reader_filepath_or_buffer( path_or_data, - compression, + *, mode="rb", fs=None, iotypes=(BytesIO,), @@ -1613,32 +1616,38 @@ def get_reader_filepath_or_buffer( bytes_per_thread=_BYTES_PER_THREAD_DEFAULT, warn_on_raw_text_input=None, warn_meta=None, + expand_dir_pattern=None, ): """{docstring}""" - path_or_data = stringify_pathlike(path_or_data) - - if isinstance(path_or_data, str): - # Get a filesystem object if one isn't already available - paths = [path_or_data] + # Convert path_or_data to a list of input data sources + input_sources = [ + stringify_pathlike(source) + for source in ( + path_or_data if is_list_like(path_or_data) else [path_or_data] + ) + ] + if not input_sources: + raise ValueError("Empty input source list: {input_sources}.") + + filepaths_or_buffers = [] + string_paths = [isinstance(source, str) for source in input_sources] + if any(string_paths): + # Sources are all strings. Thes strings are typically + # file paths, but they may also be raw text strings. + + # Don't allow a mix of source types + if not all(string_paths): + raise ValueError("Invalid input source list: {input_sources}.") + + # Make sure we define a filesystem (if possible) + paths = input_sources + raw_text_input = False if fs is None: - fs, paths = _get_filesystem_and_paths( - path_or_data, storage_options - ) - if fs is None: - if warn_on_raw_text_input: - # Do not remove until pandas 3.0 support is added. - assert ( - PANDAS_LT_300 - ), "Need to drop after pandas-3.0 support is added." - warnings.warn( - f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " - "deprecated and will be removed in a future version. " - "To read from a literal string, wrap it in a " - "'StringIO' object.", - FutureWarning, - ) - return path_or_data, compression + fs, paths = _get_filesystem_and_paths(paths, storage_options) + + # Expand directories (if necessary) + paths = _maybe_expand_directories(paths, expand_dir_pattern, fs) if _is_local_filesystem(fs): # Doing this as `read_json` accepts a json string @@ -1660,7 +1669,7 @@ def get_reader_filepath_or_buffer( if len(paths): if fs.exists(paths[0]): - path_or_data = paths if len(paths) > 1 else paths[0] + filepaths_or_buffers = paths # raise FileNotFound if path looks like json # following pandas @@ -1670,21 +1679,40 @@ def get_reader_filepath_or_buffer( tuple(f".json{c}" for c in compression_extensions) ): raise FileNotFoundError( - f"{path_or_data} could not be resolved to any files" + f"{input_sources} could not be resolved to any files" ) - elif warn_on_raw_text_input: - # Do not remove until pandas 3.0 support is added. - assert ( - PANDAS_LT_300 - ), "Need to drop after pandas-3.0 support is added." - warnings.warn( - f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " - "deprecated and will be removed in a future version. " - "To read from a literal string, wrap it in a " - "'StringIO' object.", - FutureWarning, + else: + raw_text_input = True + else: + raw_text_input = True + + elif fs is not None: + # TODO: We can use cat_ranges and/or parquet-aware logic + # to copy all remote data into host memory at once here. + # The current solution iterates over files, and copies + # ALL data from each file (even when we are performing + # partial IO, and don't need the entire file) + if len(paths) == 0: + raise FileNotFoundError( + f"{input_sources} could not be resolved to any files" + ) + filepaths_or_buffers = [ + BytesIO( + _fsspec_data_transfer( + fpath, + fs=fs, + mode=mode, + bytes_per_thread=bytes_per_thread, ) - elif warn_on_raw_text_input: + ) + for fpath in paths + ] + else: + raw_text_input = True + + if raw_text_input: + filepaths_or_buffers = input_sources + if warn_on_raw_text_input: # Do not remove until pandas 3.0 support is added. assert ( PANDAS_LT_300 @@ -1697,35 +1725,25 @@ def get_reader_filepath_or_buffer( FutureWarning, ) - else: - if len(paths) == 0: - raise FileNotFoundError( - f"{path_or_data} could not be resolved to any files" - ) - path_or_data = [ - BytesIO( - _fsspec_data_transfer( - fpath, - fs=fs, - mode=mode, - bytes_per_thread=bytes_per_thread, + else: + # Sources are already buffers or file-like objects + for source in input_sources: + if not isinstance(source, iotypes) and is_file_like(source): + if isinstance(source, TextIOWrapper): + source = source.buffer + filepaths_or_buffers.append( + BytesIO( + _fsspec_data_transfer( + source, + mode=mode, + bytes_per_thread=bytes_per_thread, + ) ) ) - for fpath in paths - ] - if len(path_or_data) == 1: - path_or_data = path_or_data[0] - - elif not isinstance(path_or_data, iotypes) and is_file_like(path_or_data): - if isinstance(path_or_data, TextIOWrapper): - path_or_data = path_or_data.buffer - path_or_data = BytesIO( - _fsspec_data_transfer( - path_or_data, mode=mode, bytes_per_thread=bytes_per_thread - ) - ) + else: + filepaths_or_buffers.append(source) - return path_or_data, compression + return filepaths_or_buffers def get_writer_filepath_or_buffer(path_or_data, mode, storage_options=None): From d15d470e526de205bed8808a9c15d0a4d7642667 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 26 Aug 2024 12:03:41 -0500 Subject: [PATCH 120/270] Preserve Series name in duplicated method. (#16655) Closes #16654. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16655 --- python/cudf/cudf/core/indexed_frame.py | 4 +++- python/cudf/cudf/tests/test_series.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 60253b9ae5d..ad6aa56d472 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3198,8 +3198,10 @@ def duplicated(self, subset=None, keep="first"): """ subset = self._preprocess_subset(subset) + name = None if isinstance(self, cudf.Series): columns = [self._column] + name = self.name else: columns = [self._data[n] for n in subset] distinct = libcudf.stream_compaction.distinct_indices( @@ -3211,7 +3213,7 @@ def duplicated(self, subset=None, keep="first"): [as_column(True, length=len(self), dtype=bool)], bounds_check=False, )[0] - return cudf.Series._from_column(result, index=self.index) + return cudf.Series._from_column(result, index=self.index, name=name) @_performance_tracking def _empty_like(self, keep_index=True) -> Self: diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index c7aea563535..8d673e23ab2 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2115,8 +2115,9 @@ def test_series_hasnans(data): ], ) @pytest.mark.parametrize("keep", ["first", "last", False]) -def test_series_duplicated(data, index, keep): - gs = cudf.Series(data, index=index) +@pytest.mark.parametrize("name", [None, "a"]) +def test_series_duplicated(data, index, keep, name): + gs = cudf.Series(data, index=index, name=name) ps = gs.to_pandas() assert_eq(gs.duplicated(keep=keep), ps.duplicated(keep=keep)) From f5113228c3aa89d49e71d42d11c38afe52695aa6 Mon Sep 17 00:00:00 2001 From: "Marcus D. Hanwell" Date: Mon, 26 Aug 2024 16:19:30 -0400 Subject: [PATCH 121/270] bug-fix: Don't enable the CUDA language if testing was requested when finding cudf (#16615) This PR removes CMake code enabling the CUDA language if the testing component was requested. Closes #16614 Authors: - Marcus D. Hanwell (https://github.com/cryos) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16615 --- cpp/CMakeLists.txt | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6b8bb26825b..a6f72ed6b75 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1069,23 +1069,12 @@ if(CUDF_ENABLE_ARROW_PARQUET) ) endif() -string( - APPEND - install_code_string - [=[ -if(testing IN_LIST cudf_FIND_COMPONENTS) - enable_language(CUDA) -endif() -]=] -) - rapids_export( INSTALL cudf EXPORT_SET cudf-exports ${_components_export_string} GLOBAL_TARGETS cudf cudftestutil NAMESPACE cudf:: DOCUMENTATION doc_string - FINAL_CODE_BLOCK install_code_string ) # ################################################################################################## From c4591c06db5347ea2bf6e37ead678343042a7932 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:23:43 -0400 Subject: [PATCH 122/270] Use non-mangled type names in nvbench output (#16649) Uses the `NVBENCH_DECLARE_TYPE_STRINGS` feature to produce readable type names in the nvbench output. Example previous output for `cudf::timestamp_ms` would appear like this: ``` | cuda::std::__4::chrono::time_point > > | 100000 | 23840x | 25.138 us | 21.98% | 20.979 us | 9.54% | 4.767G | 38.134 GB/s | 4.38% | ``` Adding the nvbench name feature changes this to: ``` | cudf::timestamp_ms | 100000 | 24752x | 24.387 us | 21.58% | 20.208 us | 3.79% | 4.948G | 39.588 GB/s | 4.55% | ``` Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/16649 --- cpp/benchmarks/reduction/minmax.cpp | 2 ++ cpp/benchmarks/reduction/reduce.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cpp/benchmarks/reduction/minmax.cpp b/cpp/benchmarks/reduction/minmax.cpp index c89e22d3f44..636de303cc4 100644 --- a/cpp/benchmarks/reduction/minmax.cpp +++ b/cpp/benchmarks/reduction/minmax.cpp @@ -47,6 +47,8 @@ static void reduction_minmax(nvbench::state& state, nvbench::type_list set_throughputs(state); } +NVBENCH_DECLARE_TYPE_STRINGS(cudf::timestamp_ms, "cudf::timestamp_ms", "cudf::timestamp_ms"); + using Types = nvbench::type_list; NVBENCH_BENCH_TYPES(reduction_minmax, NVBENCH_TYPE_AXES(Types)) diff --git a/cpp/benchmarks/reduction/reduce.cpp b/cpp/benchmarks/reduction/reduce.cpp index 14bf90c4943..a30c27c519c 100644 --- a/cpp/benchmarks/reduction/reduce.cpp +++ b/cpp/benchmarks/reduction/reduce.cpp @@ -81,6 +81,8 @@ static void reduction(nvbench::state& state, nvbench::type_list; using AggKinds = nvbench::enum_type_list Date: Tue, 27 Aug 2024 09:34:51 -0400 Subject: [PATCH 123/270] Fix integer overflow in indexalator pointer logic (#16643) Fixes integer overflow in the indexalator logic when incrementing/decrementing its data pointer. Any sufficiently large int32 input values used in computing the byte-pointer position causes an overflow when multiplying the value by the byte-width of the underlying index type. For example, this overflow would occur when accessing rows greater than 536,870,912 with an underlying index type of int32 (4-bytes). Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Paul Mattione (https://github.com/pmattione-nvidia) URL: https://github.com/rapidsai/cudf/pull/16643 --- cpp/include/cudf/detail/indexalator.cuh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/include/cudf/detail/indexalator.cuh b/cpp/include/cudf/detail/indexalator.cuh index ec7b1c3e6b6..f0510c86c3a 100644 --- a/cpp/include/cudf/detail/indexalator.cuh +++ b/cpp/include/cudf/detail/indexalator.cuh @@ -93,7 +93,7 @@ struct input_indexalator : base_normalator { */ __device__ inline cudf::size_type operator[](size_type idx) const { - void const* tp = p_ + (idx * this->width_); + void const* tp = p_ + (static_cast(idx) * this->width_); return type_dispatcher(this->dtype_, normalize_type{}, tp); } @@ -109,7 +109,7 @@ struct input_indexalator : base_normalator { CUDF_HOST_DEVICE input_indexalator(void const* data, data_type dtype, cudf::size_type offset = 0) : base_normalator(dtype), p_{static_cast(data)} { - p_ += offset * this->width_; + p_ += static_cast(offset) * this->width_; } protected: @@ -165,7 +165,7 @@ struct output_indexalator : base_normalator __device__ inline output_indexalator const operator[](size_type idx) const { output_indexalator tmp{*this}; - tmp.p_ += (idx * this->width_); + tmp.p_ += static_cast(idx) * this->width_; return tmp; } From efa97704d0c1ee83d04ab59f1746194c86743656 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 27 Aug 2024 11:02:22 -0500 Subject: [PATCH 124/270] Drop Python 3.9 support (#16637) Contributes to https://github.com/rapidsai/build-planning/issues/88 Finishes the work of dropping Python 3.9 support. This project stopped building / testing against Python 3.9 as of https://github.com/rapidsai/shared-workflows/pull/235. This PR updates configuration and docs to reflect that. ## Notes for Reviewers ### How I tested this Checked that there were no remaining uses like this: ```shell git grep -E '3\.9' git grep '39' git grep 'py39' ``` And similar for variations on Python 3.8 (to catch things that were missed the last time this was done). Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16637 --- README.md | 2 +- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-125_arch-x86_64.yaml | 2 +- cpp/cmake/thirdparty/get_arrow.cmake | 2 +- dependencies.yaml | 6 +---- python/cudf/pyproject.toml | 3 +-- python/cudf_kafka/pyproject.toml | 2 +- .../cudf_polars/containers/dataframe.py | 13 +++++---- python/cudf_polars/cudf_polars/dsl/ir.py | 27 ++++++++++++------- .../cudf_polars/typing/__init__.py | 4 +-- .../cudf_polars/cudf_polars/utils/sorting.py | 2 +- python/cudf_polars/pyproject.toml | 12 ++++++--- python/custreamz/pyproject.toml | 3 +-- python/dask_cudf/pyproject.toml | 3 +-- python/pylibcudf/pyproject.toml | 3 +-- 15 files changed, 47 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index fd8b0365807..f1b010394d6 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ conda install -c rapidsai -c conda-forge -c nvidia \ We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD of our latest development branch. -Note: cuDF is supported only on Linux, and with Python versions 3.9 and later. +Note: cuDF is supported only on Linux, and with Python versions 3.10 and later. See the [RAPIDS installation guide](https://docs.rapids.ai/install) for more OS and version info. diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 5cf7508ba51..fcd6e27a7f6 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -76,7 +76,7 @@ dependencies: - pytest-xdist - pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 -- python>=3.9,<3.12 +- python>=3.10,<3.12 - pytorch>=2.1.0 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 28b927254f7..bedc3a90885 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -74,7 +74,7 @@ dependencies: - pytest-xdist - pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 -- python>=3.9,<3.12 +- python>=3.10,<3.12 - pytorch>=2.1.0 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 0afdc526981..e3e6a07661a 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -45,7 +45,7 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION) APPEND initial_code_block [=[ -find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) +find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) execute_process( COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_library_dirs()[0])" OUTPUT_VARIABLE CUDF_PYARROW_WHEEL_DIR diff --git a/dependencies.yaml b/dependencies.yaml index 194577817db..04b5940c9fb 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -584,10 +584,6 @@ dependencies: specific: - output_types: conda matrices: - - matrix: - py: "3.9" - packages: - - python=3.9 - matrix: py: "3.10" packages: @@ -598,7 +594,7 @@ dependencies: - python=3.11 - matrix: packages: - - python>=3.9,<3.12 + - python>=3.10,<3.12 run_common: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index e7bac17f8ba..a6d26d17d46 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -16,7 +16,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "cachetools", "cubinlinker", @@ -42,7 +42,6 @@ classifiers = [ "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 2d0222a3fe9..01e7299a33a 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -16,7 +16,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "cudf==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index 7c28e7b9a6c..a5c99e2bc11 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -105,7 +105,9 @@ def from_polars(cls, df: pl.DataFrame) -> Self: return cls( [ NamedColumn(column, h_col.name).copy_metadata(h_col) - for column, h_col in zip(d_table.columns(), df.iter_columns()) + for column, h_col in zip( + d_table.columns(), df.iter_columns(), strict=True + ) ] ) @@ -134,8 +136,10 @@ def from_table(cls, table: plc.Table, names: Sequence[str]) -> Self: if table.num_columns() != len(names): raise ValueError("Mismatching name and table length.") return cls( - # TODO: strict=True when we drop py39 - [NamedColumn(c, name) for c, name in zip(table.columns(), names)] + [ + NamedColumn(c, name) + for c, name in zip(table.columns(), names, strict=True) + ] ) def sorted_like( @@ -165,8 +169,7 @@ def sorted_like( subset = self.column_names_set if subset is None else subset self.columns = [ c.sorted_like(other) if c.name in subset else c - # TODO: strict=True when we drop py39 - for c, other in zip(self.columns, like.columns) + for c, other in zip(self.columns, like.columns, strict=True) ] return self diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 019f00f4fca..ebc7dee6bfb 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -310,7 +310,8 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: *( (piece.tbl, piece.column_names(include_children=False)) for piece in pieces - ) + ), + strict=True, ) df = DataFrame.from_table( plc.concatenate.concatenate(list(tables)), @@ -426,7 +427,8 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: pdf = pdf.select(self.projection) df = DataFrame.from_polars(pdf) assert all( - c.obj.type() == dtype for c, dtype in zip(df.columns, self.schema.values()) + c.obj.type() == dtype + for c, dtype in zip(df.columns, self.schema.values(), strict=True) ) if self.predicate is not None: (mask,) = broadcast(self.predicate.evaluate(df), target_length=df.num_rows) @@ -600,9 +602,10 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: for i, table in enumerate(raw_tables): (column,) = table.columns() raw_columns.append(NamedColumn(column, f"tmp{i}")) - mapping = dict(zip(replacements, raw_columns)) + mapping = dict(zip(replacements, raw_columns, strict=True)) result_keys = [ - NamedColumn(gk, k.name) for gk, k in zip(group_keys.columns(), keys) + NamedColumn(gk, k.name) + for gk, k in zip(group_keys.columns(), keys, strict=True) ] result_subs = DataFrame(raw_columns) results = [ @@ -752,7 +755,9 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: columns = plc.join.cross_join(left.table, right.table).columns() left_cols = [ NamedColumn(new, old.name).sorted_like(old) - for new, old in zip(columns[: left.num_columns], left.columns) + for new, old in zip( + columns[: left.num_columns], left.columns, strict=True + ) ] right_cols = [ NamedColumn( @@ -761,7 +766,9 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: if old.name not in left.column_names_set else f"{old.name}{suffix}", ) - for new, old in zip(columns[left.num_columns :], right.columns) + for new, old in zip( + columns[left.num_columns :], right.columns, strict=True + ) ] return DataFrame([*left_cols, *right_cols]) # TODO: Waiting on clarity based on https://github.com/pola-rs/polars/issues/17184 @@ -803,6 +810,7 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: for left_col, right_col in zip( left.select_columns(left_on.column_names_set), right.select_columns(right_on.column_names_set), + strict=True, ) ) ) @@ -909,7 +917,7 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: result = DataFrame( [ NamedColumn(c, old.name).sorted_like(old) - for c, old in zip(table.columns(), df.columns) + for c, old in zip(table.columns(), df.columns, strict=True) ] ) if keys_sorted or self.stable: @@ -974,7 +982,8 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: self.null_order, ) columns = [ - NamedColumn(c, old.name) for c, old in zip(table.columns(), df.columns) + NamedColumn(c, old.name) + for c, old in zip(table.columns(), df.columns, strict=True) ] # If a sort key is in the result table, set the sortedness property for k, i in enumerate(keys_in_result): @@ -1089,7 +1098,7 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: # final tag is "swapping" which is useful for the # optimiser (it blocks some pushdown operations) old, new, _ = self.options - return df.rename_columns(dict(zip(old, new))) + return df.rename_columns(dict(zip(old, new, strict=True))) elif self.name == "explode": df = self.df.evaluate(cache=cache) ((to_explode,),) = self.options diff --git a/python/cudf_polars/cudf_polars/typing/__init__.py b/python/cudf_polars/cudf_polars/typing/__init__.py index 02440e67fde..5276073e62a 100644 --- a/python/cudf_polars/cudf_polars/typing/__init__.py +++ b/python/cudf_polars/cudf_polars/typing/__init__.py @@ -13,9 +13,7 @@ from polars.polars import _expr_nodes as pl_expr, _ir_nodes as pl_ir if TYPE_CHECKING: - from typing import Callable - - from typing_extensions import TypeAlias + from typing import Callable, TypeAlias import polars as pl diff --git a/python/cudf_polars/cudf_polars/utils/sorting.py b/python/cudf_polars/cudf_polars/utils/sorting.py index 17ea44e5b1b..6ce216cbf8f 100644 --- a/python/cudf_polars/cudf_polars/utils/sorting.py +++ b/python/cudf_polars/cudf_polars/utils/sorting.py @@ -45,7 +45,7 @@ def sort_order( null_precedence = [] if len(descending) != len(nulls_last) or len(descending) != num_keys: raise ValueError("Mismatching length of arguments in sort_order") - for asc, null_last in zip(column_order, nulls_last): + for asc, null_last in zip(column_order, nulls_last, strict=True): if (asc == plc.types.Order.ASCENDING) ^ (not null_last): null_precedence.append(plc.types.NullOrder.AFTER) elif (asc == plc.types.Order.ASCENDING) ^ null_last: diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index c380853035d..0382e3ce6a2 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -17,7 +17,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "polars>=1.0,<1.3", "pylibcudf==24.10.*,>=0.0.0a0", @@ -28,7 +28,6 @@ classifiers = [ "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] @@ -62,7 +61,7 @@ exclude_also = [ [tool.ruff] line-length = 88 indent-width = 4 -target-version = "py39" +target-version = "py310" fix = true [tool.ruff.lint] @@ -115,6 +114,9 @@ ignore = [ "TD003", # Missing issue link on the line following this TODO # tryceratops "TRY003", # Avoid specifying long messages outside the exception class + # pyupgrade + "UP035", # Import from `collections.abc` instead: `Callable` + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` # Lints below are turned off because of conflicts with the ruff # formatter # See https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules @@ -137,6 +139,10 @@ fixable = ["ALL"] [tool.ruff.lint.per-file-ignores] "**/tests/**/*.py" = ["D"] +"**/cudf_polars/typing/__init__.py" = [ + # pyupgrade + "UP007", # Use `X | Y` for type annotations +] [tool.ruff.lint.flake8-pytest-style] # https://docs.astral.sh/ruff/settings/#lintflake8-pytest-style diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index d6b88167262..be5331236a5 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -17,7 +17,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "confluent-kafka>=1.9.0,<1.10.0a0", "cudf==24.10.*,>=0.0.0a0", @@ -31,7 +31,6 @@ classifiers = [ "Topic :: Apache Kafka", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index d5da7030a75..93bf532d67f 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -17,7 +17,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "cudf==24.10.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", @@ -32,7 +32,6 @@ classifiers = [ "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index 5f5594b462b..0d673ea4cc3 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -16,7 +16,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "cuda-python>=11.7.1,<12.0a0", "libcudf==24.10.*,>=0.0.0a0", @@ -32,7 +32,6 @@ classifiers = [ "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] From f1cc962df38b1fc113b579bef57a27f93d11cec2 Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Tue, 27 Aug 2024 09:16:39 -0700 Subject: [PATCH 125/270] Fix `cudf::rank` not getting enough params (#16666) Fix issue #16624 Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) Approvers: - Mark Harris (https://github.com/harrism) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16666 --- cpp/benchmarks/sort/rank_lists.cpp | 2 ++ cpp/benchmarks/sort/rank_structs.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cpp/benchmarks/sort/rank_lists.cpp b/cpp/benchmarks/sort/rank_lists.cpp index fbdb40b3537..7015fe08089 100644 --- a/cpp/benchmarks/sort/rank_lists.cpp +++ b/cpp/benchmarks/sort/rank_lists.cpp @@ -37,6 +37,8 @@ void nvbench_rank_lists(nvbench::state& state, nvbench::type_list Date: Tue, 27 Aug 2024 09:30:16 -0700 Subject: [PATCH 126/270] Add `num_multiprocessors` utility (#16628) This PR introduces a new `num_multiprocessors` utility and moves the existing `elements_per_thread` host utility to the new `cuda.hpp` header. Needed by #16619. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - David Wendt (https://github.com/davidwendt) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/16628 --- cpp/CMakeLists.txt | 1 + cpp/benchmarks/join/generate_input_tables.cuh | 10 +--- cpp/include/cudf/detail/copy_if.cuh | 1 + cpp/include/cudf/detail/utilities/cuda.cuh | 29 --------- cpp/include/cudf/detail/utilities/cuda.hpp | 59 +++++++++++++++++++ cpp/src/io/comp/debrotli.cu | 18 +++--- cpp/src/utilities/cuda.cpp | 34 +++++++++++ 7 files changed, 105 insertions(+), 47 deletions(-) create mode 100644 cpp/include/cudf/detail/utilities/cuda.hpp create mode 100644 cpp/src/utilities/cuda.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a6f72ed6b75..4080c5d02da 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -666,6 +666,7 @@ add_library( src/unary/math_ops.cu src/unary/nan_ops.cu src/unary/null_ops.cu + src/utilities/cuda.cpp src/utilities/cuda_memcpy.cu src/utilities/default_stream.cpp src/utilities/host_memory.cpp diff --git a/cpp/benchmarks/join/generate_input_tables.cuh b/cpp/benchmarks/join/generate_input_tables.cuh index f7984b29d6b..75bbe8174d3 100644 --- a/cpp/benchmarks/join/generate_input_tables.cuh +++ b/cpp/benchmarks/join/generate_input_tables.cuh @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include #include @@ -150,13 +151,8 @@ void generate_input_tables(key_type* const build_tbl, CUDF_CUDA_TRY(cudaOccupancyMaxActiveBlocksPerMultiprocessor( &num_blocks_init_probe_tbl, init_probe_tbl, block_size, 0)); - int dev_id{-1}; - CUDF_CUDA_TRY(cudaGetDevice(&dev_id)); - - int num_sms{-1}; - CUDF_CUDA_TRY(cudaDeviceGetAttribute(&num_sms, cudaDevAttrMultiProcessorCount, dev_id)); - - int const num_states = + auto const num_sms = cudf::detail::num_multiprocessors(); + auto const num_states = num_sms * std::max(num_blocks_init_build_tbl, num_blocks_init_probe_tbl) * block_size; rmm::device_uvector devStates(num_states, cudf::get_default_stream()); diff --git a/cpp/include/cudf/detail/copy_if.cuh b/cpp/include/cudf/detail/copy_if.cuh index b6310e6cd2f..4071fa01fb2 100644 --- a/cpp/include/cudf/detail/copy_if.cuh +++ b/cpp/include/cudf/detail/copy_if.cuh @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/cpp/include/cudf/detail/utilities/cuda.cuh b/cpp/include/cudf/detail/utilities/cuda.cuh index 5007af7f9f1..d31ca3d92d1 100644 --- a/cpp/include/cudf/detail/utilities/cuda.cuh +++ b/cpp/include/cudf/detail/utilities/cuda.cuh @@ -189,35 +189,6 @@ __device__ T single_lane_block_sum_reduce(T lane_value) return result; } -/** - * @brief Get the number of elements that can be processed per thread. - * - * @param[in] kernel The kernel for which the elements per thread needs to be assessed - * @param[in] total_size Number of elements - * @param[in] block_size Expected block size - * - * @return cudf::size_type Elements per thread that can be processed for given specification. - */ -template -cudf::size_type elements_per_thread(Kernel kernel, - cudf::size_type total_size, - cudf::size_type block_size, - cudf::size_type max_per_thread = 32) -{ - CUDF_FUNC_RANGE(); - - // calculate theoretical occupancy - int max_blocks = 0; - CUDF_CUDA_TRY(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks, kernel, block_size, 0)); - - int device = 0; - CUDF_CUDA_TRY(cudaGetDevice(&device)); - int num_sms = 0; - CUDF_CUDA_TRY(cudaDeviceGetAttribute(&num_sms, cudaDevAttrMultiProcessorCount, device)); - int per_thread = total_size / (max_blocks * num_sms * block_size); - return std::clamp(per_thread, 1, max_per_thread); -} - /** * @brief Finds the smallest value not less than `number_to_round` and modulo `modulus` is * zero. Expects modulus to be a power of 2. diff --git a/cpp/include/cudf/detail/utilities/cuda.hpp b/cpp/include/cudf/detail/utilities/cuda.hpp new file mode 100644 index 00000000000..58c7ae8ed6a --- /dev/null +++ b/cpp/include/cudf/detail/utilities/cuda.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include + +namespace CUDF_EXPORT cudf { +namespace detail { + +/** + * @brief Get the number of multiprocessors on the device + */ +cudf::size_type num_multiprocessors(); + +/** + * @brief Get the number of elements that can be processed per thread. + * + * @param[in] kernel The kernel for which the elements per thread needs to be assessed + * @param[in] total_size Number of elements + * @param[in] block_size Expected block size + * + * @return cudf::size_type Elements per thread that can be processed for given specification. + */ +template +cudf::size_type elements_per_thread(Kernel kernel, + cudf::size_type total_size, + cudf::size_type block_size, + cudf::size_type max_per_thread = 32) +{ + CUDF_FUNC_RANGE(); + + // calculate theoretical occupancy + int max_blocks = 0; + CUDF_CUDA_TRY(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks, kernel, block_size, 0)); + + int per_thread = total_size / (max_blocks * num_multiprocessors() * block_size); + return std::clamp(per_thread, 1, max_per_thread); +} + +} // namespace detail +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/io/comp/debrotli.cu b/cpp/src/io/comp/debrotli.cu index 861820f47e7..72649dbe427 100644 --- a/cpp/src/io/comp/debrotli.cu +++ b/cpp/src/io/comp/debrotli.cu @@ -58,6 +58,7 @@ THE SOFTWARE. #include "gpuinflate.hpp" #include "io/utilities/block_utils.cuh" +#include #include #include @@ -2047,19 +2048,14 @@ CUDF_KERNEL void __launch_bounds__(block_size, 2) */ size_t __host__ get_gpu_debrotli_scratch_size(int max_num_inputs) { - int sm_count = 0; - int dev = 0; uint32_t max_fb_size, min_fb_size, fb_size; - CUDF_CUDA_TRY(cudaGetDevice(&dev)); - if (cudaSuccess == cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, dev)) { - // printf("%d SMs on device %d\n", sm_count, dev); - max_num_inputs = - min(max_num_inputs, sm_count * 3); // no more than 3 blocks/sm at most due to 32KB smem use - if (max_num_inputs <= 0) { - max_num_inputs = sm_count * 2; // Target 2 blocks/SM by default for scratch mem computation - } + auto const sm_count = cudf::detail::num_multiprocessors(); + // no more than 3 blocks/sm at most due to 32KB smem use + max_num_inputs = std::min(max_num_inputs, sm_count * 3); + if (max_num_inputs <= 0) { + max_num_inputs = sm_count * 2; // Target 2 blocks/SM by default for scratch mem computation } - max_num_inputs = min(max(max_num_inputs, 1), 512); + max_num_inputs = std::min(std::max(max_num_inputs, 1), 512); // Max fb size per block occurs if all huffman tables for all 3 group types fail local_alloc() // with num_htrees=256 (See HuffmanTreeGroupAlloc) max_fb_size = 256 * (630 + 1080 + 920) * 2; // 1.3MB diff --git a/cpp/src/utilities/cuda.cpp b/cpp/src/utilities/cuda.cpp new file mode 100644 index 00000000000..53ca0608170 --- /dev/null +++ b/cpp/src/utilities/cuda.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +namespace cudf::detail { + +cudf::size_type num_multiprocessors() +{ + int device = 0; + CUDF_CUDA_TRY(cudaGetDevice(&device)); + int num_sms = 0; + CUDF_CUDA_TRY(cudaDeviceGetAttribute(&num_sms, cudaDevAttrMultiProcessorCount, device)); + return num_sms; +} + +} // namespace cudf::detail From dd585e84756992bee0ecbae6f77107d64cddaede Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 27 Aug 2024 12:58:02 -0400 Subject: [PATCH 127/270] Prune workflows based on changed files (#16642) Only run tests based on things that have actually changed. For example, if only Python files have changed, we don't need to run the C++ tests. Contributes to https://github.com/rapidsai/build-planning/issues/94 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Robert Maynard (https://github.com/robertmaynard) URL: https://github.com/rapidsai/cudf/pull/16642 --- .github/workflows/pr.yaml | 88 ++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2e2a8b6b9bc..35c7e3d95b6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,6 +12,7 @@ concurrency: jobs: pr-builder: needs: + - changed-files - checks - conda-cpp-build - conda-cpp-checks @@ -37,6 +38,63 @@ jobs: - pandas-tests-diff secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + if: always() + with: + needs: ${{ toJSON(needs) }} + changed-files: + runs-on: ubuntu-latest + name: "Check changed files" + outputs: + test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} + test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} + test_notebooks: ${{ steps.changed-files.outputs.notebooks_any_changed == 'true' }} + test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + steps: + - name: Get PR info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Checkout code repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + fetch-depth: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).commits }} + persist-credentials: false + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + files_yaml: | + cpp: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!java/**' + - '!notebooks/**' + - '!python/**' + java: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + notebooks: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!java/**' + python: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!java/**' + - '!notebooks/**' checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 @@ -56,9 +114,10 @@ jobs: build_type: pull-request enable_check_symbols: true conda-cpp-tests: - needs: conda-cpp-build + needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_cpp == 'true' with: build_type: pull-request conda-python-build: @@ -68,24 +127,27 @@ jobs: with: build_type: pull-request conda-python-cudf-tests: - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: build_type: pull-request script: "ci/test_python_cudf.sh" conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: build_type: pull-request script: "ci/test_python_other.sh" conda-java-tests: - needs: conda-cpp-build + needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + if: needs.changed-files.outputs.test_java == 'true' with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -103,9 +165,10 @@ jobs: container_image: "rapidsai/ci-wheel:latest" run_script: "ci/configure_cpp_static.sh" conda-notebook-tests: - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + if: needs.changed-files.outputs.test_notebooks == 'true' with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -145,9 +208,10 @@ jobs: build_type: pull-request script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: - needs: wheel-build-cudf + needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: build_type: pull-request script: ci/test_wheel_cudf.sh @@ -161,9 +225,10 @@ jobs: build_type: pull-request script: "ci/build_wheel_cudf_polars.sh" wheel-tests-cudf-polars: - needs: wheel-build-cudf-polars + needs: [wheel-build-cudf-polars, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -181,9 +246,10 @@ jobs: build_type: pull-request script: "ci/build_wheel_dask_cudf.sh" wheel-tests-dask-cudf: - needs: wheel-build-dask-cudf + needs: [wheel-build-dask-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -200,9 +266,10 @@ jobs: build-all -DBUILD_BENCHMARKS=ON --verbose; sccache -s; unit-tests-cudf-pandas: - needs: wheel-build-cudf + needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -210,9 +277,10 @@ jobs: script: ci/cudf_pandas_scripts/run_tests.sh pandas-tests: # run the Pandas unit tests using PR branch - needs: wheel-build-cudf + needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) From 6747d2dc9d0deb4585b6306fed8a41bdf65e5558 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 27 Aug 2024 14:48:14 -0400 Subject: [PATCH 128/270] Update rapidsai/pre-commit-hooks (#16669) This PR updates rapidsai/pre-commit-hooks to the version 0.4.0. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16669 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b17eae0842..f861fb57916 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -144,7 +144,7 @@ repos: - id: ruff-format files: python/.*$ - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.3.1 + rev: v0.4.0 hooks: - id: verify-copyright exclude: | From 1a2aad27b7e136f87be80debed6da7d3528ebda1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 27 Aug 2024 13:33:47 -0700 Subject: [PATCH 129/270] Remove arrow dependency (#16640) This PR removes libarrow as a dependency of libcudf since we no longer use any of its APIs in our C++ code. The following places remain dependent on libarrow: - tests: We have tests demonstrating how to interoperate with libarrow objects, as well as other tests that leverage Arrow for I/O. - examples: We have an example demonstrating interop with libarrow arrays. - JNI: The JNI is still using libarrow to handle ingestion or production of Arrow buffers. In all three cases above, we are now statically linking libarrow. We also always pull it in via CPM, which means that we never require libarrow to exist on the user's system anymore. Of the above three cases, we should expect the first two to persist indefinitely. The JNI could be updated to use nanoarrow instead if desired, but that is not critical since the primary benefit of removing libarrow as a direct dependency is to remove it as a constraint for package managers such as conda in environments where we must match the version of Arrow required by other dependencies. pyarrow remains a dependency of the cudf Python packages. For now, this PR retains the tight pinning on 16.1 since we know that this version works. A future PR will loosen this pinning since we are no longer constrained to ABI-compatible versions and can support a range of pyarrow versions that support the necessary Python APIs (I believe pyarrow>=13 will work, but that remains to be tested). Resolves #15193 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - James Lamb (https://github.com/jameslamb) - Robert Maynard (https://github.com/robertmaynard) - https://github.com/jakirkham - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cudf/pull/16640 --- ci/build_wheel_cudf.sh | 1 - ci/build_wheel_libcudf.sh | 2 +- ci/build_wheel_pylibcudf.sh | 1 - .../all_cuda-118_arch-x86_64.yaml | 6 - .../all_cuda-125_arch-x86_64.yaml | 6 - conda/recipes/cudf/meta.yaml | 4 +- conda/recipes/libcudf/conda_build_config.yaml | 3 - conda/recipes/libcudf/meta.yaml | 2 - conda/recipes/pylibcudf/meta.yaml | 4 +- cpp/CMakeLists.txt | 27 +- cpp/cmake/thirdparty/get_arrow.cmake | 285 +++++++----------- cpp/examples/interop/CMakeLists.txt | 7 + cpp/tests/CMakeLists.txt | 23 +- dependencies.yaml | 42 +-- java/src/main/native/CMakeLists.txt | 4 + python/cudf/CMakeLists.txt | 1 - python/cudf/cudf/_lib/CMakeLists.txt | 3 - python/cudf/cudf/_lib/io/CMakeLists.txt | 2 - python/cudf/pyproject.toml | 2 - .../cudf_kafka/cudf_kafka/_lib/CMakeLists.txt | 2 - python/cudf_kafka/pyproject.toml | 2 - python/libcudf/CMakeLists.txt | 9 +- python/libcudf/libcudf/load.py | 4 - python/libcudf/pyproject.toml | 1 - python/pylibcudf/CMakeLists.txt | 1 - .../cmake/Modules/LinkPyarrowHeaders.cmake | 40 --- python/pylibcudf/pylibcudf/io/CMakeLists.txt | 5 - .../pylibcudf/libcudf/io/CMakeLists.txt | 3 - python/pylibcudf/pyproject.toml | 4 +- 29 files changed, 145 insertions(+), 351 deletions(-) delete mode 100644 python/pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index cf33703f544..e5565c4b53c 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -22,7 +22,6 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ - --exclude libarrow.so.1601 \ --exclude libnvcomp.so \ --exclude libnvcomp_bitcomp.so \ --exclude libnvcomp_gdeflate.so \ diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 9694c3f6144..8975381ceba 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -10,6 +10,6 @@ package_dir="python/libcudf" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" mkdir -p ${package_dir}/final_dist -python -m auditwheel repair --exclude libarrow.so.1601 -w ${package_dir}/final_dist ${package_dir}/dist/* +python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 7181a49d397..0e4745bda28 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -20,7 +20,6 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ - --exclude libarrow.so.1601 \ --exclude libnvcomp.so \ --exclude libnvcomp_bitcomp.so \ --exclude libnvcomp_gdeflate.so \ diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index fcd6e27a7f6..96596958636 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -37,15 +37,11 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==16.1.0.* -- libarrow-dataset==16.1.0.* -- libarrow==16.1.0.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 - libcurand=10.3.0.86 - libkvikio==24.10.*,>=0.0.0a0 -- libparquet==16.1.0.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.10.*,>=0.0.0a0 - make @@ -56,7 +52,6 @@ dependencies: - ninja - notebook - numba>=0.57 -- numpy - numpy>=1.23,<3.0a0 - numpydoc - nvcc_linux-64=11.8 @@ -68,7 +63,6 @@ dependencies: - pandoc - pre-commit - ptxcompiler -- pyarrow==16.1.0.* - pydata-sphinx-theme!=0.14.2 - pytest-benchmark - pytest-cases>=3.8.2 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index bedc3a90885..efc5f76b90f 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -38,13 +38,9 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==16.1.0.* -- libarrow-dataset==16.1.0.* -- libarrow==16.1.0.* - libcufile-dev - libcurand-dev - libkvikio==24.10.*,>=0.0.0a0 -- libparquet==16.1.0.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.10.*,>=0.0.0a0 - make @@ -55,7 +51,6 @@ dependencies: - ninja - notebook - numba>=0.57 -- numpy - numpy>=1.23,<3.0a0 - numpydoc - nvcomp==3.0.6 @@ -65,7 +60,6 @@ dependencies: - pandas>=2.0,<2.2.3dev0 - pandoc - pre-commit -- pyarrow==16.1.0.* - pydata-sphinx-theme!=0.14.2 - pynvjitlink>=0.0.0a0 - pytest-benchmark diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index b2dad767da4..53f52a35651 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -64,8 +64,6 @@ requirements: - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 - dlpack >=0.8,<1.0 - - numpy 2.0 - - pyarrow ==16.1.0.* - libcudf ={{ version }} - pylibcudf ={{ version }} - rmm ={{ minor_version }} @@ -84,7 +82,7 @@ requirements: - cupy >=12.0.0 - numba >=0.57 - numpy >=1.23,<3.0a0 - - {{ pin_compatible('pyarrow', max_pin='x.x') }} + - pyarrow ==16.1.0.* - libcudf ={{ version }} - pylibcudf ={{ version }} - {{ pin_compatible('rmm', max_pin='x.x') }} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index ff7458caf82..4b1c4cca828 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -19,9 +19,6 @@ c_stdlib_version: cmake_version: - ">=3.26.4,!=3.30.0" -libarrow_version: - - "==16.1.0" - dlpack_version: - ">=0.8,<1.0" diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index aa1c94a4bca..1c2e9e8dd98 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -64,7 +64,6 @@ requirements: {% endif %} - cuda-version ={{ cuda_version }} - nvcomp {{ nvcomp_version }} - - libarrow {{ libarrow_version }} - dlpack {{ dlpack_version }} - librdkafka {{ librdkafka_version }} - fmt {{ fmt_version }} @@ -92,7 +91,6 @@ outputs: - cmake {{ cmake_version }} host: - cuda-version ={{ cuda_version }} - - libarrow {{ libarrow_version }} run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} diff --git a/conda/recipes/pylibcudf/meta.yaml b/conda/recipes/pylibcudf/meta.yaml index fef78467027..67b9b76bb8c 100644 --- a/conda/recipes/pylibcudf/meta.yaml +++ b/conda/recipes/pylibcudf/meta.yaml @@ -64,8 +64,6 @@ requirements: - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 - dlpack >=0.8,<1.0 - - numpy 2.0 - - pyarrow ==16.1.0.* - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} @@ -81,7 +79,7 @@ requirements: - typing_extensions >=4.0.0 - pandas >=2.0,<2.2.3dev0 - numpy >=1.23,<3.0a0 - - {{ pin_compatible('pyarrow', max_pin='x.x') }} + - pyarrow ==16.1.0.* - {{ pin_compatible('rmm', max_pin='x.x') }} - fsspec >=0.6.0 {% if cuda_major == "11" %} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4080c5d02da..1040fcb7b91 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -54,11 +54,6 @@ mark_as_advanced(CUDF_BUILD_TESTUTIL) option(CUDF_USE_PROPRIETARY_NVCOMP "Download and use NVCOMP with proprietary extensions" ON) option(CUDF_LARGE_STRINGS_DISABLED "Build with large string support disabled" OFF) mark_as_advanced(CUDF_LARGE_STRINGS_DISABLED) -option(CUDF_USE_ARROW_STATIC "Build and statically link Arrow libraries" OFF) -option(CUDF_ENABLE_ARROW_ORC "Build the Arrow ORC adapter" OFF) -option(CUDF_ENABLE_ARROW_PYTHON "Find (or build) Arrow with Python support" OFF) -option(CUDF_ENABLE_ARROW_PARQUET "Find (or build) Arrow with Parquet support" OFF) -option(CUDF_ENABLE_ARROW_S3 "Build/Enable AWS S3 Arrow filesystem support" OFF) option( CUDF_USE_PER_THREAD_DEFAULT_STREAM "Build cuDF with per-thread default stream, including passing the per-thread default @@ -81,8 +76,6 @@ option(CUDA_ENABLE_LINEINFO option(CUDA_WARNINGS_AS_ERRORS "Enable -Werror=all-warnings for all CUDA compilation" ON) # cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) -option(USE_LIBARROW_FROM_PYARROW "Only use the libarrow contained in pyarrow" OFF) -mark_as_advanced(USE_LIBARROW_FROM_PYARROW) set(DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL ON) if(CUDA_STATIC_RUNTIME OR NOT BUILD_SHARED_LIBS) @@ -100,8 +93,6 @@ message(VERBOSE "CUDF: Configure CMake to build tests: ${BUILD_TESTS}") message(VERBOSE "CUDF: Configure CMake to build (google & nvbench) benchmarks: ${BUILD_BENCHMARKS}") message(VERBOSE "CUDF: Build cuDF shared libraries: ${BUILD_SHARED_LIBS}") message(VERBOSE "CUDF: Use a file cache for JIT compiled kernels: ${JITIFY_USE_CACHE}") -message(VERBOSE "CUDF: Build and statically link Arrow libraries: ${CUDF_USE_ARROW_STATIC}") -message(VERBOSE "CUDF: Build and enable S3 filesystem support for Arrow: ${CUDF_ENABLE_ARROW_S3}") message(VERBOSE "CUDF: Build with per-thread default stream: ${CUDF_USE_PER_THREAD_DEFAULT_STREAM}") message( VERBOSE @@ -192,8 +183,6 @@ include(cmake/thirdparty/get_nvcomp.cmake) include(cmake/thirdparty/get_cccl.cmake) # find rmm include(cmake/thirdparty/get_rmm.cmake) -# find arrow -include(cmake/thirdparty/get_arrow.cmake) # find flatbuffers include(cmake/thirdparty/get_flatbuffers.cmake) # find dlpack @@ -807,7 +796,7 @@ add_dependencies(cudf jitify_preprocess_run) # Specify the target module library dependencies target_link_libraries( cudf - PUBLIC ${ARROW_LIBRARIES} CCCL::CCCL rmm::rmm $ + PUBLIC CCCL::CCCL rmm::rmm $ PRIVATE $ cuco::cuco ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio $ nanoarrow ) @@ -1056,20 +1045,6 @@ following IMPORTED GLOBAL targets: ]=] ) -if(CUDF_ENABLE_ARROW_PARQUET) - string( - APPEND - install_code_string - [=[ - if(NOT Parquet_DIR) - set(Parquet_DIR "${Arrow_DIR}") - endif() - set(ArrowDataset_DIR "${Arrow_DIR}") - find_dependency(ArrowDataset) - ]=] - ) -endif() - rapids_export( INSTALL cudf EXPORT_SET cudf-exports ${_components_export_string} diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index e3e6a07661a..07cbf5150f4 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -22,82 +22,8 @@ include_guard(GLOBAL) -# Generate a FindArrow module for the case where we need to search for arrow within a pip install -# pyarrow. -function(find_libarrow_in_python_wheel PYARROW_VERSION) - string(REPLACE "." ";" PYARROW_VER_COMPONENTS "${PYARROW_VERSION}") - list(GET PYARROW_VER_COMPONENTS 0 PYARROW_MAJOR_VER) - list(GET PYARROW_VER_COMPONENTS 1 PYARROW_MINOR_VER) - - # Ensure that the major and minor versions are two digits long - string(LENGTH ${PYARROW_MAJOR_VER} PYARROW_MAJOR_LENGTH) - string(LENGTH ${PYARROW_MINOR_VER} PYARROW_MINOR_LENGTH) - if(${PYARROW_MAJOR_LENGTH} EQUAL 1) - set(PYARROW_MAJOR_VER "0${PYARROW_MAJOR_VER}") - endif() - if(${PYARROW_MINOR_LENGTH} EQUAL 1) - set(PYARROW_MINOR_VER "0${PYARROW_MINOR_VER}") - endif() - - set(PYARROW_LIB "libarrow.so.${PYARROW_MAJOR_VER}${PYARROW_MINOR_VER}") - - string( - APPEND - initial_code_block - [=[ -find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) -execute_process( - COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_library_dirs()[0])" - OUTPUT_VARIABLE CUDF_PYARROW_WHEEL_DIR - OUTPUT_STRIP_TRAILING_WHITESPACE - COMMAND_ERROR_IS_FATAL ANY -) -list(APPEND CMAKE_PREFIX_PATH "${CUDF_PYARROW_WHEEL_DIR}") -]=] - ) - string( - APPEND - final_code_block - [=[ -list(POP_BACK CMAKE_PREFIX_PATH) -]=] - ) - rapids_find_generate_module( - Arrow NO_CONFIG - VERSION "${PYARROW_VERSION}" - LIBRARY_NAMES "${PYARROW_LIB}" - BUILD_EXPORT_SET cudf-exports - INSTALL_EXPORT_SET cudf-exports - HEADER_NAMES arrow/python/arrow_to_pandas.h INITIAL_CODE_BLOCK initial_code_block - FINAL_CODE_BLOCK final_code_block - ) - - find_package(Arrow ${PYARROW_VERSION} MODULE REQUIRED GLOBAL) - add_library(arrow_shared ALIAS Arrow::Arrow) - - rapids_export_package(BUILD Arrow cudf-exports) - rapids_export_package(INSTALL Arrow cudf-exports) -endfunction() - # This function finds arrow and sets any additional necessary environment variables. -function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENABLE_PYTHON - ENABLE_PARQUET PYARROW_LIBARROW -) - - if(PYARROW_LIBARROW) - # Generate a FindArrow.cmake to find pyarrow's libarrow.so - find_libarrow_in_python_wheel(${VERSION}) - set(ARROW_FOUND - TRUE - PARENT_SCOPE - ) - set(ARROW_LIBRARIES - arrow_shared - PARENT_SCOPE - ) - return() - endif() - +function(find_and_configure_arrow VERSION BUILD_STATIC EXCLUDE_FROM_ALL ENABLE_PARQUET) if(BUILD_STATIC) if(TARGET arrow_static) set(ARROW_FOUND @@ -124,10 +50,6 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB endif() endif() - if(NOT ARROW_ARMV8_ARCH) - set(ARROW_ARMV8_ARCH "armv8-a") - endif() - if(NOT ARROW_SIMD_LEVEL) set(ARROW_SIMD_LEVEL "NONE") endif() @@ -150,14 +72,6 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB set(ARROW_OPENSSL_USE_SHARED ON) endif() - set(ARROW_PYTHON_OPTIONS "") - if(ENABLE_PYTHON) - list(APPEND ARROW_PYTHON_OPTIONS "ARROW_PYTHON ON") - # Arrow's logic to build Boost from source is busted, so we have to get it from the system. - list(APPEND ARROW_PYTHON_OPTIONS "BOOST_SOURCE SYSTEM") - list(APPEND ARROW_PYTHON_OPTIONS "ARROW_DEPENDENCY_SOURCE AUTO") - endif() - set(ARROW_PARQUET_OPTIONS "") if(ENABLE_PARQUET) # Arrow's logic to build Boost from source is busted, so we have to get it from the system. @@ -174,6 +88,7 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB GIT_REPOSITORY https://github.com/apache/arrow.git GIT_TAG apache-arrow-${VERSION} GIT_SHALLOW TRUE SOURCE_SUBDIR cpp + EXCLUDE_FROM_ALL ${EXCLUDE_FROM_ALL} OPTIONS "CMAKE_VERBOSE_MAKEFILE ON" "ARROW_ACERO ON" "ARROW_IPC ON" @@ -181,16 +96,14 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB "ARROW_WITH_BACKTRACE ON" "ARROW_CXXFLAGS -w" "ARROW_JEMALLOC OFF" - "ARROW_S3 ${ENABLE_S3}" - "ARROW_ORC ${ENABLE_ORC}" - # e.g. needed by blazingsql-io + "ARROW_S3 OFF" + "ARROW_ORC OFF" ${ARROW_PARQUET_OPTIONS} "ARROW_PARQUET ${ENABLE_PARQUET}" "ARROW_FILESYSTEM ON" - ${ARROW_PYTHON_OPTIONS} + "ARROW_PYTHON OFF" # Arrow modifies CMake's GLOBAL RULE_LAUNCH_COMPILE unless this is off "ARROW_USE_CCACHE OFF" - "ARROW_ARMV8_ARCH ${ARROW_ARMV8_ARCH}" "ARROW_SIMD_LEVEL ${ARROW_SIMD_LEVEL}" "ARROW_BUILD_STATIC ${ARROW_BUILD_STATIC}" "ARROW_BUILD_SHARED ${ARROW_BUILD_SHARED}" @@ -269,7 +182,6 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB endif() if(Arrow_ADDED) - set(arrow_code_string [=[ if (TARGET cudf::arrow_shared AND (NOT TARGET arrow_shared)) @@ -324,101 +236,106 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB get_target_property(interface_libs arrow_static INTERFACE_LINK_LIBRARIES) endif() endif() - rapids_export( - BUILD Arrow - VERSION ${VERSION} - EXPORT_SET arrow_targets - GLOBAL_TARGETS arrow_shared arrow_static - NAMESPACE cudf:: - FINAL_CODE_BLOCK arrow_code_string - ) - - if(ENABLE_PARQUET) - - set(arrow_acero_code_string - [=[ - if (TARGET cudf::arrow_acero_shared AND (NOT TARGET arrow_acero_shared)) - add_library(arrow_acero_shared ALIAS cudf::arrow_acero_shared) - endif() - if (TARGET cudf::arrow_acero_static AND (NOT TARGET arrow_acero_static)) - add_library(arrow_acero_static ALIAS cudf::arrow_acero_static) - endif() - ]=] - ) + include(rapids-export) + if(NOT EXCLUDE_FROM_ALL) rapids_export( - BUILD ArrowAcero + BUILD Arrow VERSION ${VERSION} - EXPORT_SET arrow_acero_targets - GLOBAL_TARGETS arrow_acero_shared arrow_acero_static + EXPORT_SET arrow_targets + GLOBAL_TARGETS arrow_shared arrow_static NAMESPACE cudf:: - FINAL_CODE_BLOCK arrow_acero_code_string + FINAL_CODE_BLOCK arrow_code_string ) - set(arrow_dataset_code_string - [=[ - if (TARGET cudf::arrow_dataset_shared AND (NOT TARGET arrow_dataset_shared)) - add_library(arrow_dataset_shared ALIAS cudf::arrow_dataset_shared) - endif() - if (TARGET cudf::arrow_dataset_static AND (NOT TARGET arrow_dataset_static)) - add_library(arrow_dataset_static ALIAS cudf::arrow_dataset_static) - endif() - ]=] - ) + if(ENABLE_PARQUET) + set(arrow_acero_code_string + [=[ + if (TARGET cudf::arrow_acero_shared AND (NOT TARGET arrow_acero_shared)) + add_library(arrow_acero_shared ALIAS cudf::arrow_acero_shared) + endif() + if (TARGET cudf::arrow_acero_static AND (NOT TARGET arrow_acero_static)) + add_library(arrow_acero_static ALIAS cudf::arrow_acero_static) + endif() + ]=] + ) - rapids_export( - BUILD ArrowDataset - VERSION ${VERSION} - EXPORT_SET arrow_dataset_targets - GLOBAL_TARGETS arrow_dataset_shared arrow_dataset_static - NAMESPACE cudf:: - FINAL_CODE_BLOCK arrow_dataset_code_string - ) + rapids_export( + BUILD ArrowAcero + VERSION ${VERSION} + EXPORT_SET arrow_acero_targets + GLOBAL_TARGETS arrow_acero_shared arrow_acero_static + NAMESPACE cudf:: + FINAL_CODE_BLOCK arrow_acero_code_string + ) - set(parquet_code_string - [=[ - if (TARGET cudf::parquet_shared AND (NOT TARGET parquet_shared)) - add_library(parquet_shared ALIAS cudf::parquet_shared) - endif() - if (TARGET cudf::parquet_static AND (NOT TARGET parquet_static)) - add_library(parquet_static ALIAS cudf::parquet_static) - endif() - ]=] - ) + set(arrow_dataset_code_string + [=[ + if (TARGET cudf::arrow_dataset_shared AND (NOT TARGET arrow_dataset_shared)) + add_library(arrow_dataset_shared ALIAS cudf::arrow_dataset_shared) + endif() + if (TARGET cudf::arrow_dataset_static AND (NOT TARGET arrow_dataset_static)) + add_library(arrow_dataset_static ALIAS cudf::arrow_dataset_static) + endif() + ]=] + ) - rapids_export( - BUILD Parquet - VERSION ${VERSION} - EXPORT_SET parquet_targets - GLOBAL_TARGETS parquet_shared parquet_static - NAMESPACE cudf:: - FINAL_CODE_BLOCK parquet_code_string - ) + rapids_export( + BUILD ArrowDataset + VERSION ${VERSION} + EXPORT_SET arrow_dataset_targets + GLOBAL_TARGETS arrow_dataset_shared arrow_dataset_static + NAMESPACE cudf:: + FINAL_CODE_BLOCK arrow_dataset_code_string + ) + set(parquet_code_string + [=[ + if (TARGET cudf::parquet_shared AND (NOT TARGET parquet_shared)) + add_library(parquet_shared ALIAS cudf::parquet_shared) + endif() + if (TARGET cudf::parquet_static AND (NOT TARGET parquet_static)) + add_library(parquet_static ALIAS cudf::parquet_static) + endif() + ]=] + ) + + rapids_export( + BUILD Parquet + VERSION ${VERSION} + EXPORT_SET parquet_targets + GLOBAL_TARGETS parquet_shared parquet_static + NAMESPACE cudf:: + FINAL_CODE_BLOCK parquet_code_string + ) + endif() endif() endif() - # We generate the arrow-configfiles when we built arrow locally, so always do `find_dependency` - rapids_export_package(BUILD Arrow cudf-exports) - rapids_export_package(INSTALL Arrow cudf-exports) - if(ENABLE_PARQUET) - rapids_export_package(BUILD Parquet cudf-exports) - rapids_export_package(BUILD ArrowDataset cudf-exports) - endif() + if(NOT EXCLUDE_FROM_ALL) + # We generate the arrow-configfiles when we built arrow locally, so always do `find_dependency` + rapids_export_package(BUILD Arrow cudf-exports) + rapids_export_package(INSTALL Arrow cudf-exports) - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root( - BUILD Arrow [=[${CMAKE_CURRENT_LIST_DIR}]=] EXPORT_SET cudf-exports - ) - rapids_export_find_package_root( - BUILD Parquet [=[${CMAKE_CURRENT_LIST_DIR}]=] - EXPORT_SET cudf-exports - CONDITION ENABLE_PARQUET - ) - rapids_export_find_package_root( - BUILD ArrowDataset [=[${CMAKE_CURRENT_LIST_DIR}]=] - EXPORT_SET cudf-exports - CONDITION ENABLE_PARQUET - ) + if(ENABLE_PARQUET) + rapids_export_package(BUILD Parquet cudf-exports) + rapids_export_package(BUILD ArrowDataset cudf-exports) + endif() + + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root( + BUILD Arrow [=[${CMAKE_CURRENT_LIST_DIR}]=] EXPORT_SET cudf-exports + ) + rapids_export_find_package_root( + BUILD Parquet [=[${CMAKE_CURRENT_LIST_DIR}]=] + EXPORT_SET cudf-exports + CONDITION ENABLE_PARQUET + ) + rapids_export_find_package_root( + BUILD ArrowDataset [=[${CMAKE_CURRENT_LIST_DIR}]=] + EXPORT_SET cudf-exports + CONDITION ENABLE_PARQUET + ) + endif() set(ARROW_LIBRARIES "${ARROW_LIBRARIES}" @@ -435,7 +352,21 @@ if(NOT DEFINED CUDF_VERSION_Arrow) ) endif() +# Default to static arrow builds +if(NOT DEFINED CUDF_USE_ARROW_STATIC) + set(CUDF_USE_ARROW_STATIC ON) +endif() + +# Default to excluding from installation since we generally privately and statically link Arrow. +if(NOT DEFINED CUDF_EXCLUDE_ARROW_FROM_ALL) + set(CUDF_EXCLUDE_ARROW_FROM_ALL OFF) +endif() + +if(NOT DEFINED CUDF_ENABLE_ARROW_PARQUET) + set(CUDF_ENABLE_ARROW_PARQUET OFF) +endif() + find_and_configure_arrow( - ${CUDF_VERSION_Arrow} ${CUDF_USE_ARROW_STATIC} ${CUDF_ENABLE_ARROW_S3} ${CUDF_ENABLE_ARROW_ORC} - ${CUDF_ENABLE_ARROW_PYTHON} ${CUDF_ENABLE_ARROW_PARQUET} ${USE_LIBARROW_FROM_PYARROW} + ${CUDF_VERSION_Arrow} ${CUDF_USE_ARROW_STATIC} ${CUDF_EXCLUDE_ARROW_FROM_ALL} + ${CUDF_ENABLE_ARROW_PARQUET} ) diff --git a/cpp/examples/interop/CMakeLists.txt b/cpp/examples/interop/CMakeLists.txt index a1f99c1d2fd..2816f613d3d 100644 --- a/cpp/examples/interop/CMakeLists.txt +++ b/cpp/examples/interop/CMakeLists.txt @@ -15,6 +15,13 @@ project( include(../fetch_dependencies.cmake) +# The Arrow CMake is currently broken if the build type is not set +set(CMAKE_BUILD_TYPE Release) +# No need to install Arrow libs when only the final example executable is shipped. +set(CUDF_EXCLUDE_ARROW_FROM_ALL ON) +include(../../cmake/thirdparty/get_arrow.cmake) + add_executable(interop interop.cpp) target_link_libraries(interop PRIVATE cudf::cudf) target_compile_features(interop PRIVATE cxx_std_17) +target_link_libraries(interop PRIVATE ${ARROW_LIBRARIES}) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index ac77a362e1c..f86acbcc51b 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -24,8 +24,8 @@ rapids_test_init() # properties and linking to build the test function(ConfigureTest CMAKE_TEST_NAME) set(options) - set(one_value GPUS PERCENT STREAM_MODE EXTRA_LIB) - set(multi_value) + set(one_value GPUS PERCENT STREAM_MODE) + set(multi_value EXTRA_LIBS) cmake_parse_arguments(_CUDF_TEST "${options}" "${one_value}" "${multi_value}" ${ARGN}) if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT) set(_CUDF_TEST_GPUS 1) @@ -57,7 +57,7 @@ function(ConfigureTest CMAKE_TEST_NAME) target_link_libraries( ${CMAKE_TEST_NAME} PRIVATE cudftestutil GTest::gmock GTest::gmock_main GTest::gtest GTest::gtest_main - nvtx3::nvtx3-cpp $ "${_CUDF_TEST_EXTRA_LIB}" + nvtx3::nvtx3-cpp $ "${_CUDF_TEST_EXTRA_LIBS}" ) rapids_cuda_set_runtime(${CMAKE_TEST_NAME} USE_STATIC ${CUDA_STATIC_RUNTIME}) rapids_test_add( @@ -78,6 +78,14 @@ function(ConfigureTest CMAKE_TEST_NAME) endif() endfunction() +# ################################################################################################## +# dependencies ################################################################################### +# ################################################################################################## + +# No need to install Arrow libs when only the final test executables are shipped. +set(CUDF_EXCLUDE_ARROW_FROM_ALL ON) +include(../cmake/thirdparty/get_arrow.cmake) + # ################################################################################################## # test sources ################################################################################## # ################################################################################################## @@ -197,7 +205,7 @@ ConfigureTest( QUANTILES_TEST quantiles/percentile_approx_test.cpp quantiles/quantile_test.cpp quantiles/quantiles_test.cpp GPUS 1 - PERCENT 70 + PERCENT 70 EXTRA_LIBS ${ARROW_LIBRARIES} ) # ################################################################################################## @@ -276,8 +284,9 @@ ConfigureTest( interop/from_arrow_host_test.cpp interop/from_arrow_stream_test.cpp interop/dlpack_test.cpp - EXTRA_LIB + EXTRA_LIBS nanoarrow + ${ARROW_LIBRARIES} ) # ################################################################################################## @@ -288,7 +297,7 @@ ConfigureTest(ROW_SELECTION_TEST io/row_selection_test.cpp) ConfigureTest( CSV_TEST io/csv_test.cpp GPUS 1 - PERCENT 30 + PERCENT 30 EXTRA_LIBS ${ARROW_LIBRARIES} ) ConfigureTest( FILE_IO_TEST io/file_io_test.cpp @@ -316,7 +325,7 @@ ConfigureTest( ConfigureTest( JSON_TEST io/json/json_test.cpp io/json/json_chunked_reader.cu GPUS 1 - PERCENT 30 + PERCENT 30 EXTRA_LIBS ${ARROW_LIBRARIES} ) ConfigureTest(JSON_WRITER_TEST io/json/json_writer.cpp) ConfigureTest(JSON_TYPE_CAST_TEST io/json/json_type_cast_test.cu) diff --git a/dependencies.yaml b/dependencies.yaml index 04b5940c9fb..b55860815bf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -17,7 +17,6 @@ files: - depends_on_rmm - develop - docs - - libarrow_build - notebooks - py_version - rapids_build_skbuild @@ -40,7 +39,6 @@ files: output: none includes: - cuda_version - - libarrow_run - test_cpp test_python: output: none @@ -58,7 +56,6 @@ files: - build_all - cuda - cuda_version - - libarrow_run - test_java test_notebooks: output: none @@ -77,7 +74,6 @@ files: - cuda - cuda_version - docs - - libarrow_run - py_version py_build_cudf: output: pyproject @@ -137,7 +133,6 @@ files: includes: - build_base - build_cpp - - build_python_libcudf - depends_on_librmm py_run_libcudf: output: pyproject @@ -389,38 +384,6 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cython>=3.0.3 - # Hard pin the patch version used during the build. This must be kept - # in sync with the version pinned in get_arrow.cmake. - - &pyarrow_build pyarrow==16.1.0.* - - output_types: pyproject - packages: - # Hard pin the version used during the build. - # Sync with conda build constraint & wheel run constraint. - - numpy==2.0.* - build_python_libcudf: - common: - - output_types: [conda, requirements, pyproject] - packages: - - *pyarrow_build - libarrow_build: - common: - - output_types: conda - packages: - # Hard pin the Arrow patch version used during the build. This must - # be kept in sync with the version pinned in get_arrow.cmake. - - libarrow-acero==16.1.0.* - - libarrow-dataset==16.1.0.* - - libarrow==16.1.0.* - - libparquet==16.1.0.* - libarrow_run: - common: - - output_types: conda - packages: - # Allow runtime version to float up to patch version - - libarrow-acero>=16.1.0,<16.2.0a0 - - libarrow-dataset>=16.1.0,<16.2.0a0 - - libarrow>=16.1.0,<16.2.0a0 - - libparquet>=16.1.0,<16.2.0a0 pyarrow_run: common: - output_types: [conda, requirements, pyproject] @@ -600,7 +563,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - fsspec>=0.6.0 - - numpy>=1.23,<3.0a0 + - &numpy numpy>=1.23,<3.0a0 - pandas>=2.0,<2.2.3dev0 run_pylibcudf: common: @@ -731,6 +694,7 @@ dependencies: - *cmake_ver - maven - openjdk=8.* + - boost test_python_common: common: - output_types: [conda, requirements, pyproject] @@ -744,7 +708,7 @@ dependencies: packages: - fastavro>=0.22.9 - hypothesis - - numpy + - *numpy - pandas test_python_cudf: common: diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 22059c5bc7f..c18a90140b6 100644 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -212,6 +212,10 @@ target_compile_definitions( ) target_link_options(cudfjni PRIVATE "-Wl,--no-undefined") +set(CUDF_ENABLE_ARROW_PARQUET ON) +include(../../../../cpp/cmake/thirdparty/get_arrow.cmake) +target_link_libraries(cudfjni PRIVATE ${ARROW_LIBRARIES}) + if(USE_GDS) add_library(cufilejni src/CuFileJni.cpp) set_target_properties( diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 72f20b30052..7193ada5b93 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -35,7 +35,6 @@ include(../../cpp/cmake/thirdparty/get_dlpack.cmake) include(rapids-cython-core) rapids_cython_init() -include(../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) add_subdirectory(cudf/_lib) add_subdirectory(udf_cpp) diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 5ea378fc0e5..5d4b5421f16 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -65,9 +65,6 @@ rapids_cython_create_modules( target_link_libraries(strings_udf PUBLIC cudf_strings_udf) target_include_directories(interop PUBLIC "$") -set(targets_using_arrow_headers avro csv orc json parquet) -link_to_pyarrow_headers("${targets_using_arrow_headers}") - include(${rapids-cmake-dir}/export/find_package_root.cmake) include(../../../../cpp/cmake/thirdparty/get_nanoarrow.cmake) target_link_libraries(interop PUBLIC nanoarrow) diff --git a/python/cudf/cudf/_lib/io/CMakeLists.txt b/python/cudf/cudf/_lib/io/CMakeLists.txt index 620229a1275..e7408cf2852 100644 --- a/python/cudf/cudf/_lib/io/CMakeLists.txt +++ b/python/cudf/cudf/_lib/io/CMakeLists.txt @@ -19,5 +19,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX io_ ASSOCIATED_TARGETS cudf ) - -link_to_pyarrow_headers("${RAPIDS_CYTHON_CREATED_TARGETS}") diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index a6d26d17d46..8386935fab0 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -129,8 +129,6 @@ requires = [ "libcudf==24.10.*,>=0.0.0a0", "librmm==24.10.*,>=0.0.0a0", "ninja", - "numpy==2.0.*", - "pyarrow==16.1.0.*", "pylibcudf==24.10.*,>=0.0.0a0", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt b/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt index 1b205537d73..4490c41c7a9 100644 --- a/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt +++ b/python/cudf_kafka/cudf_kafka/_lib/CMakeLists.txt @@ -20,5 +20,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ) -include(../../../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) -link_to_pyarrow_headers("${RAPIDS_CYTHON_CREATED_TARGETS}") diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 01e7299a33a..6ca798bb11c 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -106,6 +106,4 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", "ninja", - "numpy==2.0.*", - "pyarrow==16.1.0.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 09c7ed2e217..96eb6c3bb30 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -32,9 +32,6 @@ endif() unset(cudf_FOUND) -# For wheels, this should always be true -set(USE_LIBARROW_FROM_PYARROW ON) - # Find Python early so that later commands can use it find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) @@ -46,13 +43,11 @@ set(CUDA_STATIC_RUNTIME ON) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) -include(../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) - add_subdirectory(../../cpp cudf-cpp) # Ensure other libraries needed by libcudf.so get installed alongside it. include(cmake/Modules/WheelHelpers.cmake) install_aliased_imported_targets( - TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY} + TARGETS cudf nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION + ${CMAKE_LIBRARY_OUTPUT_DIRECTORY} ) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index f6ba0d51bdb..ba134710868 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -18,10 +18,6 @@ def load_library(): - # This is loading the libarrow shared library in situations where it comes from the - # pyarrow package (i.e. when installed as a wheel). - import pyarrow # noqa: F401 - # Dynamically load libcudf.so. Prefer a system library if one is present to # avoid clobbering symbols that other packages might expect, but if no # other library is present use the one in the wheel. diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index fd01f7f6e2f..43878d0aec2 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -71,5 +71,4 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "librmm==24.10.*,>=0.0.0a0", "ninja", - "pyarrow==16.1.0.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcudf/CMakeLists.txt b/python/pylibcudf/CMakeLists.txt index 340ad120377..a4b831790fb 100644 --- a/python/pylibcudf/CMakeLists.txt +++ b/python/pylibcudf/CMakeLists.txt @@ -36,7 +36,6 @@ include(rapids-cython-core) rapids_cython_init() -include(cmake/Modules/LinkPyarrowHeaders.cmake) add_subdirectory(pylibcudf) if(DEFINED cython_lib_dir) diff --git a/python/pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake b/python/pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake deleted file mode 100644 index d432f9fe1f5..00000000000 --- a/python/pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= -include_guard(GLOBAL) - -find_package(Python REQUIRED COMPONENTS Development NumPy) - -execute_process( - COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_include())" - OUTPUT_VARIABLE PYARROW_INCLUDE_DIR - ERROR_VARIABLE PYARROW_ERROR - RESULT_VARIABLE PYARROW_RESULT - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -if(${PYARROW_RESULT}) - message(FATAL_ERROR "Error while trying to obtain pyarrow include directory:\n${PYARROW_ERROR}") -endif() - -# Due to cudf's scalar.pyx needing to cimport pylibcudf's scalar.pyx (because there are parts of -# cudf Cython that need to directly access the c_obj underlying the pylibcudf Scalar) the -# requirement for arrow headers infects all of cudf. These requirements will go away once all -# scalar-related Cython code is removed from cudf. -function(link_to_pyarrow_headers targets) - foreach(target IN LISTS targets) - # PyArrow headers require numpy headers. - target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}") - target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}") - endforeach() -endfunction() diff --git a/python/pylibcudf/pylibcudf/io/CMakeLists.txt b/python/pylibcudf/pylibcudf/io/CMakeLists.txt index 55bea4fc262..bcc2151f5b6 100644 --- a/python/pylibcudf/pylibcudf/io/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/io/CMakeLists.txt @@ -20,8 +20,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX pylibcudf_io_ ASSOCIATED_TARGETS cudf ) - -set(targets_using_arrow_headers pylibcudf_io_avro pylibcudf_io_csv pylibcudf_io_datasource - pylibcudf_io_json pylibcudf_io_parquet pylibcudf_io_types -) -link_to_pyarrow_headers("${targets_using_arrow_headers}") diff --git a/python/pylibcudf/pylibcudf/libcudf/io/CMakeLists.txt b/python/pylibcudf/pylibcudf/libcudf/io/CMakeLists.txt index 6831063ecb9..9f5f74506e9 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/libcudf/io/CMakeLists.txt @@ -21,6 +21,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cudf MODULE_PREFIX cpp_io_ ) - -set(targets_using_arrow_headers cpp_io_json cpp_io_types) -link_to_pyarrow_headers("${targets_using_arrow_headers}") diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index 0d673ea4cc3..e4c6edc6141 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -40,7 +40,7 @@ classifiers = [ test = [ "fastavro>=0.22.9", "hypothesis", - "numpy", + "numpy>=1.23,<3.0a0", "pandas", "pytest-cov", "pytest-xdist", @@ -104,8 +104,6 @@ requires = [ "libcudf==24.10.*,>=0.0.0a0", "librmm==24.10.*,>=0.0.0a0", "ninja", - "numpy==2.0.*", - "pyarrow==16.1.0.*", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From d0e5cdfc4df197bfb4846a243e3d9ea9d7b87aab Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:17:51 -1000 Subject: [PATCH 130/270] Allow for binops between two differently sized DecimalDtypes (#16638) Currently cudf Python has some custom logic for determining the resulting dtype of a binop between 2 decimal dtypes since Python decimal dtype support `precision` and libcudf doesn't. But libcudf does require that the 2 operands have the same decimal type when calculating the binop, so we must ensure the inputs are cast to the same, resulting dtype. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16638 --- python/cudf/cudf/core/column/decimal.py | 17 ++++++++++++++--- python/cudf/cudf/tests/test_decimal.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/column/decimal.py b/python/cudf/cudf/core/column/decimal.py index 3b979ef2e97..8803ebd6791 100644 --- a/python/cudf/cudf/core/column/decimal.py +++ b/python/cudf/cudf/core/column/decimal.py @@ -135,9 +135,15 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str): # are computed outside of libcudf if op in {"__add__", "__sub__", "__mul__", "__div__"}: output_type = _get_decimal_type(lhs.dtype, rhs.dtype, op) + lhs = lhs.astype( + type(output_type)(lhs.dtype.precision, lhs.dtype.scale) + ) + rhs = rhs.astype( + type(output_type)(rhs.dtype.precision, rhs.dtype.scale) + ) result = libcudf.binaryop.binaryop(lhs, rhs, op, output_type) - # TODO: Why is this necessary? Why isn't the result's - # precision already set correctly based on output_type? + # libcudf doesn't support precision, so result.dtype doesn't + # maintain output_type.precision result.dtype.precision = output_type.precision elif op in { "__eq__", @@ -430,7 +436,11 @@ def _with_type_metadata( return self -def _get_decimal_type(lhs_dtype, rhs_dtype, op): +def _get_decimal_type( + lhs_dtype: DecimalDtype, + rhs_dtype: DecimalDtype, + op: str, +) -> DecimalDtype: """ Returns the resulting decimal type after calculating precision & scale when performing the binary operation @@ -441,6 +451,7 @@ def _get_decimal_type(lhs_dtype, rhs_dtype, op): # This should at some point be hooked up to libcudf's # binary_operation_fixed_point_scale + # Note: libcudf decimal types don't have a concept of precision p1, p2 = lhs_dtype.precision, rhs_dtype.precision s1, s2 = lhs_dtype.scale, rhs_dtype.scale diff --git a/python/cudf/cudf/tests/test_decimal.py b/python/cudf/cudf/tests/test_decimal.py index b63788d20b7..048b3a656e3 100644 --- a/python/cudf/cudf/tests/test_decimal.py +++ b/python/cudf/cudf/tests/test_decimal.py @@ -398,3 +398,13 @@ def test_decimal_overflow(): s = cudf.Series([1, 2], dtype=cudf.Decimal128Dtype(precision=38, scale=0)) result = s * Decimal("1.0") assert_eq(cudf.Decimal128Dtype(precision=38, scale=1), result.dtype) + + +def test_decimal_binop_upcast_operands(): + ser1 = cudf.Series([0.51, 1.51, 2.51]).astype(cudf.Decimal64Dtype(18, 2)) + ser2 = cudf.Series([0.90, 0.96, 0.99]).astype(cudf.Decimal128Dtype(19, 2)) + result = ser1 + ser2 + expected = cudf.Series([1.41, 2.47, 3.50]).astype( + cudf.Decimal128Dtype(20, 2) + ) + assert_eq(result, expected) From 88de8dd5bc0d2476a554107626d72ceb6d65cbab Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:26:27 -1000 Subject: [PATCH 131/270] Fix interval_range right child non-zero offset (#16651) xref https://github.com/rapidsai/cudf/issues/16507 Similar to what is done in `IntervalIndex.from_breaks`, `interval_index` generates the right edges by slicing a range of fencepost edges. However, we don't want to maintain the new `offset` (`1`) on the right edge after slicing as it adversely impacts subsequent indexing operations. ~~Additionally, I noticed that `Index(struct_data)` would automatically convert it to an `IntervalIndex`, but `IntervalIndex` has a strict requirement on the data have `left/right` keys, so making this raise a `NotImplementedError` instead~~ ^ Will tackle this in a follow up, looks like there are cases where this is valid Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16651 --- python/cudf/cudf/core/index.py | 12 +++++++++++- python/cudf/cudf/tests/indexes/test_interval.py | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 6a5e718c2c5..df8af856f4f 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3250,7 +3250,7 @@ def interval_range( freq=None, name=None, closed="right", -) -> "IntervalIndex": +) -> IntervalIndex: """ Returns a fixed frequency IntervalIndex. @@ -3347,6 +3347,16 @@ def interval_range( ) left_col = bin_edges.slice(0, len(bin_edges) - 1) right_col = bin_edges.slice(1, len(bin_edges)) + # For indexing, children should both have 0 offset + right_col = type(right_col)( + data=right_col.data, + dtype=right_col.dtype, + size=right_col.size, + mask=right_col.mask, + offset=0, + null_count=right_col.null_count, + children=right_col.children, + ) if len(right_col) == 0 or len(left_col) == 0: dtype = IntervalDtype("int64", closed) diff --git a/python/cudf/cudf/tests/indexes/test_interval.py b/python/cudf/cudf/tests/indexes/test_interval.py index a567c27f584..6653a94c9be 100644 --- a/python/cudf/cudf/tests/indexes/test_interval.py +++ b/python/cudf/cudf/tests/indexes/test_interval.py @@ -407,3 +407,9 @@ def test_interval_range_name(): expected = pd.interval_range(start=0, periods=5, freq=2, name="foo") result = cudf.interval_range(start=0, periods=5, freq=2, name="foo") assert_eq(result, expected) + + +def test_from_interval_range_indexing(): + result = cudf.interval_range(start=0, end=1, name="a").repeat(2) + expected = pd.interval_range(start=0, end=1, name="a").repeat(2) + assert_eq(result, expected) From e2a15cb1ba856616b7de08e2f1a5c06d6d7c4a35 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:47:31 -0400 Subject: [PATCH 132/270] Fix strings::detail::copy_range when target contains nulls (#16626) Fixes the logic in `cudf::strings::detail::copy_range` handling of nulls in the target range. The optimization check for nulls is removed simplifying the logic and making it more reliable as well. The benchmark showed no significant change in performance. Also adds a specific gtest for this case. Error was introduced in #15010 Closes #16618 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16626 --- cpp/src/strings/copying/copy_range.cu | 23 +++-------------------- cpp/tests/copying/copy_range_tests.cpp | 10 ++++++++++ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/cpp/src/strings/copying/copy_range.cu b/cpp/src/strings/copying/copy_range.cu index 9f8c47602f8..2434de1795e 100644 --- a/cpp/src/strings/copying/copy_range.cu +++ b/cpp/src/strings/copying/copy_range.cu @@ -40,20 +40,14 @@ struct compute_element_size { size_type source_begin; size_type target_begin; size_type target_end; - bool source_has_nulls; - bool target_has_nulls; __device__ cudf::size_type operator()(cudf::size_type idx) { if (idx >= target_begin && idx < target_end) { auto const str_idx = source_begin + (idx - target_begin); - return source_has_nulls && d_source.is_null_nocheck(str_idx) - ? 0 - : d_source.element(str_idx).size_bytes(); + return d_source.is_null(str_idx) ? 0 : d_source.element(str_idx).size_bytes(); } else { - return target_has_nulls && d_target.is_null_nocheck(idx) - ? 0 - : d_target.element(idx).size_bytes(); + return d_target.is_null(idx) ? 0 : d_target.element(idx).size_bytes(); } } }; @@ -97,20 +91,9 @@ std::unique_ptr copy_range(strings_column_view const& source, mr); }(); - auto [check_source, check_target] = [target, null_count = null_count] { - // check validities for both source & target - if (target.has_nulls()) { return std::make_pair(true, true); } - // check validities for source only - if (null_count > 0) { return std::make_pair(true, false); } - // no need to check validities - return std::make_pair(false, false); - }(); - // create offsets auto sizes_begin = cudf::detail::make_counting_transform_iterator( - 0, - compute_element_size{ - d_source, d_target, source_begin, target_begin, target_end, check_source, check_target}); + 0, compute_element_size{d_source, d_target, source_begin, target_begin, target_end}); auto [offsets_column, chars_bytes] = cudf::strings::detail::make_offsets_child_column( sizes_begin, sizes_begin + target.size(), stream, mr); auto d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_column->view()); diff --git a/cpp/tests/copying/copy_range_tests.cpp b/cpp/tests/copying/copy_range_tests.cpp index 223946ddcee..25d93da277b 100644 --- a/cpp/tests/copying/copy_range_tests.cpp +++ b/cpp/tests/copying/copy_range_tests.cpp @@ -232,6 +232,16 @@ TEST_F(CopyRangeTestFixture, CopyWithNullsString) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*p_ret, expected); } +TEST_F(CopyRangeTestFixture, CopyWithTargetNullsString) +{ + auto target = + cudf::test::strings_column_wrapper({"a", "b", "", "d", "", "é"}, {1, 1, 0, 1, 1, 1}); + auto source = cudf::test::strings_column_wrapper({"A", "B", "C", "D", "E", "F"}); + auto result = cudf::copy_range(source, target, 1, 5, 1); + auto expected = cudf::test::strings_column_wrapper({"a", "B", "C", "D", "E", "é"}); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(result->view(), expected); +} + TEST_F(CopyRangeTestFixture, CopyNoNullsString) { cudf::size_type size{100}; From d1412e00092d752e4e34371042d7dbfe972ba5d7 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:52:57 -0400 Subject: [PATCH 133/270] Rework strings::slice benchmark to use nvbench (#16563) Moves google-benchmark for `cudf::strings::slice_strings` to nvbench. This is to help measure performance improvements in follow on work for strings-slice. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16563 --- cpp/benchmarks/CMakeLists.txt | 2 +- cpp/benchmarks/string/slice.cpp | 89 ++++++++++++++++----------------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 6db282a7728..7f3edfa0a01 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -325,7 +325,6 @@ ConfigureBench( string/filter.cpp string/repeat_strings.cpp string/replace.cpp - string/slice.cpp string/translate.cpp string/url_decode.cu ) @@ -346,6 +345,7 @@ ConfigureNVBench( string/like.cpp string/replace_re.cpp string/reverse.cpp + string/slice.cpp string/split.cpp string/split_re.cpp ) diff --git a/cpp/benchmarks/string/slice.cpp b/cpp/benchmarks/string/slice.cpp index 0f973a7c8b5..1898f0340b6 100644 --- a/cpp/benchmarks/string/slice.cpp +++ b/cpp/benchmarks/string/slice.cpp @@ -14,11 +14,8 @@ * limitations under the License. */ -#include "string_bench_args.hpp" - #include -#include -#include +#include #include @@ -29,56 +26,56 @@ #include +#include + #include -class StringSlice : public cudf::benchmark {}; +static void bench_slice(nvbench::state& state) +{ + auto const num_rows = static_cast(state.get_int64("num_rows")); + auto const row_width = static_cast(state.get_int64("row_width")); + auto const stype = state.get_string("type"); -enum slice_type { position, multi_position }; + if (static_cast(num_rows) * static_cast(row_width) >= + static_cast(std::numeric_limits::max())) { + state.skip("Skip benchmarks greater than size_type limit"); + } -static void BM_slice(benchmark::State& state, slice_type rt) -{ - cudf::size_type const n_rows{static_cast(state.range(0))}; - cudf::size_type const max_str_length{static_cast(state.range(1))}; data_profile const profile = data_profile_builder().distribution( - cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const column = create_random_column(cudf::type_id::STRING, row_count{n_rows}, profile); + cudf::type_id::STRING, distribution_id::NORMAL, 0, row_width); + auto const column = create_random_column(cudf::type_id::STRING, row_count{num_rows}, profile); cudf::strings_column_view input(column->view()); - auto starts_itr = thrust::constant_iterator(max_str_length / 3); - auto stops_itr = thrust::constant_iterator(max_str_length / 2); - cudf::test::fixed_width_column_wrapper starts(starts_itr, starts_itr + n_rows); - cudf::test::fixed_width_column_wrapper stops(stops_itr, stops_itr + n_rows); + auto starts_itr = thrust::constant_iterator(row_width / 4); + auto starts = + cudf::test::fixed_width_column_wrapper(starts_itr, starts_itr + num_rows); + auto stops_itr = thrust::constant_iterator(row_width / 3); + auto stops = + cudf::test::fixed_width_column_wrapper(stops_itr, stops_itr + num_rows); - for (auto _ : state) { - cuda_event_timer raii(state, true, cudf::get_default_stream()); - switch (rt) { - case position: - cudf::strings::slice_strings(input, max_str_length / 3, max_str_length / 2); - break; - case multi_position: cudf::strings::slice_strings(input, starts, stops); break; - } + auto stream = cudf::get_default_stream(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); + // gather some throughput statistics as well + auto chars_size = input.chars_size(stream); + state.add_element_count(chars_size, "chars_size"); // number of bytes + state.add_global_memory_reads(chars_size); // all bytes are read + auto output_size = (row_width / 3 - row_width / 4) * num_rows; + state.add_global_memory_writes(output_size); + + if (stype == "multi") { + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { + cudf::strings::slice_strings(input, starts, stops, stream); + }); + } else { + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { + cudf::strings::slice_strings(input, row_width / 4, row_width / 3, 1, stream); + }); } - state.SetBytesProcessed(state.iterations() * input.chars_size(cudf::get_default_stream())); + set_throughputs(state); } -static void generate_bench_args(benchmark::internal::Benchmark* b) -{ - int const min_rows = 1 << 12; - int const max_rows = 1 << 24; - int const row_mult = 8; - int const min_rowlen = 1 << 5; - int const max_rowlen = 1 << 13; - int const len_mult = 2; - generate_string_bench_args(b, min_rows, max_rows, row_mult, min_rowlen, max_rowlen, len_mult); -} - -#define STRINGS_BENCHMARK_DEFINE(name) \ - BENCHMARK_DEFINE_F(StringSlice, name) \ - (::benchmark::State & st) { BM_slice(st, slice_type::name); } \ - BENCHMARK_REGISTER_F(StringSlice, name) \ - ->Apply(generate_bench_args) \ - ->UseManualTime() \ - ->Unit(benchmark::kMillisecond); - -STRINGS_BENCHMARK_DEFINE(position) -STRINGS_BENCHMARK_DEFINE(multi_position) +NVBENCH_BENCH(bench_slice) + .set_name("slice") + .add_int64_axis("row_width", {32, 64, 128, 256, 512, 1024, 2048}) + .add_int64_axis("num_rows", {262144, 2097152, 16777216}) + .add_string_axis("type", {"position", "multi"}); From 60f30d831325d5816e6968e8037796b8ce1dc579 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 27 Aug 2024 17:45:33 -0700 Subject: [PATCH 134/270] Use `make_host_vector` instead of `make_std_vector` to facilitate pinned memory optimizations (#16386) Replaced most of `make_std_vector` calls with `make_host_vector` to allow pinned memory and kernel copies, when enabled. Skipped places where the change would impact the public API. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Nghia Truong (https://github.com/ttnghia) - Shruti Shivakumar (https://github.com/shrshi) URL: https://github.com/rapidsai/cudf/pull/16386 --- cpp/include/cudf/detail/gather.cuh | 2 +- cpp/src/io/csv/csv_gpu.cu | 4 +-- cpp/src/io/csv/csv_gpu.hpp | 2 +- cpp/src/io/csv/reader_impl.cu | 2 +- cpp/src/io/json/json_column.cu | 42 +++++++++++----------- cpp/src/io/orc/writer_impl.cu | 8 ++--- cpp/src/io/orc/writer_impl.hpp | 5 +-- cpp/src/io/parquet/predicate_pushdown.cpp | 2 +- cpp/src/io/parquet/reader_impl_chunking.cu | 20 +++++------ cpp/src/io/parquet/writer_impl.cu | 22 ++++++------ cpp/src/io/utilities/datasource.cpp | 4 +-- cpp/src/text/jaccard.cu | 2 +- 12 files changed, 57 insertions(+), 58 deletions(-) diff --git a/cpp/include/cudf/detail/gather.cuh b/cpp/include/cudf/detail/gather.cuh index 41f5494f78f..df6fe6e6ccb 100644 --- a/cpp/include/cudf/detail/gather.cuh +++ b/cpp/include/cudf/detail/gather.cuh @@ -609,7 +609,7 @@ void gather_bitmask(table_view const& source, stream); // Copy the valid counts into each column - auto const valid_counts = make_std_vector_sync(d_valid_counts, stream); + auto const valid_counts = make_host_vector_sync(d_valid_counts, stream); for (size_t i = 0; i < target.size(); ++i) { if (target[i]->nullable()) { auto const null_count = target_rows - valid_counts[i]; diff --git a/cpp/src/io/csv/csv_gpu.cu b/cpp/src/io/csv/csv_gpu.cu index 7a05d0aebaf..5a0c6decfda 100644 --- a/cpp/src/io/csv/csv_gpu.cu +++ b/cpp/src/io/csv/csv_gpu.cu @@ -794,7 +794,7 @@ device_span __host__ remove_blank_rows(cudf::io::parse_options_view co return row_offsets.subspan(0, new_end - row_offsets.begin()); } -std::vector detect_column_types( +cudf::detail::host_vector detect_column_types( cudf::io::parse_options_view const& options, device_span const data, device_span const column_flags, @@ -812,7 +812,7 @@ std::vector detect_column_types( data_type_detection<<>>( options, data, column_flags, row_starts, d_stats); - return detail::make_std_vector_sync(d_stats, stream); + return detail::make_host_vector_sync(d_stats, stream); } void decode_row_column_data(cudf::io::parse_options_view const& options, diff --git a/cpp/src/io/csv/csv_gpu.hpp b/cpp/src/io/csv/csv_gpu.hpp index 06c60319371..aa3d9f6c7b7 100644 --- a/cpp/src/io/csv/csv_gpu.hpp +++ b/cpp/src/io/csv/csv_gpu.hpp @@ -199,7 +199,7 @@ device_span remove_blank_rows(cudf::io::parse_options_view const& opti * * @return stats Histogram of each dtypes' occurrence for each column */ -std::vector detect_column_types( +cudf::detail::host_vector detect_column_types( cudf::io::parse_options_view const& options, device_span data, device_span column_flags, diff --git a/cpp/src/io/csv/reader_impl.cu b/cpp/src/io/csv/reader_impl.cu index 40d4372ae9d..e27b06682bb 100644 --- a/cpp/src/io/csv/reader_impl.cu +++ b/cpp/src/io/csv/reader_impl.cu @@ -614,7 +614,7 @@ std::vector decode_data(parse_options const& parse_opts, d_valid_counts, stream); - auto const h_valid_counts = cudf::detail::make_std_vector_sync(d_valid_counts, stream); + auto const h_valid_counts = cudf::detail::make_host_vector_sync(d_valid_counts, stream); for (int i = 0; i < num_active_columns; ++i) { out_buffers[i].null_count() = num_records - h_valid_counts[i]; } diff --git a/cpp/src/io/json/json_column.cu b/cpp/src/io/json/json_column.cu index e5e21e054a6..8d6890045be 100644 --- a/cpp/src/io/json/json_column.cu +++ b/cpp/src/io/json/json_column.cu @@ -77,16 +77,16 @@ void print_tree(host_span input, tree_meta_t const& d_gpu_tree, rmm::cuda_stream_view stream) { - print_vec(cudf::detail::make_std_vector_sync(d_gpu_tree.node_categories, stream), + print_vec(cudf::detail::make_host_vector_sync(d_gpu_tree.node_categories, stream), "node_categories", to_cat); - print_vec(cudf::detail::make_std_vector_sync(d_gpu_tree.parent_node_ids, stream), + print_vec(cudf::detail::make_host_vector_sync(d_gpu_tree.parent_node_ids, stream), "parent_node_ids", to_int); print_vec( - cudf::detail::make_std_vector_sync(d_gpu_tree.node_levels, stream), "node_levels", to_int); - auto node_range_begin = cudf::detail::make_std_vector_sync(d_gpu_tree.node_range_begin, stream); - auto node_range_end = cudf::detail::make_std_vector_sync(d_gpu_tree.node_range_end, stream); + cudf::detail::make_host_vector_sync(d_gpu_tree.node_levels, stream), "node_levels", to_int); + auto node_range_begin = cudf::detail::make_host_vector_sync(d_gpu_tree.node_range_begin, stream); + auto node_range_end = cudf::detail::make_host_vector_sync(d_gpu_tree.node_range_end, stream); print_vec(node_range_begin, "node_range_begin", to_int); print_vec(node_range_end, "node_range_end", to_int); for (int i = 0; i < int(node_range_begin.size()); i++) { @@ -373,9 +373,9 @@ std::vector copy_strings_to_host_sync( auto to_host = [stream](auto const& col) { if (col.is_empty()) return std::vector{}; auto const scv = cudf::strings_column_view(col); - auto const h_chars = cudf::detail::make_std_vector_async( + auto const h_chars = cudf::detail::make_host_vector_async( cudf::device_span(scv.chars_begin(stream), scv.chars_size(stream)), stream); - auto const h_offsets = cudf::detail::make_std_vector_async( + auto const h_offsets = cudf::detail::make_host_vector_async( cudf::device_span(scv.offsets().data() + scv.offset(), scv.size() + 1), stream); @@ -523,25 +523,23 @@ void make_device_json_column(device_span input, row_array_parent_col_id, stream); auto num_columns = d_unique_col_ids.size(); - auto unique_col_ids = cudf::detail::make_std_vector_async(d_unique_col_ids, stream); + auto unique_col_ids = cudf::detail::make_host_vector_async(d_unique_col_ids, stream); auto column_categories = - cudf::detail::make_std_vector_async(d_column_tree.node_categories, stream); - auto column_parent_ids = - cudf::detail::make_std_vector_async(d_column_tree.parent_node_ids, stream); + cudf::detail::make_host_vector_async(d_column_tree.node_categories, stream); + auto const column_parent_ids = + cudf::detail::make_host_vector_async(d_column_tree.parent_node_ids, stream); auto column_range_beg = - cudf::detail::make_std_vector_async(d_column_tree.node_range_begin, stream); - auto max_row_offsets = cudf::detail::make_std_vector_async(d_max_row_offsets, stream); + cudf::detail::make_host_vector_async(d_column_tree.node_range_begin, stream); + auto const max_row_offsets = cudf::detail::make_host_vector_async(d_max_row_offsets, stream); std::vector column_names = copy_strings_to_host_sync( input, d_column_tree.node_range_begin, d_column_tree.node_range_end, stream); - stream.synchronize(); // array of arrays column names if (is_array_of_arrays) { TreeDepthT const row_array_children_level = is_enabled_lines ? 1 : 2; auto values_column_indices = get_values_column_indices(row_array_children_level, tree, col_ids, num_columns, stream); auto h_values_column_indices = - cudf::detail::make_std_vector_async(values_column_indices, stream); - stream.synchronize(); + cudf::detail::make_host_vector_sync(values_column_indices, stream); std::transform(unique_col_ids.begin(), unique_col_ids.end(), column_names.begin(), @@ -611,11 +609,13 @@ void make_device_json_column(device_span input, return thrust::get<0>(a) < thrust::get<0>(b); }); - std::vector is_str_column_all_nulls{}; - if (is_enabled_mixed_types_as_string) { - is_str_column_all_nulls = cudf::detail::make_std_vector_sync( - is_all_nulls_each_column(input, d_column_tree, tree, col_ids, options, stream), stream); - } + auto const is_str_column_all_nulls = [&, &column_tree = d_column_tree]() { + if (is_enabled_mixed_types_as_string) { + return cudf::detail::make_host_vector_sync( + is_all_nulls_each_column(input, column_tree, tree, col_ids, options, stream), stream); + } + return cudf::detail::make_empty_host_vector(0, stream); + }(); // use hash map because we may skip field name's col_ids std::unordered_map> columns; diff --git a/cpp/src/io/orc/writer_impl.cu b/cpp/src/io/orc/writer_impl.cu index 04eee68e757..ede9fd060b8 100644 --- a/cpp/src/io/orc/writer_impl.cu +++ b/cpp/src/io/orc/writer_impl.cu @@ -1978,7 +1978,7 @@ encoder_decimal_info decimal_chunk_sizes(orc_table_view& orc_table, // Gather the row group sizes and copy to host auto d_tmp_rowgroup_sizes = rmm::device_uvector(segmentation.num_rowgroups(), stream); - std::map> rg_sizes; + std::map> rg_sizes; for (auto const& [col_idx, esizes] : elem_sizes) { // Copy last elem in each row group - equal to row group size thrust::tabulate(rmm::exec_policy(stream), @@ -1991,14 +1991,14 @@ encoder_decimal_info decimal_chunk_sizes(orc_table_view& orc_table, return src[rg_bounds[idx][col_idx].end - 1]; }); - rg_sizes[col_idx] = cudf::detail::make_std_vector_async(d_tmp_rowgroup_sizes, stream); + rg_sizes.emplace(col_idx, cudf::detail::make_host_vector_async(d_tmp_rowgroup_sizes, stream)); } return {std::move(elem_sizes), std::move(rg_sizes)}; } std::map decimal_column_sizes( - std::map> const& chunk_sizes) + std::map> const& chunk_sizes) { std::map column_sizes; std::transform(chunk_sizes.cbegin(), @@ -2056,7 +2056,7 @@ auto set_rowgroup_char_counts(orc_table_view& orc_table, orc_table.d_string_column_indices, stream); - auto const h_counts = cudf::detail::make_std_vector_sync(counts, stream); + auto const h_counts = cudf::detail::make_host_vector_sync(counts, stream); for (auto col_idx : orc_table.string_column_indices) { auto& str_column = orc_table.column(col_idx); diff --git a/cpp/src/io/orc/writer_impl.hpp b/cpp/src/io/orc/writer_impl.hpp index f5f8b3cfed9..cae849ee315 100644 --- a/cpp/src/io/orc/writer_impl.hpp +++ b/cpp/src/io/orc/writer_impl.hpp @@ -90,8 +90,9 @@ struct stripe_rowgroups { */ struct encoder_decimal_info { std::map> - elem_sizes; ///< Column index -> per-element size map - std::map> rg_sizes; ///< Column index -> per-rowgroup size map + elem_sizes; ///< Column index -> per-element size map + std::map> + rg_sizes; ///< Column index -> per-rowgroup size map }; /** diff --git a/cpp/src/io/parquet/predicate_pushdown.cpp b/cpp/src/io/parquet/predicate_pushdown.cpp index 5ca090b05b3..c8b8b7a1193 100644 --- a/cpp/src/io/parquet/predicate_pushdown.cpp +++ b/cpp/src/io/parquet/predicate_pushdown.cpp @@ -468,7 +468,7 @@ std::optional>> aggregate_reader_metadata::fi auto validity_it = cudf::detail::make_counting_transform_iterator( 0, [bitmask = host_bitmask.data()](auto bit_index) { return bit_is_set(bitmask, bit_index); }); - auto is_row_group_required = cudf::detail::make_std_vector_sync( + auto const is_row_group_required = cudf::detail::make_host_vector_sync( device_span(predicate.data(), predicate.size()), stream); // Return only filtered row groups based on predicate diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index 54ba898b058..00d62c45962 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -77,9 +77,9 @@ void print_cumulative_page_info(device_span d_pages, device_span d_c_info, rmm::cuda_stream_view stream) { - std::vector pages = cudf::detail::make_std_vector_sync(d_pages, stream); - std::vector chunks = cudf::detail::make_std_vector_sync(d_chunks, stream); - std::vector c_info = cudf::detail::make_std_vector_sync(d_c_info, stream); + auto const pages = cudf::detail::make_host_vector_sync(d_pages, stream); + auto const chunks = cudf::detail::make_host_vector_sync(d_chunks, stream); + auto const c_info = cudf::detail::make_host_vector_sync(d_c_info, stream); printf("------------\nCumulative sizes by page\n"); @@ -647,7 +647,7 @@ std::tuple, size_t, size_t> compute_next_subpass( auto [aggregated_info, page_keys_by_split] = adjust_cumulative_sizes(c_info, pages, stream); // bring back to the cpu - auto const h_aggregated_info = cudf::detail::make_std_vector_sync(aggregated_info, stream); + auto const h_aggregated_info = cudf::detail::make_host_vector_sync(aggregated_info, stream); // print_cumulative_row_info(h_aggregated_info, "adjusted"); // TODO: if the user has explicitly specified skip_rows/num_rows we could be more intelligent @@ -694,8 +694,7 @@ std::vector compute_page_splits_by_row(device_span h_aggregated_info = - cudf::detail::make_std_vector_sync(aggregated_info, stream); + auto const h_aggregated_info = cudf::detail::make_host_vector_sync(aggregated_info, stream); // print_cumulative_row_info(h_aggregated_info, "adjusted"); std::vector splits; @@ -1304,9 +1303,8 @@ void reader::impl::setup_next_pass(read_mode mode) printf("\tskip_rows: %'lu\n", pass.skip_rows); printf("\tnum_rows: %'lu\n", pass.num_rows); printf("\tbase mem usage: %'lu\n", pass.base_mem_size); - auto const num_columns = _input_columns.size(); - std::vector h_page_offsets = - cudf::detail::make_std_vector_sync(pass.page_offsets, _stream); + auto const num_columns = _input_columns.size(); + auto const h_page_offsets = cudf::detail::make_host_vector_sync(pass.page_offsets, _stream); for (size_t c_idx = 0; c_idx < num_columns; c_idx++) { printf("\t\tColumn %'lu: num_pages(%'d)\n", c_idx, @@ -1426,7 +1424,7 @@ void reader::impl::setup_next_subpass(read_mode mode) subpass.pages = subpass.page_buf; } - std::vector h_spans = cudf::detail::make_std_vector_async(page_indices, _stream); + auto const h_spans = cudf::detail::make_host_vector_async(page_indices, _stream); subpass.pages.device_to_host_async(_stream); _stream.synchronize(); @@ -1464,7 +1462,7 @@ void reader::impl::setup_next_subpass(read_mode mode) printf("\t\tTotal expected usage: %'lu\n", total_expected_size == 0 ? subpass.decomp_page_data.size() + pass.base_mem_size : total_expected_size + pass.base_mem_size); - std::vector h_page_indices = cudf::detail::make_std_vector_sync(page_indices, _stream); + auto const h_page_indices = cudf::detail::make_host_vector_sync(page_indices, _stream); for (size_t c_idx = 0; c_idx < num_columns; c_idx++) { printf("\t\tColumn %'lu: pages(%'lu - %'lu)\n", c_idx, diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index c2c5dbb4a56..74992aa733f 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -2230,20 +2230,20 @@ auto convert_table_to_parquet_data(table_input_metadata& table_meta, bool need_sync{false}; // need to fetch the histogram data from the device - std::vector h_def_histogram; - std::vector h_rep_histogram; - if (stats_granularity == statistics_freq::STATISTICS_COLUMN) { - if (def_histogram_bfr_size > 0) { - h_def_histogram = - std::move(cudf::detail::make_std_vector_async(def_level_histogram, stream)); + auto const h_def_histogram = [&]() { + if (stats_granularity == statistics_freq::STATISTICS_COLUMN && def_histogram_bfr_size > 0) { need_sync = true; + return cudf::detail::make_host_vector_async(def_level_histogram, stream); } - if (rep_histogram_bfr_size > 0) { - h_rep_histogram = - std::move(cudf::detail::make_std_vector_async(rep_level_histogram, stream)); + return cudf::detail::make_host_vector(0, stream); + }(); + auto const h_rep_histogram = [&]() { + if (stats_granularity == statistics_freq::STATISTICS_COLUMN && rep_histogram_bfr_size > 0) { need_sync = true; + return cudf::detail::make_host_vector_async(rep_level_histogram, stream); } - } + return cudf::detail::make_host_vector(0, stream); + }(); for (int r = 0; r < num_rowgroups; r++) { int p = rg_to_part[r]; @@ -2265,7 +2265,7 @@ auto convert_table_to_parquet_data(table_input_metadata& table_meta, update_chunk_encoding_stats(column_chunk_meta, ck, write_v2_headers); if (ck.ck_stat_size != 0) { - std::vector const stats_blob = cudf::detail::make_std_vector_sync( + auto const stats_blob = cudf::detail::make_host_vector_sync( device_span(dev_bfr, ck.ck_stat_size), stream); CompactProtocolReader cp(stats_blob.data(), stats_blob.size()); cp.read(&column_chunk_meta.statistics); diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index 91be154e09d..e4313eba454 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -297,10 +297,10 @@ class device_buffer_source final : public datasource { { auto const count = std::min(size, this->size() - offset); auto const stream = cudf::get_default_stream(); - auto h_data = cudf::detail::make_std_vector_async( + auto h_data = cudf::detail::make_host_vector_async( cudf::device_span{_d_buffer.data() + offset, count}, stream); stream.synchronize(); - return std::make_unique>>(std::move(h_data)); + return std::make_unique>>(std::move(h_data)); } [[nodiscard]] bool supports_device_read() const override { return true; } diff --git a/cpp/src/text/jaccard.cu b/cpp/src/text/jaccard.cu index e465fb79c89..e856b89b836 100644 --- a/cpp/src/text/jaccard.cu +++ b/cpp/src/text/jaccard.cu @@ -376,7 +376,7 @@ std::pair, rmm::device_uvector> hash_subs sub_offsets.begin(), sub_offsets.end(), indices.begin()); - return cudf::detail::make_std_vector_sync(indices, stream); + return cudf::detail::make_host_vector_sync(indices, stream); }(); // Call segmented sort with the sort sections From 1a96e4cca188f4e0500a87c391ef105b49a42288 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 27 Aug 2024 18:57:48 -1000 Subject: [PATCH 135/270] Fix loc/iloc.__setitem__[:, loc] with non cupy types (#16677) Discovered in https://github.com/rapidsai/cudf/pull/16652, `DataFrame.iloc/loc.__setitem__` with a non-cupy type e.g. `"category"` failed because the indexing path unconditionally tries to `cupy.asarray` the value to be set which only accepts types recognized by cupy. We can skip this `asarray` if we have a numpy/pandas/cudf object Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16677 --- python/cudf/cudf/core/dataframe.py | 10 ++++++---- python/cudf/cudf/tests/test_indexing.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 14b63c2b0d7..d54a800aedf 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -414,8 +414,9 @@ def _setitem_tuple_arg(self, key, value): ) else: - value = cupy.asarray(value) - if value.ndim == 2: + if not is_column_like(value): + value = cupy.asarray(value) + if getattr(value, "ndim", 1) == 2: # If the inner dimension is 1, it's broadcastable to # all columns of the dataframe. indexed_shape = columns_df.loc[key[0]].shape @@ -558,8 +559,9 @@ def _setitem_tuple_arg(self, key, value): else: # TODO: consolidate code path with identical counterpart # in `_DataFrameLocIndexer._setitem_tuple_arg` - value = cupy.asarray(value) - if value.ndim == 2: + if not is_column_like(value): + value = cupy.asarray(value) + if getattr(value, "ndim", 1) == 2: indexed_shape = columns_df.iloc[key[0]].shape if value.shape[1] == 1: if value.shape[0] != indexed_shape[0]: diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index 716b4dc6acd..9df2852dde8 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -2369,3 +2369,13 @@ def test_duplicate_labels_raises(): df[["a", "a"]] with pytest.raises(ValueError): df.loc[:, ["a", "a"]] + + +@pytest.mark.parametrize("indexer", ["iloc", "loc"]) +@pytest.mark.parametrize("dtype", ["category", "timedelta64[ns]"]) +def test_loc_iloc_setitem_col_slice_non_cupy_types(indexer, dtype): + df_pd = pd.DataFrame(range(2), dtype=dtype) + df_cudf = cudf.DataFrame.from_pandas(df_pd) + getattr(df_pd, indexer)[:, 0] = getattr(df_pd, indexer)[:, 0] + getattr(df_cudf, indexer)[:, 0] = getattr(df_cudf, indexer)[:, 0] + assert_eq(df_pd, df_cudf) From 569939f40094b266a768a270d8966c5f7277c46a Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 28 Aug 2024 08:36:14 -0500 Subject: [PATCH 136/270] Fix slowdown in DataFrame repr in jupyter notebook (#16656) Fixes: #15747 This PR fixes slow-down in `DataFrame` repr inside a jupyter notebook. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Murray (https://github.com/Matt711) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/16656 --- .gitignore | 4 ++ ci/cudf_pandas_scripts/run_tests.sh | 3 + .../all_cuda-118_arch-x86_64.yaml | 4 ++ .../all_cuda-125_arch-x86_64.yaml | 4 ++ dependencies.yaml | 8 ++- python/cudf/cudf/core/dataframe.py | 4 +- python/cudf/cudf/pandas/_wrappers/pandas.py | 28 ++++++++ .../data/repr_slow_down_test.ipynb | 69 +++++++++++++++++++ .../cudf_pandas_tests/test_cudf_pandas.py | 36 ++++++++++ python/cudf/pyproject.toml | 4 ++ 10 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 python/cudf/cudf_pandas_tests/data/repr_slow_down_test.ipynb diff --git a/.gitignore b/.gitignore index 153c7f59744..619e1464b2a 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,7 @@ jupyter_execute # clang tooling compile_commands.json .clangd/ + +# pytest artifacts +rmm_log.txt +python/cudf/cudf_pandas_tests/data/rmm_log.txt diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 39056d58d56..52964496b36 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -61,6 +61,9 @@ else "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" fi +python -m pip install ipykernel +python -m ipykernel install --user --name python3 + python -m pytest -p cudf.pandas \ --cov-config=./python/cudf/.coveragerc \ --cov=cudf \ diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 96596958636..c4c32da8af2 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -37,6 +37,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython +- jupyter_client - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 @@ -48,6 +49,8 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nbconvert +- nbformat - nbsphinx - ninja - notebook @@ -57,6 +60,7 @@ dependencies: - nvcc_linux-64=11.8 - nvcomp==3.0.6 - nvtx>=0.2.1 +- openpyxl - packaging - pandas - pandas>=2.0,<2.2.3dev0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index efc5f76b90f..7439c9543a5 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -38,6 +38,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython +- jupyter_client - libcufile-dev - libcurand-dev - libkvikio==24.10.*,>=0.0.0a0 @@ -47,6 +48,8 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nbconvert +- nbformat - nbsphinx - ninja - notebook @@ -55,6 +58,7 @@ dependencies: - numpydoc - nvcomp==3.0.6 - nvtx>=0.2.1 +- openpyxl - packaging - pandas - pandas>=2.0,<2.2.3dev0 diff --git a/dependencies.yaml b/dependencies.yaml index b55860815bf..5be291b3671 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -31,6 +31,7 @@ files: - test_python_cudf - test_python_dask_cudf - test_python_pylibcudf + - test_python_cudf_pandas test_static_build: output: none includes: @@ -49,6 +50,7 @@ files: - test_python_common - test_python_cudf - test_python_dask_cudf + - test_python_cudf_pandas test_java: output: none includes: @@ -934,9 +936,13 @@ dependencies: # installation issues with `psycopg2`. - pandas[test, pyarrow, performance, computation, fss, excel, parquet, feather, hdf5, spss, html, xml, plot, output-formatting, clipboard, compression] - pytest-reportlog + - ipython test_python_cudf_pandas: common: - - output_types: [requirements, pyproject] + - output_types: [conda, requirements, pyproject] packages: - ipython + - jupyter_client + - nbconvert + - nbformat - openpyxl diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index d54a800aedf..a309b9117eb 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -680,7 +680,9 @@ class DataFrame(IndexedFrame, Serializable, GetAttrGetItemMixin): 3 3 0.3 """ - _PROTECTED_KEYS = frozenset(("_data", "_index")) + _PROTECTED_KEYS = frozenset( + ("_data", "_index", "_ipython_canary_method_should_not_exist_") + ) _accessors: set[Any] = set() _loc_indexer_type = _DataFrameLocIndexer _iloc_indexer_type = _DataFrameIlocIndexer diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index 478108f36f1..6d03063fa27 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -61,6 +61,12 @@ TimeGrouper as pd_TimeGrouper, ) +try: + from IPython import get_ipython + + ipython_shell = get_ipython() +except ImportError: + ipython_shell = None cudf.set_option("mode.pandas_compatible", True) @@ -208,6 +214,12 @@ def _DataFrame__dir__(self): ] +def ignore_ipython_canary_check(self, **kwargs): + raise AttributeError( + "_ipython_canary_method_should_not_exist_ doesn't exist" + ) + + DataFrame = make_final_proxy_type( "DataFrame", cudf.DataFrame, @@ -220,10 +232,26 @@ def _DataFrame__dir__(self): "_constructor": _FastSlowAttribute("_constructor"), "_constructor_sliced": _FastSlowAttribute("_constructor_sliced"), "_accessors": set(), + "_ipython_canary_method_should_not_exist_": ignore_ipython_canary_check, }, ) +def custom_repr_html(obj): + # This custom method is need to register a html format + # for ipython + return _fast_slow_function_call( + lambda obj: obj._repr_html_(), + obj, + )[0] + + +if ipython_shell: + # See: https://ipython.readthedocs.io/en/stable/config/integrating.html#formatters-for-third-party-types + html_formatter = ipython_shell.display_formatter.formatters["text/html"] + html_formatter.for_type(DataFrame, custom_repr_html) + + Series = make_final_proxy_type( "Series", cudf.Series, diff --git a/python/cudf/cudf_pandas_tests/data/repr_slow_down_test.ipynb b/python/cudf/cudf_pandas_tests/data/repr_slow_down_test.ipynb new file mode 100644 index 00000000000..c7d39b78810 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/data/repr_slow_down_test.ipynb @@ -0,0 +1,69 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext cudf.pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "np.random.seed(0)\n", + "\n", + "num_rows = 25_000_000\n", + "num_columns = 12\n", + "\n", + "# Create a DataFrame with random data\n", + "df = pd.DataFrame(np.random.randint(0, 100, size=(num_rows, num_columns)),\n", + " columns=[f'Column_{i}' for i in range(1, num_columns + 1)])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 028f5f173ac..0827602852d 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -14,9 +14,12 @@ import types from io import BytesIO, StringIO +import jupyter_client +import nbformat import numpy as np import pyarrow as pa import pytest +from nbconvert.preprocessors import ExecutePreprocessor from numba import NumbaDeprecationWarning from pytz import utc @@ -1650,3 +1653,36 @@ def test_change_index_name(index): assert s.index.name == name assert df.index.name == name + + +def test_notebook_slow_repr(): + notebook_filename = ( + os.path.dirname(os.path.abspath(__file__)) + + "/data/repr_slow_down_test.ipynb" + ) + with open(notebook_filename, "r", encoding="utf-8") as f: + nb = nbformat.read(f, as_version=4) + + ep = ExecutePreprocessor( + timeout=20, kernel_name=jupyter_client.KernelManager().kernel_name + ) + + try: + ep.preprocess(nb, {"metadata": {"path": "./"}}) + except Exception as e: + assert False, f"Error executing the notebook: {e}" + + # Collect the outputs + html_result = nb.cells[2]["outputs"][0]["data"]["text/html"] + for string in { + "div", + "Column_1", + "Column_2", + "Column_3", + "Column_4", + "tbody", + "
", + }: + assert ( + string in html_result + ), f"Expected string {string} not found in the output" diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 8386935fab0..0c1d5015078 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -63,11 +63,15 @@ test = [ "tzdata", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. pandas-tests = [ + "ipython", "pandas[test, pyarrow, performance, computation, fss, excel, parquet, feather, hdf5, spss, html, xml, plot, output-formatting, clipboard, compression]", "pytest-reportlog", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. cudf-pandas-tests = [ "ipython", + "jupyter_client", + "nbconvert", + "nbformat", "openpyxl", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 5491b394921ca3e03f09c9e789f1ba00da9db0b1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 28 Aug 2024 11:03:10 -0500 Subject: [PATCH 137/270] switch from typing.Callable to collections.abc.Callable (#16670) Follow-up to #16637. Once this project's minimum support Python version was bumped up to Python 3.10, `ruff` started raising this error from `pyupgrade`: ```text Import from `collections.abc` instead: `Callable` ``` * ruff docs: https://docs.astral.sh/ruff/rules/deprecated-import/ * `typing` docs saying that `typing.Callable` is deprecated starting in Python 3.9 https://docs.python.org/3/library/typing.html#typing.Callable * context: https://github.com/rapidsai/cudf/pull/16637#discussion_r1727482177 This proposes accepting that suggestion, so that `cudf` won't be broken whenever `Callable` is removed from the `typing` module. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Lawrence Mitchell (https://github.com/wence-) - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/cudf/pull/16670 --- pyproject.toml | 4 +++- python/cudf/cudf/_typing.py | 3 ++- python/cudf/cudf/core/column/numerical.py | 4 +++- python/cudf/cudf/core/column_accessor.py | 4 ++-- python/cudf/cudf/core/dataframe.py | 4 ++-- python/cudf/cudf/core/dtypes.py | 4 +++- python/cudf/cudf/core/frame.py | 4 ++-- python/cudf/cudf/core/udf/utils.py | 5 ++++- python/cudf/cudf/io/parquet.py | 6 +++++- python/cudf/cudf/options.py | 4 ++-- python/cudf/cudf/pandas/fast_slow_proxy.py | 4 ++-- python/cudf/cudf/utils/ioutils.py | 2 +- python/cudf_polars/cudf_polars/dsl/ir.py | 4 ++-- python/cudf_polars/cudf_polars/typing/__init__.py | 3 ++- python/cudf_polars/pyproject.toml | 1 - python/dask_cudf/dask_cudf/io/json.py | 2 +- 16 files changed, 36 insertions(+), 22 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e15cb7b3cdd..8f9aa165e5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,9 @@ select = [ # non-pep585-annotation "UP006", # non-pep604-annotation - "UP007" + "UP007", + # Import from `collections.abc` instead: `Callable` + "UP035", ] ignore = [ # whitespace before : diff --git a/python/cudf/cudf/_typing.py b/python/cudf/cudf/_typing.py index 34c96cc8cb3..6e8ad556b08 100644 --- a/python/cudf/cudf/_typing.py +++ b/python/cudf/cudf/_typing.py @@ -1,7 +1,8 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. import sys -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, TypeVar, Union +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Dict, Iterable, TypeVar, Union import numpy as np from pandas import Period, Timedelta, Timestamp diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 90bec049831..7f391c8a79c 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -3,7 +3,7 @@ from __future__ import annotations import functools -from typing import TYPE_CHECKING, Any, Callable, Sequence, cast +from typing import TYPE_CHECKING, Any, Sequence, cast import numpy as np import pandas as pd @@ -28,6 +28,8 @@ from .numerical_base import NumericalBaseColumn if TYPE_CHECKING: + from collections.abc import Callable + from cudf._typing import ( ColumnBinaryOperand, ColumnLike, diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 34076fa0060..09b0f453692 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -6,7 +6,7 @@ import sys from collections import abc from functools import cached_property, reduce -from typing import TYPE_CHECKING, Any, Callable, Mapping, cast +from typing import TYPE_CHECKING, Any, Mapping, cast import numpy as np import pandas as pd @@ -639,7 +639,7 @@ def _pad_key( def rename_levels( self, - mapper: Mapping[abc.Hashable, abc.Hashable] | Callable, + mapper: Mapping[abc.Hashable, abc.Hashable] | abc.Callable, level: int | None = None, ) -> Self: """ diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index a309b9117eb..6065e0e1eeb 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -13,8 +13,8 @@ import textwrap import warnings from collections import abc, defaultdict -from collections.abc import Iterator -from typing import TYPE_CHECKING, Any, Callable, Literal, MutableMapping, cast +from collections.abc import Callable, Iterator +from typing import TYPE_CHECKING, Any, Literal, MutableMapping, cast import cupy import numba diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 6d532e01cba..2110e610c37 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -7,7 +7,7 @@ import textwrap import warnings from functools import cached_property -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any import numpy as np import pandas as pd @@ -27,6 +27,8 @@ PANDAS_NUMPY_DTYPE = pd.core.dtypes.dtypes.PandasDtype if TYPE_CHECKING: + from collections.abc import Callable + from cudf._typing import Dtype from cudf.core.buffer import Buffer diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 3e1efd7c97a..cbe1e97d834 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -6,7 +6,7 @@ import pickle import warnings from collections import abc -from typing import TYPE_CHECKING, Any, Callable, Literal, MutableMapping +from typing import TYPE_CHECKING, Any, Literal, MutableMapping # TODO: The `numpy` import is needed for typing purposes during doc builds # only, need to figure out why the `np` alias is insufficient then remove. @@ -403,7 +403,7 @@ def __arrow_array__(self, type=None): @_performance_tracking def _to_array( self, - get_array: Callable, + get_array: abc.Callable, module: ModuleType, copy: bool, dtype: Dtype | None = None, diff --git a/python/cudf/cudf/core/udf/utils.py b/python/cudf/cudf/core/udf/utils.py index d616761cb3b..6d7362952c9 100644 --- a/python/cudf/cudf/core/udf/utils.py +++ b/python/cudf/cudf/core/udf/utils.py @@ -3,7 +3,7 @@ import functools import os -from typing import Any, Callable +from typing import TYPE_CHECKING, Any import cachetools import cupy as cp @@ -41,6 +41,9 @@ from cudf.utils.performance_tracking import _performance_tracking from cudf.utils.utils import initfunc +if TYPE_CHECKING: + from collections.abc import Callable + # Maximum size of a string column is 2 GiB _STRINGS_UDF_DEFAULT_HEAP_SIZE = os.environ.get("STRINGS_UDF_HEAP_SIZE", 2**31) _heap_size = 0 diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 6b895abbf66..d6b2ae2f31c 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -10,7 +10,7 @@ from collections import defaultdict from contextlib import ExitStack from functools import partial, reduce -from typing import Callable +from typing import TYPE_CHECKING from uuid import uuid4 import numpy as np @@ -24,6 +24,10 @@ from cudf.utils import ioutils from cudf.utils.performance_tracking import _performance_tracking +if TYPE_CHECKING: + from collections.abc import Callable + + BYTE_SIZES = { "kb": 1000, "mb": 1000000, diff --git a/python/cudf/cudf/options.py b/python/cudf/cudf/options.py index 94e73021cec..df7bbe22a61 100644 --- a/python/cudf/cudf/options.py +++ b/python/cudf/cudf/options.py @@ -5,10 +5,10 @@ import textwrap from contextlib import ContextDecorator from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from collections.abc import Container + from collections.abc import Callable, Container @dataclass diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index bb678fd1efe..4b0fd9a5b36 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -10,9 +10,9 @@ import pickle import types import warnings -from collections.abc import Iterator +from collections.abc import Callable, Iterator from enum import IntEnum -from typing import Any, Callable, Literal, Mapping +from typing import Any, Literal, Mapping import numpy as np diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index e5944d7093c..94974e595b1 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -4,9 +4,9 @@ import os import urllib import warnings +from collections.abc import Callable from io import BufferedWriter, BytesIO, IOBase, TextIOWrapper from threading import Thread -from typing import Callable import fsspec import fsspec.implementations.local diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index ebc7dee6bfb..e334e6f5cc5 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -18,7 +18,7 @@ import types from functools import cache from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar import pyarrow as pa import pylibcudf as plc @@ -31,7 +31,7 @@ from cudf_polars.utils import sorting if TYPE_CHECKING: - from collections.abc import MutableMapping + from collections.abc import Callable, MutableMapping from typing import Literal from cudf_polars.typing import Schema diff --git a/python/cudf_polars/cudf_polars/typing/__init__.py b/python/cudf_polars/cudf_polars/typing/__init__.py index 5276073e62a..adab10bdded 100644 --- a/python/cudf_polars/cudf_polars/typing/__init__.py +++ b/python/cudf_polars/cudf_polars/typing/__init__.py @@ -13,7 +13,8 @@ from polars.polars import _expr_nodes as pl_expr, _ir_nodes as pl_ir if TYPE_CHECKING: - from typing import Callable, TypeAlias + from collections.abc import Callable + from typing import TypeAlias import polars as pl diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 0382e3ce6a2..f2bab9e6623 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -115,7 +115,6 @@ ignore = [ # tryceratops "TRY003", # Avoid specifying long messages outside the exception class # pyupgrade - "UP035", # Import from `collections.abc` instead: `Callable` "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` # Lints below are turned off because of conflicts with the ruff # formatter diff --git a/python/dask_cudf/dask_cudf/io/json.py b/python/dask_cudf/dask_cudf/io/json.py index 8705d98e9d6..98c5ceedb76 100644 --- a/python/dask_cudf/dask_cudf/io/json.py +++ b/python/dask_cudf/dask_cudf/io/json.py @@ -81,7 +81,7 @@ def read_json( If str, this value will be used as the ``engine`` argument when :func:`cudf.read_json` is used to create each partition. - If a :obj:`~typing.Callable`, this value will be used as the + If a :obj:`~collections.abc.Callable`, this value will be used as the underlying function used to create each partition from JSON data. The default value is "auto", so that ``engine=partial(cudf.read_json, engine="auto")`` will be From c600a65e4fd82a4a6eb00feaee032b62872de761 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Wed, 28 Aug 2024 09:55:11 -0700 Subject: [PATCH 138/270] Update documentation for Dask cuDF (#16671) General documentation update for Dask cuDF: - Adds `README.md` file to `dask_cudf` (this is currently a symlink to cudf's README, which isn't terribly helpful) - Emphasizes direct usage of the `dask.dataframe` API (rather than the explicit `dask_cudf` API) - Including the `to_backend` API - Advertises query-planning support - Includes a simple Dask CUDA example (and best-practices link) Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Mads R. B. Kristensen (https://github.com/madsbk) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16671 --- docs/cudf/source/user_guide/10min.ipynb | 6 +- python/dask_cudf/README.md | 136 +++++++++++++++++++++++- 2 files changed, 140 insertions(+), 2 deletions(-) mode change 120000 => 100644 python/dask_cudf/README.md diff --git a/docs/cudf/source/user_guide/10min.ipynb b/docs/cudf/source/user_guide/10min.ipynb index c3da2558db8..2eaa75b3189 100644 --- a/docs/cudf/source/user_guide/10min.ipynb +++ b/docs/cudf/source/user_guide/10min.ipynb @@ -15,7 +15,11 @@ "\n", "[Dask](https://dask.org/) is a flexible library for parallel computing in Python that makes scaling out your workflow smooth and simple. On the CPU, Dask uses Pandas to execute operations in parallel on DataFrame partitions.\n", "\n", - "[Dask-cuDF](https://github.com/rapidsai/cudf/tree/main/python/dask_cudf) extends Dask where necessary to allow its DataFrame partitions to be processed using cuDF GPU DataFrames instead of Pandas DataFrames. For instance, when you call `dask_cudf.read_csv(...)`, your cluster's GPUs do the work of parsing the CSV file(s) by calling [`cudf.read_csv()`](https://docs.rapids.ai/api/cudf/stable/api_docs/api/cudf.read_csv.html).\n", + "[Dask cuDF](https://github.com/rapidsai/cudf/tree/main/python/dask_cudf) extends Dask where necessary to allow its DataFrame partitions to be processed using cuDF GPU DataFrames instead of Pandas DataFrames. For instance, when you call `dask_cudf.read_csv(...)`, your cluster's GPUs do the work of parsing the CSV file(s) by calling [`cudf.read_csv()`](https://docs.rapids.ai/api/cudf/stable/api_docs/api/cudf.read_csv.html).\n", + "\n", + "\n", + "> [!NOTE] \n", + "> This notebook uses the explicit Dask cuDF API (`dask_cudf`) for clarity. However, we strongly recommend that you use Dask's [configuration infrastructure](https://docs.dask.org/en/latest/configuration.html) to set the `\"dataframe.backend\"` to `\"cudf\"`, and work with the `dask.dataframe` API directly. Please see the [Dask cuDF documentation](https://github.com/rapidsai/cudf/tree/main/python/dask_cudf) for more information.\n", "\n", "\n", "## When to use cuDF and Dask-cuDF\n", diff --git a/python/dask_cudf/README.md b/python/dask_cudf/README.md deleted file mode 120000 index fe840054137..00000000000 --- a/python/dask_cudf/README.md +++ /dev/null @@ -1 +0,0 @@ -../../README.md \ No newline at end of file diff --git a/python/dask_cudf/README.md b/python/dask_cudf/README.md new file mode 100644 index 00000000000..6edb9f87d48 --- /dev/null +++ b/python/dask_cudf/README.md @@ -0,0 +1,135 @@ +#
 Dask cuDF - A GPU Backend for Dask DataFrame
+ +Dask cuDF (a.k.a. dask-cudf or `dask_cudf`) is an extension library for [Dask DataFrame](https://docs.dask.org/en/stable/dataframe.html). When installed, Dask cuDF is automatically registered as the `"cudf"` [dataframe backend](https://docs.dask.org/en/stable/how-to/selecting-the-collection-backend.html) for Dask DataFrame. + +## Using Dask cuDF + +### The Dask DataFrame API (Recommended) + +Simply set the `"dataframe.backend"` [configuration](https://docs.dask.org/en/stable/configuration.html) to `"cudf"` in Dask, and the public Dask DataFrame API will leverage `cudf` automatically: + +```python +import dask +dask.config.set({"dataframe.backend": "cudf"}) + +import dask.dataframe as dd +# This gives us a cuDF-backed dataframe +df = dd.read_parquet("data.parquet", ...) +``` + +> [!IMPORTANT] +> The `"dataframe.backend"` configuration will only be used for collection creation when the following APIs are used: `read_parquet`, `read_json`, `read_csv`, `read_orc`, `read_hdf`, and `from_dict`. For example, if `from_map`, `from_pandas`, `from_delayed`, or `from_array` are used, the backend of the new collection will depend on the input to the function: + +```python +import pandas as pd +import cudf + +# This gives us a Pandas-backed dataframe +dd.from_pandas(pd.DataFrame({"a": range(10)})) + +# This gives us a cuDF-backed dataframe +dd.from_pandas(cudf.DataFrame({"a": range(10)})) +``` + +A cuDF-backed DataFrame collection can be moved to the `"pandas"` backend: + +```python +df = df.to_backend("pandas") +``` + +Similarly, a Pandas-backed DataFrame collection can be moved to the `"cudf"` backend: + +```python +df = df.to_backend("cudf") +``` + +### The Explicit Dask cuDF API + +In addition to providing the `"cudf"` backend for Dask DataFrame, Dask cuDF also provides an explicit `dask_cudf` API: + +```python +import dask_cudf + +# This always gives us a cuDF-backed dataframe +df = dask_cudf.read_parquet("data.parquet", ...) +``` + +> [!NOTE] +> This API is used implicitly by the Dask DataFrame API when the `"cudf"` backend is enabled. Therefore, using it directly will not provide any performance benefit over the CPU/GPU-portable `dask.dataframe` API. Also, using some parts of the explicit API are incompatible with automatic query planning (see the next section). + +See the [Dask cuDF's API documentation](https://docs.rapids.ai/api/dask-cudf/stable/) for further information. + +## Query Planning + +Dask cuDF now provides automatic query planning by default (RAPIDS 24.06+). As long as the `"dataframe.query-planning"` configuration is set to `True` (the default) when `dask.dataframe` is first imported, [Dask Expressions](https://github.com/dask/dask-expr) will be used under the hood. + +For example, the following user code will automatically benefit from predicate pushdown when the result is computed. + +```python +df = dd.read_parquet("/my/parquet/dataset/") +result = df.sort_values('B')['A'] +``` + +Unoptimized expression graph (`df.pprint()`): +``` +Projection: columns='A' + SortValues: by=['B'] shuffle_method='tasks' options={} + ReadParquetFSSpec: path='/my/parquet/dataset/' ... +``` + +Simplified expression graph (`df.simplify().pprint()`): +``` +Projection: columns='A' + SortValues: by=['B'] shuffle_method='tasks' options={} + ReadParquetFSSpec: path='/my/parquet/dataset/' columns=['A', 'B'] ... +``` + +> [!NOTE] +> Dask will automatically simplify the expression graph (within `optimize`) when the result is converted to a task graph (via `compute` or `persist`). The user does not need to call `simplify` themself. + + +## Using Multiple GPUs and Multiple Nodes + +Whenever possible, Dask cuDF (i.e. Dask DataFrame) will automatically try to partition your data into small-enough tasks to fit comfortably in the memory of a single GPU. This means the necessary compute tasks needed to compute a query can often be streamed to a single GPU process for out-of-core computing. This also means that the compute tasks can be executed in parallel over a multi-GPU cluster. + +> [!IMPORTANT] +> Neither Dask cuDF nor Dask DataFrame provide support for multi-GPU or multi-node execution on their own. You must deploy a distributed cluster (ideally with [Dask CUDA](https://docs.rapids.ai/api/dask-cuda/stable/)) to leverage multiple GPUs. + +In order to execute your Dask workflow on multiple GPUs, you will typically need to use [Dask CUDA](https://docs.rapids.ai/api/dask-cuda/stable/) to deploy distributed Dask cluster, and [Distributed](https://distributed.dask.org/en/stable/client.html) to define a `client` object. For example: + +```python + +from dask_cuda import LocalCUDACluster +from distributed import Client + +client = Client( + LocalCUDACluster( + CUDA_VISIBLE_DEVICES="0,1", # Use two workers (on devices 0 and 1) + rmm_pool_size=0.9, # Use 90% of GPU memory as a pool for faster allocations + enable_cudf_spill=True, # Improve device memory stability + local_directory="/fast/scratch/", # Use fast local storage for spilling + ) +) + +df = dd.read_parquet("/my/parquet/dataset/") +agg = df.groupby('B').sum() +agg.compute() # This will use the cluster defined above +``` + +> [!NOTE] +> This example uses `compute` to materialize a concrete `cudf.DataFrame` object in local memory. Never call `compute` on a large collection that cannot fit comfortably in the memory of a single GPU! See Dask's [documentation on managing computation](https://distributed.dask.org/en/stable/manage-computation.html) for more details. + +Please see the [Dask CUDA](https://docs.rapids.ai/api/dask-cuda/stable/) documentation for more information about deploying GPU-aware clusters (including [best practices](https://docs.rapids.ai/api/dask-cuda/stable/examples/best-practices/)). + +## Install + +See the [RAPIDS install page](https://docs.rapids.ai/install) for the most up-to-date information and commands for installing Dask cuDF and other RAPIDS packages. + +## Resources + +- [Dask cuDF API documentation](https://docs.rapids.ai/api/dask-cudf/stable/) +- [cuDF API documentation](https://docs.rapids.ai/api/cudf/stable/) +- [10 Minutes to cuDF and Dask cuDF](https://docs.rapids.ai/api/cudf/stable/user_guide/10min/) +- [Dask CUDA documentation](https://docs.rapids.ai/api/dask-cuda/stable/) +- [Deployment](https://docs.rapids.ai/deployment/stable/) +- [RAPIDS Community](https://rapids.ai/learn-more/#get-involved): Get help, contribute, and collaborate. From 872e01e8c11fe61051d7be46f09f285252f2c6ac Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 28 Aug 2024 12:08:38 -0500 Subject: [PATCH 139/270] Fix slowdown in `CategoricalIndex.__repr__` (#16665) Fixes: #13297 This PR fixes a slow-down in performing repr of a `CategoricalIndex` when there are too many unique values. There was no other choice to fix this in a better way by using public APIs, because all the public APIs seem to be performing categories validation even if `fastpath=True`. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Murray (https://github.com/Matt711) URL: https://github.com/rapidsai/cudf/pull/16665 --- python/cudf/cudf/core/index.py | 16 +++++++++++++++- python/cudf/cudf/testing/_utils.py | 21 +++++++++++++++++++++ python/cudf/cudf/tests/test_repr.py | 11 +++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index df8af856f4f..27c6556f976 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1443,7 +1443,21 @@ def __repr__(self): output[:break_idx].replace("'", "") + output[break_idx:] ) else: - output = repr(preprocess.to_pandas()) + # Too many non-unique categories will cause + # the output to take too long. In this case, we + # split the categories into data and categories + # and generate the repr separately and + # merge them. + pd_cats = pd.Categorical( + preprocess.astype(preprocess.categories.dtype).to_pandas() + ) + pd_preprocess = pd.CategoricalIndex(pd_cats) + data_repr = repr(pd_preprocess).split("\n") + pd_preprocess.dtype._categories = ( + preprocess.categories.to_pandas() + ) + cats_repr = repr(pd_preprocess).split("\n") + output = "\n".join(data_repr[:-1] + cats_repr[-1:]) output = output.replace("nan", str(cudf.NA)) elif preprocess._values.nullable: diff --git a/python/cudf/cudf/testing/_utils.py b/python/cudf/cudf/testing/_utils.py index a6a2d4eea00..540f12c8382 100644 --- a/python/cudf/cudf/testing/_utils.py +++ b/python/cudf/cudf/testing/_utils.py @@ -1,6 +1,7 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. import itertools +import signal import string from collections import abc from contextlib import contextmanager @@ -368,3 +369,23 @@ def sv_to_udf_str_testing_lowering(context, builder, sig, args): return cast_string_view_to_udf_string( context, builder, sig.args[0], sig.return_type, args[0] ) + + +class cudf_timeout: + """ + Context manager to raise a TimeoutError after a specified number of seconds. + """ + + def __init__(self, seconds, *, timeout_message=""): + self.seconds = int(seconds) + self.timeout_message = timeout_message + + def _timeout_handler(self, signum, frame): + raise TimeoutError(self.timeout_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self._timeout_handler) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) diff --git a/python/cudf/cudf/tests/test_repr.py b/python/cudf/cudf/tests/test_repr.py index a013745f71e..57eef9e3463 100644 --- a/python/cudf/cudf/tests/test_repr.py +++ b/python/cudf/cudf/tests/test_repr.py @@ -1480,3 +1480,14 @@ def test_interval_index_repr(): gi = cudf.from_pandas(pi) assert repr(pi) == repr(gi) + + +def test_large_unique_categories_repr(): + # Unfortunately, this is a long running test (takes about 1 minute) + # and there is no way we can reduce the time + pi = pd.CategoricalIndex(range(100_000_000)) + gi = cudf.CategoricalIndex(range(100_000_000)) + expected_repr = repr(pi) + with utils.cudf_timeout(2, timeout_message="Failed to repr fast enough"): + actual_repr = repr(gi) + assert expected_repr == actual_repr From dba6c1fe37bbc4a3b15123bfd3a5c1d5cf693fe3 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:29:38 -1000 Subject: [PATCH 140/270] Remove build_categorical_column in favor of CategoricalColumn constructor (#16617) `build_categorical_column` was largely redundant with the CategoricalColumn constructor, so in the spirit of having One Way to Do Things, replacing the former with the latter. There is usage of `build_categorical_column` in cugraph that has been replaced in https://github.com/rapidsai/cugraph/pull/4618 Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Richard (Rick) Zamora (https://github.com/rjzamora) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16617 --- python/cudf/cudf/core/_internals/where.py | 13 - python/cudf/cudf/core/column/__init__.py | 1 - python/cudf/cudf/core/column/categorical.py | 266 +++++++++---------- python/cudf/cudf/core/column/column.py | 96 +++---- python/cudf/cudf/core/column/numerical.py | 20 +- python/cudf/cudf/core/cut.py | 17 +- python/cudf/cudf/core/dataframe.py | 33 +-- python/cudf/cudf/core/df_protocol.py | 28 +- python/cudf/cudf/core/frame.py | 25 +- python/cudf/cudf/core/index.py | 18 +- python/cudf/cudf/core/indexed_frame.py | 12 +- python/cudf/cudf/core/series.py | 21 +- python/cudf/cudf/core/single_column_frame.py | 7 +- python/cudf/cudf/io/parquet.py | 16 +- python/dask_cudf/dask_cudf/backends.py | 19 +- python/dask_cudf/dask_cudf/io/parquet.py | 12 +- 16 files changed, 284 insertions(+), 320 deletions(-) diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 0c754317185..2199d4d5ba5 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -106,19 +106,6 @@ def _check_and_cast_columns_with_other( return _normalize_categorical(source_col.astype(common_dtype), other) -def _make_categorical_like(result, column): - if isinstance(column, cudf.core.column.CategoricalColumn): - result = cudf.core.column.build_categorical_column( - categories=column.categories, - codes=result, - mask=result.base_mask, - size=result.size, - offset=result.offset, - ordered=column.ordered, - ) - return result - - def _can_cast(from_dtype, to_dtype): """ Utility function to determine if we can cast diff --git a/python/cudf/cudf/core/column/__init__.py b/python/cudf/cudf/core/column/__init__.py index e7119fcdf47..5781d77ee9a 100644 --- a/python/cudf/cudf/core/column/__init__.py +++ b/python/cudf/cudf/core/column/__init__.py @@ -8,7 +8,6 @@ from cudf.core.column.column import ( ColumnBase, as_column, - build_categorical_column, build_column, column_empty, column_empty_like, diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index a7e98e5218f..de5ed15771d 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -52,6 +52,15 @@ _DEFAULT_CATEGORICAL_VALUE = np.int8(-1) +def as_unsigned_codes( + num_cats: int, codes: NumericalColumn +) -> NumericalColumn: + codes_dtype = min_unsigned_type(num_cats) + return cast( + cudf.core.column.numerical.NumericalColumn, codes.astype(codes_dtype) + ) + + class CategoricalAccessor(ColumnMethods): """ Accessor object for categorical properties of the Series values. @@ -637,13 +646,12 @@ def __setitem__(self, key, value): value = value.codes codes = self.codes codes[key] = value - out = cudf.core.column.build_categorical_column( - categories=self.categories, - codes=codes, - mask=codes.base_mask, + out = type(self)( + data=self.data, size=codes.size, - offset=self.offset, - ordered=self.ordered, + dtype=self.dtype, + mask=codes.base_mask, + children=(codes,), ) self._mimic_inplace(out, inplace=True) @@ -669,16 +677,13 @@ def _fill( def slice(self, start: int, stop: int, stride: int | None = None) -> Self: codes = self.codes.slice(start, stop, stride) - return cast( - Self, - cudf.core.column.build_categorical_column( - categories=self.categories, - codes=codes, - mask=codes.base_mask, - ordered=self.ordered, - size=codes.size, - offset=codes.offset, - ), + return type(self)( + data=self.data, # type: ignore[arg-type] + size=codes.size, + dtype=self.dtype, + mask=codes.base_mask, + offset=codes.offset, + children=(codes,), ) def _reduce( @@ -719,7 +724,7 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: ) return self.codes._binaryop(other.codes, op) - def normalize_binop_value(self, other: ScalarLike) -> CategoricalColumn: + def normalize_binop_value(self, other: ScalarLike) -> Self: if isinstance(other, column.ColumnBase): if not isinstance(other, CategoricalColumn): return NotImplemented @@ -727,30 +732,27 @@ def normalize_binop_value(self, other: ScalarLike) -> CategoricalColumn: raise TypeError( "Categoricals can only compare with the same type" ) - return other - - ary = column.as_column( + return cast(Self, other) + codes = column.as_column( self._encode(other), length=len(self), dtype=self.codes.dtype ) - return column.build_categorical_column( - categories=self.dtype.categories._values, - codes=column.as_column(ary), + return type(self)( + data=None, + size=self.size, + dtype=self.dtype, mask=self.base_mask, - ordered=self.dtype.ordered, + children=(codes,), # type: ignore[arg-type] ) - def sort_values( - self, ascending: bool = True, na_position="last" - ) -> CategoricalColumn: + def sort_values(self, ascending: bool = True, na_position="last") -> Self: codes = self.codes.sort_values(ascending, na_position) - col = column.build_categorical_column( - categories=self.dtype.categories._values, - codes=codes, - mask=codes.base_mask, + return type(self)( + data=self.data, # type: ignore[arg-type] size=codes.size, - ordered=self.dtype.ordered, + dtype=self.dtype, + mask=codes.base_mask, + children=(codes,), ) - return col def element_indexing(self, index: int) -> ScalarLike: val = self.codes.element_indexing(index) @@ -777,12 +779,12 @@ def to_pandas( if self.categories.dtype.kind == "f": new_mask = bools_to_mask(self.notnull()) - col = column.build_categorical_column( - categories=self.categories, - codes=column.as_column(self.codes, dtype=self.codes.dtype), + col = type(self)( + data=self.data, # type: ignore[arg-type] + size=self.size, + dtype=self.dtype, mask=new_mask, - ordered=self.dtype.ordered, - size=self.codes.size, + children=self.children, ) else: col = self @@ -849,15 +851,15 @@ def data_array_view( ) -> numba.cuda.devicearray.DeviceNDArray: return self.codes.data_array_view(mode=mode) - def unique(self) -> CategoricalColumn: + def unique(self) -> Self: codes = self.codes.unique() - return column.build_categorical_column( - categories=self.categories, - codes=codes, + return type(self)( + data=self.data, # type: ignore[arg-type] + size=codes.size, + dtype=self.dtype, mask=codes.base_mask, offset=codes.offset, - size=codes.size, - ordered=self.ordered, + children=(codes,), ) def _encode(self, value) -> ScalarLike: @@ -988,14 +990,17 @@ def find_and_replace( output = libcudf.replace.replace( replaced_codes, to_replace_col, replacement_col ) + codes = as_unsigned_codes(len(new_cats["cats"]), output) - result = column.build_categorical_column( - categories=new_cats["cats"], - codes=output, - mask=output.base_mask, - offset=output.offset, - size=output.size, - ordered=self.dtype.ordered, + result = type(self)( + data=self.data, # type: ignore[arg-type] + size=codes.size, + dtype=CategoricalDtype( + categories=new_cats["cats"], ordered=self.dtype.ordered + ), + mask=codes.base_mask, + offset=codes.offset, + children=(codes,), ) if result.dtype != self.dtype: warnings.warn( @@ -1082,7 +1087,7 @@ def is_monotonic_increasing(self) -> bool: def is_monotonic_decreasing(self) -> bool: return bool(self.ordered) and self.codes.is_monotonic_decreasing - def as_categorical_column(self, dtype: Dtype) -> CategoricalColumn: + def as_categorical_column(self, dtype: Dtype) -> Self: if isinstance(dtype, str) and dtype == "category": return self if isinstance(dtype, pd.CategoricalDtype): @@ -1099,7 +1104,23 @@ def as_categorical_column(self, dtype: Dtype) -> CategoricalColumn: if not isinstance(self.categories, type(dtype.categories._column)): # If both categories are of different Column types, # return a column full of Nulls. - return _create_empty_categorical_column(self, dtype) + codes = cast( + cudf.core.column.numerical.NumericalColumn, + column.as_column( + _DEFAULT_CATEGORICAL_VALUE, + length=self.size, + dtype=self.codes.dtype, + ), + ) + codes = as_unsigned_codes(len(dtype.categories), codes) + return type(self)( + data=self.data, # type: ignore[arg-type] + size=self.size, + dtype=dtype, + mask=self.base_mask, + offset=self.offset, + children=(codes,), + ) return self.set_categories( new_categories=dtype.categories, ordered=bool(dtype.ordered) @@ -1185,26 +1206,29 @@ def _concat( codes = [o for o in codes if len(o)] codes_col = libcudf.concat.concat_columns(objs) - return column.build_categorical_column( - categories=column.as_column(cats), - codes=codes_col, - mask=codes_col.base_mask, + codes_col = as_unsigned_codes( + len(cats), + cast(cudf.core.column.numerical.NumericalColumn, codes_col), + ) + return CategoricalColumn( + data=None, size=codes_col.size, + dtype=CategoricalDtype(categories=cats), + mask=codes_col.base_mask, offset=codes_col.offset, + children=(codes_col,), # type: ignore[arg-type] ) - def _with_type_metadata( - self: CategoricalColumn, dtype: Dtype - ) -> CategoricalColumn: + def _with_type_metadata(self: Self, dtype: Dtype) -> Self: if isinstance(dtype, CategoricalDtype): - return column.build_categorical_column( - categories=dtype.categories._values, - codes=self.codes, - mask=self.codes.base_mask, - ordered=dtype.ordered, + return type(self)( + data=self.data, # type: ignore[arg-type] size=self.codes.size, + dtype=dtype, + mask=self.codes.base_mask, offset=self.codes.offset, null_count=self.codes.null_count, + children=(self.codes,), ) return self @@ -1213,7 +1237,7 @@ def set_categories( new_categories: Any, ordered: bool = False, rename: bool = False, - ) -> CategoricalColumn: + ) -> Self: # See CategoricalAccessor.set_categories. ordered = ordered if ordered is not None else self.ordered @@ -1232,25 +1256,39 @@ def set_categories( "new_categories must have the same " "number of items as old categories" ) - - out_col = column.build_categorical_column( - categories=new_categories, - codes=self.base_children[0], - mask=self.base_mask, + out_col = type(self)( + data=self.data, # type: ignore[arg-type] size=self.size, + dtype=CategoricalDtype( + categories=new_categories, ordered=ordered + ), + mask=self.base_mask, offset=self.offset, - ordered=ordered, + children=(self.codes,), ) else: out_col = self if type(out_col.categories) is not type(new_categories): # If both categories are of different Column types, # return a column full of Nulls. - out_col = _create_empty_categorical_column( - self, - CategoricalDtype( + new_codes = cast( + cudf.core.column.numerical.NumericalColumn, + column.as_column( + _DEFAULT_CATEGORICAL_VALUE, + length=self.size, + dtype=self.codes.dtype, + ), + ) + new_codes = as_unsigned_codes(len(new_categories), new_codes) + out_col = type(self)( + data=self.data, # type: ignore[arg-type] + size=self.size, + dtype=CategoricalDtype( categories=new_categories, ordered=ordered ), + mask=self.base_mask, + offset=self.offset, + children=(new_codes,), ) elif ( not out_col._categories_equal(new_categories, ordered=True) @@ -1335,19 +1373,19 @@ def _set_categories( df.reset_index(drop=True, inplace=True) ordered = ordered if ordered is not None else self.ordered - new_codes = df._data["new_codes"] + new_codes = cast( + cudf.core.column.numerical.NumericalColumn, df._data["new_codes"] + ) # codes can't have masks, so take mask out before moving in - return cast( - Self, - column.build_categorical_column( - categories=new_cats, - codes=new_codes, - mask=new_codes.base_mask, - size=new_codes.size, - offset=new_codes.offset, - ordered=ordered, - ), + new_codes = as_unsigned_codes(len(new_cats), new_codes) + return type(self)( + data=self.data, # type: ignore[arg-type] + size=new_codes.size, + dtype=CategoricalDtype(categories=new_cats, ordered=ordered), + mask=new_codes.base_mask, + offset=new_codes.offset, + children=(new_codes,), ) def add_categories(self, new_categories: Any) -> Self: @@ -1425,56 +1463,16 @@ def remove_unused_categories(self) -> Self: "remove_unused_categories is currently not supported." ) - def as_ordered(self, ordered: bool): + def as_ordered(self, ordered: bool) -> Self: if self.dtype.ordered == ordered: return self - return column.build_categorical_column( - categories=self.categories, - codes=self.codes, - mask=self.base_mask, + return type(self)( + data=self.data, # type: ignore[arg-type] size=self.size, + dtype=CategoricalDtype( + categories=self.categories, ordered=ordered + ), + mask=self.base_mask, offset=self.offset, - ordered=ordered, + children=self.children, ) - - -def _create_empty_categorical_column( - categorical_column: CategoricalColumn, dtype: "CategoricalDtype" -) -> CategoricalColumn: - return column.build_categorical_column( - categories=column.as_column(dtype.categories), - codes=column.as_column( - _DEFAULT_CATEGORICAL_VALUE, - length=categorical_column.size, - dtype=categorical_column.codes.dtype, - ), - offset=categorical_column.offset, - size=categorical_column.size, - mask=categorical_column.base_mask, - ordered=dtype.ordered, - ) - - -def pandas_categorical_as_column( - categorical: ColumnLike, codes: ColumnLike | None = None -) -> CategoricalColumn: - """Creates a CategoricalColumn from a pandas.Categorical - - If ``codes`` is defined, use it instead of ``categorical.codes`` - """ - codes = categorical.codes if codes is None else codes - codes = column.as_column(codes) - - valid_codes = codes != codes.dtype.type(_DEFAULT_CATEGORICAL_VALUE) - - mask = None - if not valid_codes.all(): - mask = bools_to_mask(valid_codes) - - return column.build_categorical_column( - categories=categorical.categories, - codes=codes, - size=codes.size, - mask=mask, - ordered=categorical.ordered, - ) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 60b4126ddd4..885476a897c 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -352,13 +352,17 @@ def from_arrow(cls, array: pa.Array) -> ColumnBase: codes = libcudf.interop.from_arrow(indices_table)[0] categories = libcudf.interop.from_arrow(dictionaries_table)[0] - - return build_categorical_column( - categories=categories, - codes=codes, - mask=codes.base_mask, + codes = cudf.core.column.categorical.as_unsigned_codes( + len(categories), codes + ) + return cudf.core.column.CategoricalColumn( + data=None, size=codes.size, - ordered=array.type.ordered, + dtype=CategoricalDtype( + categories=categories, ordered=array.type.ordered + ), + mask=codes.base_mask, + children=(codes,), ) result = libcudf.interop.from_arrow(data)[0] @@ -950,10 +954,10 @@ def is_monotonic_decreasing(self) -> bool: ) def sort_values( - self: ColumnBase, + self: Self, ascending: bool = True, na_position: str = "last", - ) -> ColumnBase: + ) -> Self: if (not ascending and self.is_monotonic_decreasing) or ( ascending and self.is_monotonic_increasing ): @@ -1041,12 +1045,16 @@ def as_categorical_column(self, dtype) -> ColumnBase: and dtype._categories is not None ): cat_col = dtype._categories - labels = self._label_encoding(cats=cat_col) - return build_categorical_column( - categories=cat_col, - codes=labels, + codes = self._label_encoding(cats=cat_col) + codes = cudf.core.column.categorical.as_unsigned_codes( + len(cat_col), codes + ) + return cudf.core.column.categorical.CategoricalColumn( + data=None, + size=None, + dtype=dtype, mask=self.mask, - ordered=dtype.ordered, + children=(codes,), ) # Categories must be unique and sorted in ascending order. @@ -1058,15 +1066,16 @@ def as_categorical_column(self, dtype) -> ColumnBase: # columns include null index in factorization; remove: if self.has_nulls(): cats = cats.dropna() - min_type = min_unsigned_type(len(cats), 8) - if cudf.dtype(min_type).itemsize < labels.dtype.itemsize: - labels = labels.astype(min_type) - return build_categorical_column( - categories=cats, - codes=labels, + labels = cudf.core.column.categorical.as_unsigned_codes( + len(cats), labels + ) + return cudf.core.column.categorical.CategoricalColumn( + data=None, + size=None, + dtype=CategoricalDtype(categories=cats, ordered=ordered), mask=self.mask, - ordered=ordered, + children=(labels,), ) def as_numerical_column( @@ -1186,7 +1195,7 @@ def searchsorted( na_position=na_position, ) - def unique(self) -> ColumnBase: + def unique(self) -> Self: """ Get unique values in the data """ @@ -1695,51 +1704,6 @@ def build_column( raise TypeError(f"Unrecognized dtype: {dtype}") -def build_categorical_column( - categories: ColumnBase, - codes: ColumnBase, - mask: Buffer | None = None, - size: int | None = None, - offset: int = 0, - null_count: int | None = None, - ordered: bool = False, -) -> "cudf.core.column.CategoricalColumn": - """ - Build a CategoricalColumn - - Parameters - ---------- - categories : Column - Column of categories - codes : Column - Column of codes, the size of the resulting Column will be - the size of `codes` - mask : Buffer - Null mask - size : int, optional - offset : int, optional - ordered : bool, default False - Indicates whether the categories are ordered - """ - codes_dtype = min_unsigned_type(len(categories)) - codes = as_column(codes) - if codes.dtype != codes_dtype: - codes = codes.astype(codes_dtype) - - dtype = CategoricalDtype(categories=categories, ordered=ordered) - - result = build_column( - data=None, - dtype=dtype, - mask=mask, - size=size, - offset=offset, - null_count=null_count, - children=(codes,), - ) - return cast("cudf.core.column.CategoricalColumn", result) - - def check_invalid_array(shape: tuple, dtype): """Invalid ndarrays properties that are not supported""" if len(shape) > 1: diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 7f391c8a79c..78d2814ed26 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -651,22 +651,20 @@ def can_cast_safely(self, to_dtype: DtypeObj) -> bool: return False - def _with_type_metadata(self: ColumnBase, dtype: Dtype) -> ColumnBase: + def _with_type_metadata(self: Self, dtype: Dtype) -> ColumnBase: if isinstance(dtype, CategoricalDtype): - return column.build_categorical_column( - categories=dtype.categories._values, - codes=cudf.core.column.NumericalColumn( - self.base_data, # type: ignore[arg-type] - self.size, - dtype=self.dtype, - ), - mask=self.base_mask, - ordered=dtype.ordered, + codes = cudf.core.column.categorical.as_unsigned_codes( + len(dtype.categories), self + ) + return cudf.core.column.CategoricalColumn( + data=None, size=self.size, + dtype=dtype, + mask=self.base_mask, offset=self.offset, null_count=self.null_count, + children=(codes,), ) - return self def to_pandas( diff --git a/python/cudf/cudf/core/cut.py b/python/cudf/cudf/core/cut.py index a4ceea266b4..c9b1fa2669c 100644 --- a/python/cudf/cudf/core/cut.py +++ b/python/cudf/cudf/core/cut.py @@ -8,7 +8,8 @@ import cudf from cudf.api.types import is_list_like -from cudf.core.column import as_column, build_categorical_column +from cudf.core.column import as_column +from cudf.core.column.categorical import CategoricalColumn, as_unsigned_codes from cudf.core.index import IntervalIndex, interval_range @@ -282,13 +283,17 @@ def cut( # should allow duplicate categories. return interval_labels[index_labels] - col = build_categorical_column( - categories=interval_labels, - codes=index_labels, + index_labels = as_unsigned_codes(len(interval_labels), index_labels) + + col = CategoricalColumn( + data=None, + size=index_labels.size, + dtype=cudf.CategoricalDtype( + categories=interval_labels, ordered=ordered + ), mask=index_labels.base_mask, offset=index_labels.offset, - size=index_labels.size, - ordered=ordered, + children=(index_labels,), ) # we return a categorical index, as we don't have a Categorical method diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 6065e0e1eeb..0d632f4775f 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -48,10 +48,10 @@ ColumnBase, StructColumn, as_column, - build_categorical_column, column_empty, concat_columns, ) +from cudf.core.column.categorical import as_unsigned_codes from cudf.core.column_accessor import ColumnAccessor from cudf.core.copy_types import BooleanMask from cudf.core.groupby.groupby import DataFrameGroupBy, groupby_doc_template @@ -3067,7 +3067,6 @@ def where(self, cond, other=None, inplace=False, axis=None, level=None): from cudf.core._internals.where import ( _check_and_cast_columns_with_other, - _make_categorical_like, ) # First process the condition. @@ -3119,7 +3118,7 @@ def where(self, cond, other=None, inplace=False, axis=None, level=None): out = [] for (name, col), other_col in zip(self._data.items(), other_cols): - col, other_col = _check_and_cast_columns_with_other( + source_col, other_col = _check_and_cast_columns_with_other( source_col=col, other=other_col, inplace=inplace, @@ -3127,16 +3126,16 @@ def where(self, cond, other=None, inplace=False, axis=None, level=None): if cond_col := cond._data.get(name): result = cudf._lib.copying.copy_if_else( - col, other_col, cond_col + source_col, other_col, cond_col ) - out.append(_make_categorical_like(result, self._data[name])) + out.append(result._with_type_metadata(col.dtype)) else: out_mask = cudf._lib.null_mask.create_null_mask( - len(col), + len(source_col), state=cudf._lib.null_mask.MaskState.ALL_NULL, ) - out.append(col.set_mask(out_mask)) + out.append(source_col.set_mask(out_mask)) return self._mimic_inplace( self._from_data_like_self(self._data._from_columns_like_self(out)), @@ -3296,9 +3295,7 @@ def _insert(self, loc, name, value, nan_as_null=None, ignore_index=True): # least require a deprecation cycle because we currently support # inserting a pd.Categorical. if isinstance(value, pd.Categorical): - value = cudf.core.column.categorical.pandas_categorical_as_column( - value - ) + value = as_column(value) if _is_scalar_or_zero_d_array(value): dtype = None @@ -8510,12 +8507,16 @@ def _cast_cols_to_common_dtypes(col_idxs, list_of_columns, dtypes, categories): def _reassign_categories(categories, cols, col_idxs): for name, idx in zip(cols, col_idxs): if idx in categories: - cols[name] = build_categorical_column( - categories=categories[idx], - codes=cols[name], - mask=cols[name].base_mask, - offset=cols[name].offset, - size=cols[name].size, + codes = as_unsigned_codes(len(categories[idx]), cols[name]) + cols[name] = CategoricalColumn( + data=None, + size=codes.size, + dtype=cudf.CategoricalDtype( + categories=categories[idx], ordered=False + ), + mask=codes.base_mask, + offset=codes.offset, + children=(codes,), ) diff --git a/python/cudf/cudf/core/df_protocol.py b/python/cudf/cudf/core/df_protocol.py index a70a42c04af..5250a741d3d 100644 --- a/python/cudf/cudf/core/df_protocol.py +++ b/python/cudf/cudf/core/df_protocol.py @@ -13,7 +13,12 @@ import cudf from cudf.core.buffer import Buffer, as_buffer -from cudf.core.column import as_column, build_categorical_column, build_column +from cudf.core.column import ( + CategoricalColumn, + NumericalColumn, + as_column, + build_column, +) # Implementation of interchange protocol classes # ---------------------------------------------- @@ -830,18 +835,19 @@ def _protocol_to_cudf_column_categorical( assert buffers["data"] is not None, "data buffer should not be None" codes_buffer, codes_dtype = buffers["data"] codes_buffer = _ensure_gpu_buffer(codes_buffer, codes_dtype, allow_copy) - cdtype = protocol_dtype_to_cupy_dtype(codes_dtype) - codes = build_column( - codes_buffer._buf, - cdtype, + cdtype = np.dtype(protocol_dtype_to_cupy_dtype(codes_dtype)) + codes = NumericalColumn( + data=codes_buffer._buf, + size=None, + dtype=cdtype, ) - - cudfcol = build_categorical_column( - categories=categories, - codes=codes, - mask=codes.base_mask, + cudfcol = CategoricalColumn( + data=None, size=codes.size, - ordered=ordered, + dtype=cudf.CategoricalDtype(categories=categories, ordered=ordered), + mask=codes.base_mask, + offset=codes.offset, + children=(codes,), ) return _set_missing_values(col, cudfcol, allow_copy), buffers diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index cbe1e97d834..7b2bc85b13b 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -24,10 +24,10 @@ from cudf.core.column import ( ColumnBase, as_column, - build_categorical_column, deserialize_columns, serialize_columns, ) +from cudf.core.column.categorical import CategoricalColumn, as_unsigned_codes from cudf.core.column_accessor import ColumnAccessor from cudf.core.mixins import BinaryOperand, Scannable from cudf.utils import ioutils @@ -889,18 +889,21 @@ def from_arrow(cls, data: pa.Table) -> Self: for name in dict_dictionaries.keys() } - cudf_category_frame = { - name: build_categorical_column( - cudf_dictionaries_columns[name], - codes, - mask=codes.base_mask, + for name, codes in zip( + dict_indices_table.column_names, indices_columns + ): + categories = cudf_dictionaries_columns[name] + codes = as_unsigned_codes(len(categories), codes) + cudf_category_frame[name] = CategoricalColumn( + data=None, size=codes.size, - ordered=dict_ordered[name], - ) - for name, codes in zip( - dict_indices_table.column_names, indices_columns + dtype=cudf.CategoricalDtype( + categories=categories, + ordered=dict_ordered[name], + ), + mask=codes.base_mask, + children=(codes,), ) - } # Handle non-dict arrays cudf_non_category_frame = { diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 27c6556f976..500fc580097 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3079,22 +3079,8 @@ def __init__( name = _getdefault_name(data, name=name) if isinstance(data, CategoricalColumn): data = data - elif isinstance(data, pd.Series) and ( - isinstance(data.dtype, pd.CategoricalDtype) - ): - codes_data = column.as_column(data.cat.codes.values) - data = column.build_categorical_column( - categories=data.cat.categories, - codes=codes_data, - ordered=data.cat.ordered, - ) - elif isinstance(data, (pd.Categorical, pd.CategoricalIndex)): - codes_data = column.as_column(data.codes) - data = column.build_categorical_column( - categories=data.categories, - codes=codes_data, - ordered=data.ordered, - ) + elif isinstance(getattr(data, "dtype", None), pd.CategoricalDtype): + data = column.as_column(data) else: data = column.as_column( data, dtype="category" if dtype is None else dtype diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index ad6aa56d472..fd6bf37f0e6 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -173,17 +173,7 @@ def _drop_columns(f: Frame, columns: abc.Iterable, errors: str): def _indices_from_labels(obj, labels): if not isinstance(labels, cudf.MultiIndex): labels = cudf.core.column.as_column(labels) - - if isinstance(obj.index.dtype, cudf.CategoricalDtype): - labels = labels.astype("category") - codes = labels.codes.astype(obj.index.codes.dtype) - labels = cudf.core.column.build_categorical_column( - categories=labels.dtype.categories, - codes=codes, - ordered=labels.dtype.ordered, - ) - else: - labels = labels.astype(obj.index.dtype) + labels = labels.astype(obj.index.dtype) idx_labels = cudf.Index._from_column(labels) else: idx_labels = labels diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 4be10752651..a831a798772 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -38,7 +38,9 @@ as_column, ) from cudf.core.column.categorical import ( + _DEFAULT_CATEGORICAL_VALUE, CategoricalAccessor as CategoricalAccessor, + CategoricalColumn, ) from cudf.core.column.column import concat_columns from cudf.core.column.lists import ListMethods @@ -511,9 +513,22 @@ def from_categorical(cls, categorical, codes=None): dtype: category Categories (3, object): ['a', 'b', 'c'] """ # noqa: E501 - col = cudf.core.column.categorical.pandas_categorical_as_column( - categorical, codes=codes - ) + col = as_column(categorical) + if codes is not None: + codes = as_column(codes) + + valid_codes = codes != codes.dtype.type(_DEFAULT_CATEGORICAL_VALUE) + + mask = None + if not valid_codes.all(): + mask = libcudf.transform.bools_to_mask(valid_codes) + col = CategoricalColumn( + data=col.data, + size=codes.size, + dtype=col.dtype, + mask=mask, + children=(codes,), + ) return Series._from_column(col) @classmethod diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index eb6714029cf..55dda34a576 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -350,7 +350,6 @@ def _get_elements_from_column(self, arg) -> ScalarLike | ColumnBase: def where(self, cond, other=None, inplace=False): from cudf.core._internals.where import ( _check_and_cast_columns_with_other, - _make_categorical_like, ) if isinstance(other, cudf.DataFrame): @@ -366,14 +365,12 @@ def where(self, cond, other=None, inplace=False): if not cudf.api.types.is_scalar(other): other = cudf.core.column.as_column(other) - self_column = self._column input_col, other = _check_and_cast_columns_with_other( - source_col=self_column, other=other, inplace=inplace + source_col=self._column, other=other, inplace=inplace ) result = cudf._lib.copying.copy_if_else(input_col, other, cond) - - return _make_categorical_like(result, self_column) + return result._with_type_metadata(self.dtype) @_performance_tracking def transpose(self): diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index d6b2ae2f31c..984115dcbbe 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -20,7 +20,8 @@ import cudf from cudf._lib import parquet as libparquet from cudf.api.types import is_list_like -from cudf.core.column import as_column, build_categorical_column, column_empty +from cudf.core.column import as_column, column_empty +from cudf.core.column.categorical import CategoricalColumn, as_unsigned_codes from cudf.utils import ioutils from cudf.utils.performance_tracking import _performance_tracking @@ -811,12 +812,17 @@ def _parquet_to_frame( partition_categories[name].index(value), length=_len, ) - dfs[-1][name] = build_categorical_column( - categories=partition_categories[name], - codes=codes, + codes = as_unsigned_codes( + len(partition_categories[name]), codes + ) + dfs[-1][name] = CategoricalColumn( + data=None, size=codes.size, + dtype=cudf.CategoricalDtype( + categories=partition_categories[name], ordered=False + ), offset=codes.offset, - ordered=False, + children=(codes,), ) else: # Not building categorical columns, so diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 5bd3eb5fa7f..9347ebba5de 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -64,8 +64,11 @@ def _nonempty_index(idx): values = cudf.core.column.as_column(data) return cudf.DatetimeIndex(values, name=idx.name) elif isinstance(idx, cudf.CategoricalIndex): - values = cudf.core.column.build_categorical_column( - categories=idx.categories, codes=[0, 0], ordered=idx.ordered + values = cudf.core.column.CategoricalColumn( + data=None, + size=None, + dtype=idx.dtype, + children=(cudf.core.column.as_column([0, 0], dtype=np.uint8),), ) return cudf.CategoricalIndex(values, name=idx.name) elif isinstance(idx, cudf.MultiIndex): @@ -105,12 +108,16 @@ def _get_non_empty_data( ) codes = cudf.core.column.as_column( 0, - dtype=cudf._lib.types.size_type_dtype, + dtype=np.uint8, length=2, ) - ordered = s.ordered # type: ignore[attr-defined] - return cudf.core.column.build_categorical_column( - categories=categories, codes=codes, ordered=ordered + return cudf.core.column.CategoricalColumn( + data=None, + size=codes.size, + dtype=cudf.CategoricalDtype( + categories=categories, ordered=s.dtype.ordered + ), + children=(codes,), # type: ignore[arg-type] ) elif isinstance(s.dtype, cudf.ListDtype): leaf_type = s.dtype.leaf_type diff --git a/python/dask_cudf/dask_cudf/io/parquet.py b/python/dask_cudf/dask_cudf/io/parquet.py index c025280c240..e793d4381d1 100644 --- a/python/dask_cudf/dask_cudf/io/parquet.py +++ b/python/dask_cudf/dask_cudf/io/parquet.py @@ -19,7 +19,7 @@ create_metadata_file_dd = None import cudf -from cudf.core.column import as_column, build_categorical_column +from cudf.core.column import CategoricalColumn, as_column from cudf.io import write_to_dataset from cudf.io.parquet import _apply_post_filters, _normalize_filters from cudf.utils.dtypes import cudf_dtype_from_pa_type @@ -163,12 +163,14 @@ def _read_paths( partitions[i].keys.get_loc(index2), length=len(df), ) - df[name] = build_categorical_column( - categories=partitions[i].keys, - codes=codes, + df[name] = CategoricalColumn( + data=None, size=codes.size, + dtype=cudf.CategoricalDtype( + categories=partitions[i].keys, ordered=False + ), offset=codes.offset, - ordered=False, + children=(codes,), ) elif name not in df.columns: # Add non-categorical partition column From 925530afe8178b7e788ea1a8d4df4c0eb4d042dc Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 28 Aug 2024 13:22:18 -0700 Subject: [PATCH 141/270] Relax Arrow pin (#16681) With this change, cudf users can install any version of pyarrow greater than 14. This is the minimum version supporting the C Data Interface, which is a requirement for us (it may be possible to relax in principle, but would require changes to the cudf/pylibcudf code). A few tests are skipped due to bugs or missing features in older versions of pyarrow. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16681 --- .../all_cuda-118_arch-x86_64.yaml | 1 + .../all_cuda-125_arch-x86_64.yaml | 1 + conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/pylibcudf/meta.yaml | 2 +- dependencies.yaml | 12 ++-------- python/cudf/cudf/tests/test_parquet.py | 24 +++++++++++++++---- python/cudf/pyproject.toml | 2 +- python/libcudf/pyproject.toml | 3 --- python/pylibcudf/pylibcudf/interop.pyx | 1 - python/pylibcudf/pyproject.toml | 2 +- 10 files changed, 28 insertions(+), 22 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index c4c32da8af2..7f6967d7287 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -67,6 +67,7 @@ dependencies: - pandoc - pre-commit - ptxcompiler +- pyarrow>=14.0.0,<18.0.0a0 - pydata-sphinx-theme!=0.14.2 - pytest-benchmark - pytest-cases>=3.8.2 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 7439c9543a5..c1315e73f16 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -64,6 +64,7 @@ dependencies: - pandas>=2.0,<2.2.3dev0 - pandoc - pre-commit +- pyarrow>=14.0.0,<18.0.0a0 - pydata-sphinx-theme!=0.14.2 - pynvjitlink>=0.0.0a0 - pytest-benchmark diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 53f52a35651..e22b4a4eddc 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -82,7 +82,7 @@ requirements: - cupy >=12.0.0 - numba >=0.57 - numpy >=1.23,<3.0a0 - - pyarrow ==16.1.0.* + - pyarrow>=14.0.0,<18.0.0a0 - libcudf ={{ version }} - pylibcudf ={{ version }} - {{ pin_compatible('rmm', max_pin='x.x') }} diff --git a/conda/recipes/pylibcudf/meta.yaml b/conda/recipes/pylibcudf/meta.yaml index 67b9b76bb8c..7c1efa0176c 100644 --- a/conda/recipes/pylibcudf/meta.yaml +++ b/conda/recipes/pylibcudf/meta.yaml @@ -79,7 +79,7 @@ requirements: - typing_extensions >=4.0.0 - pandas >=2.0,<2.2.3dev0 - numpy >=1.23,<3.0a0 - - pyarrow ==16.1.0.* + - pyarrow>=14.0.0,<18.0.0a0 - {{ pin_compatible('rmm', max_pin='x.x') }} - fsspec >=0.6.0 {% if cuda_major == "11" %} diff --git a/dependencies.yaml b/dependencies.yaml index 5be291b3671..c6851d9cb90 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -19,6 +19,7 @@ files: - docs - notebooks - py_version + - pyarrow_run - rapids_build_skbuild - rapids_build_setuptools - run_common @@ -46,7 +47,6 @@ files: includes: - cuda_version - py_version - - pyarrow_run - test_python_common - test_python_cudf - test_python_dask_cudf @@ -136,13 +136,6 @@ files: - build_base - build_cpp - depends_on_librmm - py_run_libcudf: - output: pyproject - pyproject_dir: python/libcudf - extras: - table: project - includes: - - pyarrow_run py_build_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf @@ -390,8 +383,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - # Allow runtime version to float up to patch version - - pyarrow>=16.1.0,<16.2.0a0 + - pyarrow>=14.0.0,<18.0.0a0 cuda_version: specific: - output_types: conda diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index db4f1c9c8bd..879b2bd3d74 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -515,10 +515,6 @@ def test_parquet_read_filtered_multiple_files(tmpdir): ) -@pytest.mark.skipif( - version.parse(pa.__version__) < version.parse("1.0.1"), - reason="pyarrow 1.0.0 needed for various operators and operand types", -) @pytest.mark.parametrize( "predicate,expected_len", [ @@ -2393,6 +2389,10 @@ def test_parquet_writer_list_large_mixed(tmpdir): @pytest.mark.parametrize("store_schema", [True, False]) def test_parquet_writer_list_chunked(tmpdir, store_schema): + if store_schema and version.parse(pa.__version__) < version.parse( + "15.0.0" + ): + pytest.skip("https://github.com/apache/arrow/pull/37792") table1 = cudf.DataFrame( { "a": list_gen(string_gen, 128, 80, 50), @@ -2578,6 +2578,10 @@ def normalized_equals(value1, value2): @pytest.mark.parametrize("add_nulls", [True, False]) @pytest.mark.parametrize("store_schema", [True, False]) def test_parquet_writer_statistics(tmpdir, pdf, add_nulls, store_schema): + if store_schema and version.parse(pa.__version__) < version.parse( + "15.0.0" + ): + pytest.skip("https://github.com/apache/arrow/pull/37792") file_path = tmpdir.join("cudf.parquet") if "col_category" in pdf.columns: pdf = pdf.drop(columns=["col_category", "col_bool"]) @@ -2957,6 +2961,10 @@ def test_per_column_options_string_col(tmpdir, encoding): assert encoding in fmd.row_group(0).column(0).encodings +@pytest.mark.skipif( + version.parse(pa.__version__) < version.parse("16.0.0"), + reason="https://github.com/apache/arrow/pull/39748", +) @pytest.mark.parametrize( "num_rows", [200, 10000], @@ -3557,6 +3565,10 @@ def test_parquet_reader_roundtrip_structs_with_arrow_schema(tmpdir, data): @pytest.mark.parametrize("index", [None, True, False]) +@pytest.mark.skipif( + version.parse(pa.__version__) < version.parse("15.0.0"), + reason="https://github.com/apache/arrow/pull/37792", +) def test_parquet_writer_roundtrip_with_arrow_schema(index): # Ensure that the concrete and nested types are faithfully being roundtripped # across Parquet with arrow schema @@ -3707,6 +3719,10 @@ def test_parquet_writer_int96_timestamps_and_arrow_schema(): ], ) @pytest.mark.parametrize("index", [None, True, False]) +@pytest.mark.skipif( + version.parse(pa.__version__) < version.parse("15.0.0"), + reason="https://github.com/apache/arrow/pull/37792", +) def test_parquet_writer_roundtrip_structs_with_arrow_schema( tmpdir, data, index ): diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 0c1d5015078..17d1292980b 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "packaging", "pandas>=2.0,<2.2.3dev0", "ptxcompiler", - "pyarrow>=16.1.0,<16.2.0a0", + "pyarrow>=14.0.0,<18.0.0a0", "pylibcudf==24.10.*,>=0.0.0a0", "rich", "rmm==24.10.*,>=0.0.0a0", diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 43878d0aec2..5f4b9957fd0 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -37,9 +37,6 @@ classifiers = [ "Programming Language :: C++", "Environment :: GPU :: NVIDIA CUDA", ] -dependencies = [ - "pyarrow>=16.1.0,<16.2.0a0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/cudf" diff --git a/python/pylibcudf/pylibcudf/interop.pyx b/python/pylibcudf/pylibcudf/interop.pyx index d54e5b7ba1f..1a03fa5b45b 100644 --- a/python/pylibcudf/pylibcudf/interop.pyx +++ b/python/pylibcudf/pylibcudf/interop.pyx @@ -152,7 +152,6 @@ def _from_arrow_scalar(pyarrow_object, *, DataType data_type=None): @from_arrow.register(pa.Array) -@from_arrow.register(pa.ChunkedArray) def _from_arrow_column(pyarrow_object, *, DataType data_type=None): if data_type is not None: raise ValueError("data_type may not be passed for arrays") diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index e4c6edc6141..bfade41353c 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "libcudf==24.10.*,>=0.0.0a0", "nvtx>=0.2.1", "packaging", - "pyarrow>=16.1.0,<16.2.0a0", + "pyarrow>=14.0.0,<18.0.0a0", "rmm==24.10.*,>=0.0.0a0", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From fbd61142a47bd9ef6f739f97c81c88c1ca9430d4 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:06:10 -0700 Subject: [PATCH 142/270] Support reading matching projected and filter cols from Parquet files with otherwise mismatched schemas (#16394) Closes #16269. This PR adds support to read (matching) projected/selected and filter columns from Parquet files with otherwise mismatching schemas. ### Solution Description We create a `std::vector>`, one per file except 0th file. We then co-walk schema trees and populate the map with corresponding (one-to-one mapped) `schema_idx` of valid selected (projection and filter) column between 0th and the rest of the files. The same `unordered_map` is used to get the `schema_idx` of the same columns across files when creating `ColumnChunkDesc` and copying column chunk metadata into the page decoder. ### Known Limitation - [x] Nullability across files: Each selected column must still be either nullable or non-nullable across all files. See #12702 also described in [#dask/9935](https://github.com/dask/dask/pull/9935) CC @wence- Authors: - Muhammad Haseeb (https://github.com/mhaseeb123) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Lawrence Mitchell (https://github.com/wence-) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/16394 --- cpp/include/cudf/io/parquet.hpp | 37 +++ cpp/src/io/parquet/reader_impl.cpp | 13 +- cpp/src/io/parquet/reader_impl_helpers.cpp | 174 ++++++++++-- cpp/src/io/parquet/reader_impl_helpers.hpp | 53 +++- python/cudf/cudf/_lib/parquet.pyx | 10 +- python/cudf/cudf/io/parquet.py | 5 + python/cudf/cudf/tests/test_parquet.py | 248 ++++++++++++++++++ python/cudf/cudf/utils/ioutils.py | 3 + python/pylibcudf/pylibcudf/io/parquet.pxd | 1 + python/pylibcudf/pylibcudf/io/parquet.pyx | 14 +- .../pylibcudf/libcudf/io/parquet.pxd | 6 +- 11 files changed, 534 insertions(+), 30 deletions(-) diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index 12897ac77ef..64c37f9a9df 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -76,6 +76,8 @@ class parquet_reader_options { bool _use_pandas_metadata = true; // Whether to read and use ARROW schema bool _use_arrow_schema = true; + // Whether to allow reading matching select columns from mismatched Parquet files. + bool _allow_mismatched_pq_schemas = false; // Cast timestamp columns to a specific type data_type _timestamp_type{type_id::EMPTY}; @@ -138,6 +140,18 @@ class parquet_reader_options { */ [[nodiscard]] bool is_enabled_use_arrow_schema() const { return _use_arrow_schema; } + /** + * @brief Returns true/false depending on whether to read matching projected and filter columns + * from mismatched Parquet sources. + * + * @return `true` if mismatched projected and filter columns will be read from mismatched Parquet + * sources. + */ + [[nodiscard]] bool is_enabled_allow_mismatched_pq_schemas() const + { + return _allow_mismatched_pq_schemas; + } + /** * @brief Returns optional tree of metadata. * @@ -258,6 +272,15 @@ class parquet_reader_options { */ void enable_use_arrow_schema(bool val) { _use_arrow_schema = val; } + /** + * @brief Sets to enable/disable reading of matching projected and filter columns from mismatched + * Parquet sources. + * + * @param val Boolean value whether to read matching projected and filter columns from mismatched + * Parquet sources. + */ + void enable_allow_mismatched_pq_schemas(bool val) { _allow_mismatched_pq_schemas = val; } + /** * @brief Sets reader column schema. * @@ -382,6 +405,20 @@ class parquet_reader_options_builder { return *this; } + /** + * @brief Sets to enable/disable reading of matching projected and filter columns from mismatched + * Parquet sources. + * + * @param val Boolean value whether to read matching projected and filter columns from mismatched + * Parquet sources. + * @return this for chaining. + */ + parquet_reader_options_builder& allow_mismatched_pq_schemas(bool val) + { + options._allow_mismatched_pq_schemas = val; + return *this; + } + /** * @brief Sets reader metadata. * diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 2648a1f41ab..9950e2f7d7d 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -470,8 +470,10 @@ reader::impl::impl(std::size_t chunk_read_limit, _input_pass_read_limit{pass_read_limit} { // Open and parse the source dataset metadata - _metadata = - std::make_unique(_sources, options.is_enabled_use_arrow_schema()); + _metadata = std::make_unique( + _sources, + options.is_enabled_use_arrow_schema(), + options.get_columns().has_value() and options.is_enabled_allow_mismatched_pq_schemas()); // Strings may be returned as either string or categorical columns _strings_to_categorical = options.is_enabled_convert_strings_to_categories(); @@ -769,11 +771,14 @@ parquet_column_schema walk_schema(aggregate_reader_metadata const* mt, int idx) parquet_metadata read_parquet_metadata(host_span const> sources) { - // do not use arrow schema when reading information from parquet metadata. + // Do not use arrow schema when reading information from parquet metadata. static constexpr auto use_arrow_schema = false; + // Do not select any columns when only reading the parquet metadata. + static constexpr auto has_column_projection = false; + // Open and parse the source dataset metadata - auto metadata = aggregate_reader_metadata(sources, use_arrow_schema); + auto metadata = aggregate_reader_metadata(sources, use_arrow_schema, has_column_projection); return parquet_metadata{parquet_schema{walk_schema(&metadata, 0)}, metadata.get_num_rows(), diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 00f75e4e828..8b5678f202b 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -380,6 +380,17 @@ aggregate_reader_metadata::collect_keyval_metadata() const return kv_maps; } +std::vector> aggregate_reader_metadata::init_schema_idx_maps( + bool const has_cols_from_mismatched_srcs) const +{ + // Only initialize if more than 1 data sources and has select columns from mismatched data sources + if (has_cols_from_mismatched_srcs and per_file_metadata.size() > 1) { + return std::vector>{per_file_metadata.size() - 1}; + } + + return {}; +} + int64_t aggregate_reader_metadata::calc_num_rows() const { return std::accumulate( @@ -539,13 +550,18 @@ void aggregate_reader_metadata::column_info_for_row_group(row_group_info& rg_inf } aggregate_reader_metadata::aggregate_reader_metadata( - host_span const> sources, bool use_arrow_schema) + host_span const> sources, + bool use_arrow_schema, + bool has_cols_from_mismatched_srcs) : per_file_metadata(metadatas_from_sources(sources)), keyval_maps(collect_keyval_metadata()), + schema_idx_maps(init_schema_idx_maps(has_cols_from_mismatched_srcs)), num_rows(calc_num_rows()), num_row_groups(calc_num_row_groups()) { - if (per_file_metadata.size() > 0) { + // Validate that all sources have the same schema unless we are reading select columns + // from mismatched sources, in which case, we will only check the projected columns later. + if (per_file_metadata.size() > 1 and not has_cols_from_mismatched_srcs) { auto const& first_meta = per_file_metadata.front(); auto const num_cols = first_meta.row_groups.size() > 0 ? first_meta.row_groups.front().columns.size() : 0; @@ -632,7 +648,7 @@ arrow_schema_data_types aggregate_reader_metadata::collect_arrow_schema() const if (field->type_type() == flatbuf::Type::Type_Duration) { auto type_data = field->type_as_Duration(); if (type_data != nullptr) { - auto name = (field->name()) ? field->name()->str() : ""; + auto name = field->name() ? field->name()->str() : ""; // set the schema_elem type to duration type schema_elem.type = duration_from_flatbuffer(type_data); arrow_type_col_seen |= (schema_elem.type.id() != type_id::EMPTY); @@ -868,12 +884,23 @@ ColumnChunkMetaData const& aggregate_reader_metadata::get_column_metadata(size_t size_type src_idx, int schema_idx) const { + // schema_idx_maps will only have > 0 size when we are reading matching column projection from + // mismatched Parquet sources. + if (src_idx and not schema_idx_maps.empty()) { + auto const& schema_idx_map = schema_idx_maps[src_idx - 1]; + CUDF_EXPECTS(schema_idx_map.find(schema_idx) != schema_idx_map.end(), + "Unmapped schema index encountered in the specified source tree", + std::range_error); + schema_idx = schema_idx_map.at(schema_idx); + } + auto col = std::find_if(per_file_metadata[src_idx].row_groups[row_group_index].columns.begin(), per_file_metadata[src_idx].row_groups[row_group_index].columns.end(), [schema_idx](ColumnChunk const& col) { return col.schema_idx == schema_idx; }); CUDF_EXPECTS(col != std::end(per_file_metadata[src_idx].row_groups[row_group_index].columns), - "Found no metadata for schema index"); + "Found no metadata for schema index", + std::range_error); return col->meta_data; } @@ -1041,18 +1068,19 @@ aggregate_reader_metadata::select_columns( std::optional> const& filter_columns_names, bool include_index, bool strings_to_categorical, - type_id timestamp_type_id) const + type_id timestamp_type_id) { - auto find_schema_child = [&](SchemaElement const& schema_elem, std::string const& name) { - auto const& col_schema_idx = - std::find_if(schema_elem.children_idx.cbegin(), - schema_elem.children_idx.cend(), - [&](size_t col_schema_idx) { return get_schema(col_schema_idx).name == name; }); - - return (col_schema_idx != schema_elem.children_idx.end()) - ? static_cast(*col_schema_idx) - : -1; - }; + auto const find_schema_child = + [&](SchemaElement const& schema_elem, std::string const& name, int const pfm_idx = 0) { + auto const& col_schema_idx = std::find_if( + schema_elem.children_idx.cbegin(), + schema_elem.children_idx.cend(), + [&](size_t col_schema_idx) { return get_schema(col_schema_idx, pfm_idx).name == name; }); + + return (col_schema_idx != schema_elem.children_idx.end()) + ? static_cast(*col_schema_idx) + : -1; + }; std::vector output_columns; std::vector input_columns; @@ -1074,7 +1102,7 @@ aggregate_reader_metadata::select_columns( if (schema_elem.is_stub()) { // is this legit? CUDF_EXPECTS(schema_elem.num_children == 1, "Unexpected number of children for stub"); - auto child_col_name_info = (col_name_info) ? &col_name_info->children[0] : nullptr; + auto const child_col_name_info = col_name_info ? &col_name_info->children[0] : nullptr; return build_column( child_col_name_info, schema_elem.children_idx[0], out_col_array, has_list_parent); } @@ -1154,6 +1182,97 @@ aggregate_reader_metadata::select_columns( return path_is_valid; }; + // Compares two schema elements to be equal except their number of children + auto const equal_to_except_num_children = [](SchemaElement const& lhs, SchemaElement const& rhs) { + return lhs.type == rhs.type and lhs.converted_type == rhs.converted_type and + lhs.type_length == rhs.type_length and lhs.repetition_type == rhs.repetition_type and + lhs.name == rhs.name and lhs.decimal_scale == rhs.decimal_scale and + lhs.decimal_precision == rhs.decimal_precision and lhs.field_id == rhs.field_id; + }; + + // Maps a projected column's schema_idx in the zeroth per_file_metadata (source) to the + // corresponding schema_idx in pfm_idx'th per_file_metadata (destination). The projected + // column's path must match across sources, else an appropriate exception is thrown. + std::function map_column = + [&](column_name_info const* col_name_info, + int const src_schema_idx, + int const dst_schema_idx, + int const pfm_idx) { + auto const& src_schema_elem = get_schema(src_schema_idx); + auto const& dst_schema_elem = get_schema(dst_schema_idx, pfm_idx); + + // Check the schema elements to be equal except their number of children as we only care about + // the specific column paths in the schema trees. Raise an invalid_argument error if the + // schema elements don't match. + CUDF_EXPECTS(equal_to_except_num_children(src_schema_elem, dst_schema_elem), + "Encountered mismatching SchemaElement properties for a column in " + "the selected path", + std::invalid_argument); + + // If src_schema_elem is a stub, it does not exist in the column_name_info and column_buffer + // hierarchy. So continue on with mapping. + if (src_schema_elem.is_stub()) { + // Check if dst_schema_elem is also a stub i.e. has num_children == 1 that we didn't + // previously check. Raise an invalid_argument error if dst_schema_elem is not a stub. + CUDF_EXPECTS(dst_schema_elem.is_stub(), + "Encountered mismatching schemas for stub.", + std::invalid_argument); + auto const child_col_name_info = col_name_info ? &col_name_info->children[0] : nullptr; + return map_column(child_col_name_info, + src_schema_elem.children_idx[0], + dst_schema_elem.children_idx[0], + pfm_idx); + } + + // The path ends here. If this is a list/struct col (has children), then map all its children + // which must be identical. + if (col_name_info == nullptr or col_name_info->children.empty()) { + // Check the number of children to be equal to be mapped. An out_of_range error if the + // number of children isn't equal. + CUDF_EXPECTS(src_schema_elem.num_children == dst_schema_elem.num_children, + "Encountered mismatching number of children for a " + "column in the selected path", + std::out_of_range); + + std::for_each(thrust::make_counting_iterator(0), + thrust::make_counting_iterator(src_schema_elem.num_children), + [&](auto const child_idx) { + map_column(nullptr, + src_schema_elem.children_idx[child_idx], + dst_schema_elem.children_idx[child_idx], + pfm_idx); + }); + } + // The path goes further down to specific child(ren) of this column so map only those + // children. + else { + std::for_each( + col_name_info->children.cbegin(), + col_name_info->children.cend(), + [&](auto const& child_col_name_info) { + // Ensure that each named child column exists in the destination schema tree for the + // paths to align up. An out_of_range error otherwise. + CUDF_EXPECTS( + find_schema_child(dst_schema_elem, child_col_name_info.name, pfm_idx) != -1, + "Encountered mismatching schema tree depths across data sources", + std::out_of_range); + map_column(&child_col_name_info, + find_schema_child(src_schema_elem, child_col_name_info.name), + find_schema_child(dst_schema_elem, child_col_name_info.name, pfm_idx), + pfm_idx); + }); + } + + // We're at a leaf and this is an input column (one with actual data stored) so map it. + if (src_schema_elem.num_children == 0) { + // Get the schema_idx_map for this data source (pfm) + auto& schema_idx_map = schema_idx_maps[pfm_idx - 1]; + + // Map the schema index from 0th tree (src) to the one in the current (dst) tree. + schema_idx_map[src_schema_idx] = dst_schema_idx; + } + }; + std::vector output_column_schemas; // @@ -1287,7 +1406,28 @@ aggregate_reader_metadata::select_columns( for (auto& col : selected_columns) { auto const& top_level_col_schema_idx = find_schema_child(root, col.name); bool valid_column = build_column(&col, top_level_col_schema_idx, output_columns, false); - if (valid_column) output_column_schemas.push_back(top_level_col_schema_idx); + if (valid_column) { + output_column_schemas.push_back(top_level_col_schema_idx); + + // Map the column's schema_idx across the rest of the data sources if required. + if (per_file_metadata.size() > 1 and not schema_idx_maps.empty()) { + std::for_each(thrust::make_counting_iterator(static_cast(1)), + thrust::make_counting_iterator(per_file_metadata.size()), + [&](auto const pfm_idx) { + auto const& dst_root = get_schema(0, pfm_idx); + // Ensure that each top level column exists in the destination schema + // tree. An out_of_range error is thrown otherwise. + CUDF_EXPECTS( + find_schema_child(dst_root, col.name, pfm_idx) != -1, + "Encountered mismatching schema tree depths across data sources", + std::out_of_range); + map_column(&col, + top_level_col_schema_idx, + find_schema_child(dst_root, col.name, pfm_idx), + pfm_idx); + }); + } + } } } diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index 309132a5347..6f2863136b2 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -128,6 +128,7 @@ struct arrow_schema_data_types { class aggregate_reader_metadata { std::vector per_file_metadata; std::vector> keyval_maps; + std::vector> schema_idx_maps; int64_t num_rows; size_type num_row_groups; @@ -144,6 +145,19 @@ class aggregate_reader_metadata { [[nodiscard]] std::vector> collect_keyval_metadata() const; + /** + * @brief Initialize the vector of schema_idx maps. + * + * Initializes a vector of hash maps that will store the one-to-one mappings between the + * schema_idx'es of the selected columns in the zeroth per_file_metadata (source) and each + * kth per_file_metadata (destination) for k in range: [1, per_file_metadata.size()-1]. + * + * @param has_cols_from_mismatched_srcs True if we are reading select cols from mismatched + * parquet schemas. + */ + [[nodiscard]] std::vector> init_schema_idx_maps( + bool has_cols_from_mismatched_srcs) const; + /** * @brief Decodes and constructs the arrow schema from the ARROW_SCHEMA_KEY IPC message * in key value metadata section of Parquet file footer @@ -183,10 +197,28 @@ class aggregate_reader_metadata { public: aggregate_reader_metadata(host_span const> sources, - bool use_arrow_schema); + bool use_arrow_schema, + bool has_cols_from_mismatched_srcs); [[nodiscard]] RowGroup const& get_row_group(size_type row_group_index, size_type src_idx) const; + /** + * @brief Extracts the schema_idx'th column chunk metadata from row_group_index'th row group of + * the src_idx'th file. + * + * Extracts the schema_idx'th column chunk metadata from the specified row group index of the + * src_idx'th file. Note that the schema_idx is actually the index in the zeroth file which may + * not be the same in all files, in which case, the schema_idx is mapped to the corresponding + * index in the src_idx'th file and returned. A range_error error is thrown if schema_idx + * doesn't exist or isn't mapped to the src_idx file. + * + * @param row_group_index The row group index in the file to extract column chunk metadata from. + * @param src_idx The per_file_metadata index to extract extract column chunk metadata from. + * @param schema_idx The schema_idx of the column chunk to be extracted + * + * @return The requested column chunk metadata or a range_error error if the schema index isn't + * valid. + */ [[nodiscard]] ColumnChunkMetaData const& get_column_metadata(size_type row_group_index, size_type src_idx, int schema_idx) const; @@ -202,9 +234,22 @@ class aggregate_reader_metadata { [[nodiscard]] auto get_num_row_groups() const { return num_row_groups; } - [[nodiscard]] auto const& get_schema(int schema_idx) const + /** + * @brief Extracts the schema_idx'th SchemaElement from the pfm_idx'th file + * + * @param schema_idx The index of the SchemaElement to be extracted. + * @param pfm_idx The index of the per_file_metadata to extract SchemaElement from, default = 0 if + * not specified. + * + * @return The requested SchemaElement or an error if invalid schema_idx or pfm_idx. + */ + [[nodiscard]] auto const& get_schema(int schema_idx, int pfm_idx = 0) const { - return per_file_metadata[0].schema[schema_idx]; + CUDF_EXPECTS( + schema_idx >= 0 and pfm_idx >= 0 and pfm_idx < static_cast(per_file_metadata.size()), + "Parquet reader encountered an invalid schema_idx or pfm_idx", + std::invalid_argument); + return per_file_metadata[pfm_idx].schema[schema_idx]; } [[nodiscard]] auto const& get_key_value_metadata() const& { return keyval_maps; } @@ -314,7 +359,7 @@ class aggregate_reader_metadata { std::optional> const& filter_columns_names, bool include_index, bool strings_to_categorical, - type_id timestamp_type_id) const; + type_id timestamp_type_id); }; /** diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index c874a51e220..a0155671a26 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -266,7 +266,8 @@ def read_parquet_chunked( size_t chunk_read_limit=0, size_t pass_read_limit=1024000000, size_type nrows=-1, - int64_t skip_rows=0 + int64_t skip_rows=0, + allow_mismatched_pq_schemas=False ): # Note: If this function ever takes accepts filters # allow_range_index needs to be False when a filter is passed @@ -277,11 +278,12 @@ def read_parquet_chunked( plc.io.SourceInfo(filepaths_or_buffers), columns, row_groups, - use_pandas_metadata, + use_pandas_metadata=use_pandas_metadata, chunk_read_limit=chunk_read_limit, pass_read_limit=pass_read_limit, skip_rows=skip_rows, nrows=nrows, + allow_mismatched_pq_schemas=allow_mismatched_pq_schemas, ) tbl_w_meta = reader.read_chunk() @@ -323,7 +325,8 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, use_pandas_metadata=True, Expression filters=None, size_type nrows=-1, - int64_t skip_rows=0): + int64_t skip_rows=0, + allow_mismatched_pq_schemas=False): """ Cython function to call into libcudf API, see `read_parquet`. @@ -351,6 +354,7 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, use_pandas_metadata = use_pandas_metadata, skip_rows = skip_rows, nrows = nrows, + allow_mismatched_pq_schemas=allow_mismatched_pq_schemas, ) df = cudf.DataFrame._from_data( diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 984115dcbbe..526f12aa94e 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -514,6 +514,7 @@ def read_parquet( dataset_kwargs=None, nrows=None, skip_rows=None, + allow_mismatched_pq_schemas=False, *args, **kwargs, ): @@ -622,6 +623,7 @@ def read_parquet( dataset_kwargs=dataset_kwargs, nrows=nrows, skip_rows=skip_rows, + allow_mismatched_pq_schemas=allow_mismatched_pq_schemas, **kwargs, ) # Apply filters row-wise (if any are defined), and return @@ -865,6 +867,7 @@ def _read_parquet( use_pandas_metadata=None, nrows=None, skip_rows=None, + allow_mismatched_pq_schemas=False, *args, **kwargs, ): @@ -889,6 +892,7 @@ def _read_parquet( use_pandas_metadata=use_pandas_metadata, nrows=nrows if nrows is not None else -1, skip_rows=skip_rows if skip_rows is not None else 0, + allow_mismatched_pq_schemas=allow_mismatched_pq_schemas, ) else: if nrows is None: @@ -902,6 +906,7 @@ def _read_parquet( use_pandas_metadata=use_pandas_metadata, nrows=nrows, skip_rows=skip_rows, + allow_mismatched_pq_schemas=allow_mismatched_pq_schemas, ) else: if ( diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 879b2bd3d74..6623c537ddf 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -3809,3 +3809,251 @@ def test_parquet_reader_pandas_compatibility(): with cudf.option_context("io.parquet.low_memory", True): expected = cudf.read_parquet(buffer) assert_eq(expected, df) + + +@pytest.mark.parametrize("store_schema", [True, False]) +def test_parquet_reader_with_mismatched_tables(store_schema): + # cuDF tables with mixed types + df1 = cudf.DataFrame( + { + "i32": cudf.Series([None, None, None], dtype="int32"), + "i64": cudf.Series([1234, None, 123], dtype="int64"), + "list": list([[1, 2], [None, 4], [5, 6]]), + "time": cudf.Series([1234, 123, 4123], dtype="datetime64[ms]"), + "str": ["vfd", None, "ghu"], + "d_list": list( + [ + [pd.Timedelta(minutes=1), pd.Timedelta(minutes=2)], + [None, pd.Timedelta(minutes=3)], + [pd.Timedelta(minutes=8), None], + ] + ), + } + ) + + df2 = cudf.DataFrame( + { + "str": ["abc", "def", None], + "i64": cudf.Series([None, 65, 98], dtype="int64"), + "times": cudf.Series([1234, None, 4123], dtype="datetime64[us]"), + "list": list([[7, 8], [9, 10], [None, 12]]), + "d_list": list( + [ + [pd.Timedelta(minutes=4), None], + [None, None], + [pd.Timedelta(minutes=6), None], + ] + ), + } + ) + + # IO buffers + buf1 = BytesIO() + buf2 = BytesIO() + + # Write Parquet with and without arrow schema + df1.to_parquet(buf1, store_schema=store_schema) + df2.to_parquet(buf2, store_schema=store_schema) + + # Read mismatched Parquet files + got = cudf.read_parquet( + [buf1, buf2], + columns=["list", "d_list", "str"], + filters=[("i64", ">", 20)], + allow_mismatched_pq_schemas=True, + ) + + # Construct the expected table + expected = cudf.concat( + [ + df1[df1["i64"] > 20][["list", "d_list", "str"]], + df2[df2["i64"] > 20][["list", "d_list", "str"]], + ] + ).reset_index(drop=True) + + # Read with chunked reader (filter columns not supported) + got_chunked = read_parquet_chunked( + [buf1, buf2], + columns=["list", "d_list", "str"], + chunk_read_limit=240, + pass_read_limit=240, + allow_mismatched_pq_schemas=True, + ) + + # Construct the expected table without filter columns + expected_chunked = cudf.concat( + [df1[["list", "d_list", "str"]], df2[["list", "d_list", "str"]]] + ).reset_index(drop=True) + + # Check results + assert_eq(expected, got) + assert_eq(expected_chunked, got_chunked) + + +def test_parquet_reader_with_mismatched_structs(): + data1 = [ + { + "a": 1, + "b": { + "inner_a": 10, + "inner_b": {"inner_inner_b": 1, "inner_inner_a": 2}, + }, + "c": 2, + }, + { + "a": 3, + "b": {"inner_a": 30, "inner_b": {"inner_inner_a": 210}}, + "c": 4, + }, + {"a": 5, "b": {"inner_a": 50, "inner_b": None}, "c": 6}, + {"a": 7, "b": None, "c": 8}, + {"a": None, "b": {"inner_a": None, "inner_b": None}, "c": None}, + None, + { + "a": None, + "b": { + "inner_a": None, + "inner_b": {"inner_inner_b": None, "inner_inner_a": 10}, + }, + "c": 10, + }, + ] + + data2 = [ + {"a": 1, "b": {"inner_b": {"inner_inner_a": None}}}, + {"a": 3, "b": {"inner_b": {"inner_inner_a": 1}}}, + {"a": 5, "b": {"inner_b": None}}, + {"a": 7, "b": {"inner_b": {"inner_inner_b": 1, "inner_inner_a": 0}}}, + {"a": None, "b": {"inner_b": None}}, + None, + {"a": None, "b": {"inner_b": {"inner_inner_a": 1}}}, + ] + + # cuDF tables from struct data + df1 = cudf.DataFrame.from_arrow(pa.Table.from_pydict({"struct": data1})) + df2 = cudf.DataFrame.from_arrow(pa.Table.from_pydict({"struct": data2})) + + # Buffers + buf1 = BytesIO() + buf2 = BytesIO() + + # Write to parquet + df1.to_parquet(buf1) + df2.to_parquet(buf2) + + # Read the struct.b.inner_b.inner_inner_a column from parquet + got = cudf.read_parquet( + [buf1, buf2], + columns=["struct.b.inner_b.inner_inner_a"], + allow_mismatched_pq_schemas=True, + ) + got = ( + cudf.Series(got["struct"]) + .struct.field("b") + .struct.field("inner_b") + .struct.field("inner_inner_a") + ) + + # Read with chunked reader + got_chunked = read_parquet_chunked( + [buf1, buf2], + columns=["struct.b.inner_b.inner_inner_a"], + chunk_read_limit=240, + pass_read_limit=240, + allow_mismatched_pq_schemas=True, + ) + got_chunked = ( + cudf.Series(got_chunked["struct"]) + .struct.field("b") + .struct.field("inner_b") + .struct.field("inner_inner_a") + ) + + # Construct the expected series + expected = cudf.concat( + [ + cudf.Series(df1["struct"]) + .struct.field("b") + .struct.field("inner_b") + .struct.field("inner_inner_a"), + cudf.Series(df2["struct"]) + .struct.field("b") + .struct.field("inner_b") + .struct.field("inner_inner_a"), + ] + ).reset_index(drop=True) + + # Check results + assert_eq(expected, got) + assert_eq(expected, got_chunked) + + +def test_parquet_reader_with_mismatched_schemas_error(): + df1 = cudf.DataFrame( + { + "millis": cudf.Series([123, 3454, 123], dtype="timedelta64[ms]"), + "i64": cudf.Series([123, 3454, 123], dtype="int64"), + "i32": cudf.Series([123, 3454, 123], dtype="int32"), + } + ) + df2 = cudf.DataFrame( + { + "i64": cudf.Series([123, 3454, 123], dtype="int64"), + "millis": cudf.Series([123, 3454, 123], dtype="timedelta64[ms]"), + } + ) + + buf1 = BytesIO() + buf2 = BytesIO() + + df1.to_parquet(buf1, store_schema=True) + df2.to_parquet(buf2, store_schema=False) + + with pytest.raises( + ValueError, + match="Encountered mismatching SchemaElement properties for a column in the selected path", + ): + cudf.read_parquet( + [buf1, buf2], columns=["millis"], allow_mismatched_pq_schemas=True + ) + + data1 = [ + {"a": 1, "b": {"inner_a": 1, "inner_b": 6}}, + {"a": 3, "b": {"inner_a": None, "inner_b": 2}}, + ] + data2 = [ + {"b": {"inner_a": 1}, "c": "str"}, + {"b": {"inner_a": None}, "c": None}, + ] + + # cuDF tables from struct data + df1 = cudf.DataFrame.from_arrow(pa.Table.from_pydict({"struct": data1})) + df2 = cudf.DataFrame.from_arrow(pa.Table.from_pydict({"struct": data2})) + + # Buffers + buf1 = BytesIO() + buf2 = BytesIO() + + # Write to parquet + df1.to_parquet(buf1) + df2.to_parquet(buf2) + + with pytest.raises( + IndexError, + match="Encountered mismatching number of children for a column in the selected path", + ): + cudf.read_parquet( + [buf1, buf2], + columns=["struct.b"], + allow_mismatched_pq_schemas=True, + ) + + with pytest.raises( + IndexError, + match="Encountered mismatching schema tree depths across data sources", + ): + cudf.read_parquet( + [buf1, buf2], + columns=["struct.b.inner_b"], + allow_mismatched_pq_schemas=True, + ) diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 94974e595b1..6b146be0fa3 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -184,6 +184,9 @@ .. note: This option is not supported when the low-memory mode is on. +allow_mismatched_pq_schemas : boolean, default False + If True, enables reading (matching) columns specified in `columns` and `filters` + options from the input files with otherwise mismatched schemas. Returns ------- diff --git a/python/pylibcudf/pylibcudf/io/parquet.pxd b/python/pylibcudf/pylibcudf/io/parquet.pxd index 47458b00159..9c476030ded 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pxd +++ b/python/pylibcudf/pylibcudf/io/parquet.pxd @@ -28,6 +28,7 @@ cpdef read_parquet( bool use_pandas_metadata = *, int64_t skip_rows = *, size_type nrows = *, + bool allow_mismatched_pq_schemas = *, # disabled see comment in parquet.pyx for more # ReaderColumnSchema reader_column_schema = *, # DataType timestamp_type = * diff --git a/python/pylibcudf/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx index fb5244a2a9e..df1f1b14247 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -26,6 +26,7 @@ cdef parquet_reader_options _setup_parquet_reader_options( bool use_pandas_metadata = True, int64_t skip_rows = 0, size_type nrows = -1, + bool allow_mismatched_pq_schemas=False, # ReaderColumnSchema reader_column_schema = None, # DataType timestamp_type = DataType(type_id.EMPTY) ): @@ -34,6 +35,7 @@ cdef parquet_reader_options _setup_parquet_reader_options( parquet_reader_options.builder(source_info.c_obj) .convert_strings_to_categories(convert_strings_to_categories) .use_pandas_metadata(use_pandas_metadata) + .allow_mismatched_pq_schemas(allow_mismatched_pq_schemas) .use_arrow_schema(True) .build() ) @@ -80,6 +82,9 @@ cdef class ChunkedParquetReader: pass_read_limit : size_t, default 1024000000 Limit on the amount of memory used for reading and decompressing data or 0 if there is no limit. + allow_mismatched_pq_schemas : bool, default False + Whether to read (matching) columns specified in `columns` from + the input files with otherwise mismatched schemas. """ def __init__( self, @@ -91,7 +96,8 @@ cdef class ChunkedParquetReader: int64_t skip_rows = 0, size_type nrows = -1, size_t chunk_read_limit=0, - size_t pass_read_limit=1024000000 + size_t pass_read_limit=1024000000, + bool allow_mismatched_pq_schemas=False ): cdef parquet_reader_options opts = _setup_parquet_reader_options( @@ -103,6 +109,7 @@ cdef class ChunkedParquetReader: use_pandas_metadata=use_pandas_metadata, skip_rows=skip_rows, nrows=nrows, + allow_mismatched_pq_schemas=allow_mismatched_pq_schemas, ) with nogil: @@ -152,6 +159,7 @@ cpdef read_parquet( bool use_pandas_metadata = True, int64_t skip_rows = 0, size_type nrows = -1, + bool allow_mismatched_pq_schemas = False, # Disabled, these aren't used by cudf-python # we should only add them back in if there's user demand # ReaderColumnSchema reader_column_schema = None, @@ -179,6 +187,9 @@ cpdef read_parquet( The number of rows to skip from the start of the file. nrows : size_type, default -1 The number of rows to read. By default, read the entire file. + allow_mismatched_pq_schemas : bool, default False + If True, enable reading (matching) columns specified in `columns` + from the input files with otherwise mismatched schemas. Returns ------- @@ -195,6 +206,7 @@ cpdef read_parquet( use_pandas_metadata, skip_rows, nrows, + allow_mismatched_pq_schemas, ) with nogil: diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd index 222d87defa0..de6a6c1e82d 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet.pxd @@ -32,7 +32,7 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: data_type get_timestamp_type() except + bool is_enabled_use_pandas_metadata() except + bool is_enabled_arrow_schema() except + - + bool is_enabled_allow_mismatched_pq_schemas() except + # setter void set_filter(expression &filter) except + @@ -41,6 +41,7 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: void set_row_groups(vector[vector[size_type]] row_grp) except + void set_skip_rows(int64_t val) except + void enable_use_arrow_schema(bool val) except + + void enable_allow_mismatched_pq_schemas(bool val) except + void enable_use_pandas_metadata(bool val) except + void set_timestamp_type(data_type type) except + @@ -69,6 +70,9 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: parquet_reader_options_builder& use_arrow_schema( bool val ) except + + parquet_reader_options_builder& allow_mismatched_pq_schemas( + bool val + ) except + parquet_reader_options_builder& timestamp_type( data_type type ) except + From 9e9efcc9f5ed8411fb09f4d8384e14612a7f3b10 Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:19:48 +1000 Subject: [PATCH 143/270] Replace raw device_memory_resource pointer in pylibcudf Cython (#16674) Replaces a single `device_memory_resource*` in pylibcudf Cython inline C++ function with `rmm::device_async_resource_ref` to help smooth RMM refactoring effort. Authors: - Mark Harris (https://github.com/harrism) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16674 --- python/pylibcudf/pylibcudf/libcudf/interop.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index c7efff2340d..9228c017d93 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -71,7 +71,7 @@ cdef extern from *: ArrowArray* to_arrow_host_raw( cudf::table_view const& tbl, rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) { + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) { // Assumes the sync event is null and the data is already on the host. ArrowArray *arr = new ArrowArray(); auto device_arr = cudf::to_arrow_host(tbl, stream, mr); From f6e2355dfefb1a02a984425aabeca7a4fcb2bfde Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 28 Aug 2024 19:03:34 -0500 Subject: [PATCH 144/270] Handle `ordered` parameter in `CategoricalIndex.__repr__` (#16683) Thanks @mroeschke for catching this in https://github.com/rapidsai/cudf/pull/16665#discussion_r1735277661 This PR factors in the `ordered` parameter while generating the `repr` for `CategoricalIndex`. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16683 --- python/cudf/cudf/core/index.py | 1 + python/cudf/cudf/tests/test_repr.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 500fc580097..fc35ffa3744 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1456,6 +1456,7 @@ def __repr__(self): pd_preprocess.dtype._categories = ( preprocess.categories.to_pandas() ) + pd_preprocess.dtype._ordered = preprocess.dtype.ordered cats_repr = repr(pd_preprocess).split("\n") output = "\n".join(data_repr[:-1] + cats_repr[-1:]) diff --git a/python/cudf/cudf/tests/test_repr.py b/python/cudf/cudf/tests/test_repr.py index 57eef9e3463..681b467f66c 100644 --- a/python/cudf/cudf/tests/test_repr.py +++ b/python/cudf/cudf/tests/test_repr.py @@ -1491,3 +1491,11 @@ def test_large_unique_categories_repr(): with utils.cudf_timeout(2, timeout_message="Failed to repr fast enough"): actual_repr = repr(gi) assert expected_repr == actual_repr + + +@pytest.mark.parametrize("ordered", [True, False]) +def test_categorical_index_ordered(ordered): + pi = pd.CategoricalIndex(range(10), ordered=ordered) + gi = cudf.CategoricalIndex(range(10), ordered=ordered) + + assert repr(pi) == repr(gi) From f2d153b5e1d0c8410947afb438033468dc84d1b8 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:30:21 -1000 Subject: [PATCH 145/270] Have interval_range use IntervalIndex.from_breaks, remove column_empty_same_mask (#16694) To match pandas implementation, `interval_range` dispatches to `IntervalIndex.from_breaks` which allows some code deduplication. This also allows us to remove `column_empty_same_mask` which (luckily) I didn't find any usage across RAPIDS Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16694 --- python/cudf/cudf/core/column/__init__.py | 1 - python/cudf/cudf/core/column/column.py | 16 ------------- python/cudf/cudf/core/index.py | 30 +++--------------------- 3 files changed, 3 insertions(+), 44 deletions(-) diff --git a/python/cudf/cudf/core/column/__init__.py b/python/cudf/cudf/core/column/__init__.py index 5781d77ee9a..06791df7dc0 100644 --- a/python/cudf/cudf/core/column/__init__.py +++ b/python/cudf/cudf/core/column/__init__.py @@ -11,7 +11,6 @@ build_column, column_empty, column_empty_like, - column_empty_like_same_mask, concat_columns, deserialize_columns, serialize_columns, diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 885476a897c..7674565e2c3 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1483,22 +1483,6 @@ def _has_any_nan(arbitrary: pd.Series | np.ndarray) -> bool: ) -def column_empty_like_same_mask( - column: ColumnBase, dtype: Dtype -) -> ColumnBase: - """Create a new empty Column with the same length and the same mask. - - Parameters - ---------- - dtype : np.dtype like - The dtype of the data buffer. - """ - result = column_empty_like(column, dtype) - if column.nullable: - result = result.set_mask(column.mask) - return result - - def column_empty( row_count: int, dtype: Dtype = "object", masked: bool = False ) -> ColumnBase: diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index fc35ffa3744..241a276ebe2 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3346,31 +3346,7 @@ def interval_range( init=start.device_value, step=freq.device_value, ) - left_col = bin_edges.slice(0, len(bin_edges) - 1) - right_col = bin_edges.slice(1, len(bin_edges)) - # For indexing, children should both have 0 offset - right_col = type(right_col)( - data=right_col.data, - dtype=right_col.dtype, - size=right_col.size, - mask=right_col.mask, - offset=0, - null_count=right_col.null_count, - children=right_col.children, - ) - - if len(right_col) == 0 or len(left_col) == 0: - dtype = IntervalDtype("int64", closed) - data = column.column_empty_like_same_mask(left_col, dtype) - return IntervalIndex(data, closed=closed, name=name) - - interval_col = IntervalColumn( - data=None, - dtype=IntervalDtype(left_col.dtype, closed), - size=len(left_col), - children=(left_col, right_col), - ) - return IntervalIndex(interval_col, closed=closed, name=name) + return IntervalIndex.from_breaks(bin_edges, closed=closed, name=name) class IntervalIndex(Index): @@ -3520,7 +3496,7 @@ def from_breaks( left_col = breaks.slice(0, len(breaks) - 1) right_col = breaks.slice(1, len(breaks)) # For indexing, children should both have 0 offset - right_col = column.build_column( + right_col = type(right_col)( data=right_col.data, dtype=right_col.dtype, size=right_col.size, @@ -3536,7 +3512,7 @@ def from_breaks( size=len(left_col), children=(left_col, right_col), ) - return IntervalIndex(interval_col, name=name, closed=closed) + return IntervalIndex._from_column(interval_col, name=name) @classmethod def from_arrays( From eca5108d2f3120c83b26ba5e3c9a6cfaa2b0b233 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:37:15 -0400 Subject: [PATCH 146/270] Disable gtests/ERROR_TEST during compute-sanitizer memcheck test (#16691) Disables the `gtests/ERROR_TEST` when run under `compute-sanitizer` for memcheck. The `compute-sanitizer` started hanging on some of these tests. There is no value in running memcheck on any of the tests in `ERROR_TEST` Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Paul Mattione (https://github.com/pmattione-nvidia) URL: https://github.com/rapidsai/cudf/pull/16691 --- cpp/tests/error/error_handling_test.cu | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/tests/error/error_handling_test.cu b/cpp/tests/error/error_handling_test.cu index 1dfe45556c4..9c7459fa69d 100644 --- a/cpp/tests/error/error_handling_test.cu +++ b/cpp/tests/error/error_handling_test.cu @@ -50,8 +50,6 @@ CUDF_KERNEL void test_kernel(int* data) { data[threadIdx.x] = threadIdx.x; } // calls. TEST(StreamCheck, FailedKernel) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { GTEST_SKIP(); } - rmm::cuda_stream stream; int a; test_kernel<<<0, 0, 0, stream.value()>>>(&a); @@ -63,8 +61,6 @@ TEST(StreamCheck, FailedKernel) TEST(StreamCheck, CatchFailedKernel) { - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { GTEST_SKIP(); } - rmm::cuda_stream stream; int a; test_kernel<<<0, 0, 0, stream.value()>>>(&a); @@ -131,6 +127,8 @@ TEST(DebugAssert, cudf_assert_true) // 2.) The RMM Pool interferes with the death test int main(int argc, char** argv) { + if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return 0; } + ::testing::InitGoogleTest(&argc, argv); auto const cmd_opts = parse_cudf_test_opts(argc, argv); auto adaptor = make_stream_mode_adaptor(cmd_opts); From 21d05d73a66c0bc0009ff378beb58fb4f0f2bf2d Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:40:14 -0400 Subject: [PATCH 147/270] Move apply_boolean_mask benchmark to nvbench (#16616) Reworks the `apply_booleam_mask` benchmark as an nvbench benchmark under the `STREAM_COMPACTION_NVBENCH` module. `cudf::string_view` was added as a type to help measure the performance improvement in a follow on PR for `apply_boolean_mask` for strings Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16616 --- cpp/benchmarks/CMakeLists.txt | 5 +- .../stream_compaction/apply_boolean_mask.cpp | 138 ++++++------------ 2 files changed, 48 insertions(+), 95 deletions(-) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 7f3edfa0a01..99ef9e2976f 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -151,14 +151,11 @@ ConfigureBench(COPY_IF_ELSE_BENCH copying/copy_if_else.cpp) # * transpose benchmark --------------------------------------------------------------------------- ConfigureBench(TRANSPOSE_BENCH transpose/transpose.cpp) -# ################################################################################################## -# * apply_boolean_mask benchmark ------------------------------------------------------------------ -ConfigureBench(APPLY_BOOLEAN_MASK_BENCH stream_compaction/apply_boolean_mask.cpp) - # ################################################################################################## # * stream_compaction benchmark ------------------------------------------------------------------- ConfigureNVBench( STREAM_COMPACTION_NVBENCH + stream_compaction/apply_boolean_mask.cpp stream_compaction/distinct.cpp stream_compaction/distinct_count.cpp stream_compaction/stable_distinct.cpp diff --git a/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp b/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp index 492237474ff..fa017ca9e29 100644 --- a/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp +++ b/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp @@ -15,120 +15,76 @@ */ #include +#include #include +#include -#include -#include +#include namespace { -constexpr cudf::size_type hundredM = 1e8; -constexpr cudf::size_type tenM = 1e7; -constexpr cudf::size_type tenK = 1e4; -constexpr cudf::size_type fifty_percent = 50; - -void percent_range(benchmark::internal::Benchmark* b) -{ - b->Unit(benchmark::kMillisecond); - for (int percent = 0; percent <= 100; percent += 10) - b->Args({hundredM, percent}); -} - -void size_range(benchmark::internal::Benchmark* b) -{ - b->Unit(benchmark::kMillisecond); - for (int size = tenK; size <= hundredM; size *= 10) - b->Args({size, fifty_percent}); -} - template -void calculate_bandwidth(benchmark::State& state, cudf::size_type num_columns) +void calculate_bandwidth(nvbench::state& state) { - cudf::size_type const column_size{static_cast(state.range(0))}; - cudf::size_type const percent_true{static_cast(state.range(1))}; - - float const fraction = percent_true / 100.f; - cudf::size_type const column_size_out = fraction * column_size; - int64_t const mask_size = - sizeof(bool) * column_size + cudf::bitmask_allocation_size_bytes(column_size); - int64_t const validity_bytes_in = (fraction >= 1.0f / 32) - ? cudf::bitmask_allocation_size_bytes(column_size) - : 4 * column_size_out; - int64_t const validity_bytes_out = cudf::bitmask_allocation_size_bytes(column_size_out); - int64_t const column_bytes_out = sizeof(T) * column_size_out; + auto const n_rows = static_cast(state.get_int64("rows")); + auto const n_cols = static_cast(state.get_int64("columns")); + auto const percent_true = static_cast(state.get_int64("hits_%")); + + double const fraction = percent_true / 100.0; + cudf::size_type const output_size = fraction * n_rows; + int64_t const mask_size = sizeof(bool) * n_rows + cudf::bitmask_allocation_size_bytes(n_rows); + int64_t const validity_bytes_in = + (fraction >= 1.0 / 32) ? cudf::bitmask_allocation_size_bytes(n_rows) : 4 * output_size; + int64_t const validity_bytes_out = cudf::bitmask_allocation_size_bytes(output_size); + int64_t const column_bytes_out = sizeof(T) * output_size; int64_t const column_bytes_in = column_bytes_out; // we only read unmasked inputs - int64_t const bytes_read = - (column_bytes_in + validity_bytes_in) * num_columns + // reading columns - mask_size; // reading boolean mask + int64_t const bytes_read = (column_bytes_in + validity_bytes_in) * n_cols + // reading columns + mask_size; // reading boolean mask int64_t const bytes_written = - (column_bytes_out + validity_bytes_out) * num_columns; // writing columns + (column_bytes_out + validity_bytes_out) * n_cols; // writing columns - state.SetItemsProcessed(state.iterations() * column_size * num_columns); - state.SetBytesProcessed(static_cast(state.iterations()) * (bytes_read + bytes_written)); + state.add_element_count(n_rows * n_cols); + state.add_global_memory_reads(bytes_read); + state.add_global_memory_writes(bytes_written); } } // namespace -template -void BM_apply_boolean_mask(benchmark::State& state, cudf::size_type num_columns) +template +void apply_boolean_mask_benchmark(nvbench::state& state, nvbench::type_list) { - cudf::size_type const column_size{static_cast(state.range(0))}; - cudf::size_type const percent_true{static_cast(state.range(1))}; + auto const n_rows = static_cast(state.get_int64("rows")); + auto const n_cols = static_cast(state.get_int64("columns")); + auto const percent_true = static_cast(state.get_int64("hits_%")); - data_profile profile = data_profile_builder().cardinality(0).null_probability(0.0).distribution( - cudf::type_to_id(), distribution_id::UNIFORM, 0, 100); + auto const input_type = cudf::type_to_id(); + data_profile profile = data_profile_builder().cardinality(0).no_validity().distribution( + input_type, distribution_id::UNIFORM, 0, 20); - auto source_table = create_random_table( - cycle_dtypes({cudf::type_to_id()}, num_columns), row_count{column_size}, profile); + auto source_table = + create_random_table(cycle_dtypes({input_type}, n_cols), row_count{n_rows}, profile); profile.set_bool_probability_true(percent_true / 100.0); profile.set_null_probability(std::nullopt); // no null mask - auto mask = create_random_column(cudf::type_id::BOOL8, row_count{column_size}, profile); + auto mask = create_random_column(cudf::type_id::BOOL8, row_count{n_rows}, profile); + + auto stream = cudf::get_default_stream(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); + calculate_bandwidth(state); - for (auto _ : state) { - cuda_event_timer raii(state, true); - auto result = cudf::apply_boolean_mask(*source_table, mask->view()); - } + state.exec(nvbench::exec_tag::sync, [&source_table, &mask](nvbench::launch& launch) { + cudf::apply_boolean_mask(*source_table, mask->view()); + }); - calculate_bandwidth(state, num_columns); + set_throughputs(state); } -template -class ApplyBooleanMask : public cudf::benchmark { - public: - using TypeParam = T; -}; - -#define ABM_BENCHMARK_DEFINE(name, type, n_columns) \ - BENCHMARK_TEMPLATE_DEFINE_F(ApplyBooleanMask, name, type)(::benchmark::State & st) \ - { \ - BM_apply_boolean_mask(st, n_columns); \ - } - -ABM_BENCHMARK_DEFINE(float_1_col, float, 1); -ABM_BENCHMARK_DEFINE(float_2_col, float, 2); -ABM_BENCHMARK_DEFINE(float_4_col, float, 4); - -// shmoo 1, 2, 4 column float across percentage true -BENCHMARK_REGISTER_F(ApplyBooleanMask, float_1_col)->Apply(percent_range); -BENCHMARK_REGISTER_F(ApplyBooleanMask, float_2_col)->Apply(percent_range); -BENCHMARK_REGISTER_F(ApplyBooleanMask, float_4_col)->Apply(percent_range); - -// shmoo 1, 2, 4 column float across column sizes with 50% true -BENCHMARK_REGISTER_F(ApplyBooleanMask, float_1_col)->Apply(size_range); -BENCHMARK_REGISTER_F(ApplyBooleanMask, float_2_col)->Apply(size_range); -BENCHMARK_REGISTER_F(ApplyBooleanMask, float_4_col)->Apply(size_range); - -// spot benchmark other types -ABM_BENCHMARK_DEFINE(int8_1_col, int8_t, 1); -ABM_BENCHMARK_DEFINE(int16_1_col, int16_t, 1); -ABM_BENCHMARK_DEFINE(int32_1_col, int32_t, 1); -ABM_BENCHMARK_DEFINE(int64_1_col, int64_t, 1); -ABM_BENCHMARK_DEFINE(double_1_col, double, 1); -BENCHMARK_REGISTER_F(ApplyBooleanMask, int8_1_col)->Args({tenM, fifty_percent}); -BENCHMARK_REGISTER_F(ApplyBooleanMask, int16_1_col)->Args({tenM, fifty_percent}); -BENCHMARK_REGISTER_F(ApplyBooleanMask, int32_1_col)->Args({tenM, fifty_percent}); -BENCHMARK_REGISTER_F(ApplyBooleanMask, int64_1_col)->Args({tenM, fifty_percent}); -BENCHMARK_REGISTER_F(ApplyBooleanMask, double_1_col)->Args({tenM, fifty_percent}); +using data_type = nvbench::type_list; +NVBENCH_BENCH_TYPES(apply_boolean_mask_benchmark, NVBENCH_TYPE_AXES(data_type)) + .set_name("apply_boolean_mask") + .set_type_axes_names({"type"}) + .add_int64_axis("columns", {1, 4}) + .add_int64_axis("rows", {100'000, 1'000'000, 10'000'000}) + .add_int64_axis("hits_%", {10, 50, 100}); From 8c7af08073fba49c7a7e62cc30595b2962ae7e65 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 29 Aug 2024 18:18:05 -0500 Subject: [PATCH 148/270] Increase timeouts for couple of tests (#16692) This PR increases timeouts for tests. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16692 --- python/cudf/cudf/testing/_utils.py | 25 ++++++++++--------- python/cudf/cudf/tests/test_repr.py | 2 +- .../cudf_pandas_tests/test_cudf_pandas.py | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/testing/_utils.py b/python/cudf/cudf/testing/_utils.py index 540f12c8382..8cb9efa873c 100644 --- a/python/cudf/cudf/testing/_utils.py +++ b/python/cudf/cudf/testing/_utils.py @@ -1,8 +1,8 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. import itertools -import signal import string +import time from collections import abc from contextlib import contextmanager from decimal import Decimal @@ -376,16 +376,17 @@ class cudf_timeout: Context manager to raise a TimeoutError after a specified number of seconds. """ - def __init__(self, seconds, *, timeout_message=""): - self.seconds = int(seconds) - self.timeout_message = timeout_message - - def _timeout_handler(self, signum, frame): - raise TimeoutError(self.timeout_message) + def __init__(self, timeout): + self.timeout = timeout def __enter__(self): - signal.signal(signal.SIGALRM, self._timeout_handler) - signal.alarm(self.seconds) - - def __exit__(self, type, value, traceback): - signal.alarm(0) + self.start_time = time.perf_counter() + + def __exit__(self, *args): + elapsed_time = ( + time.perf_counter() - self.start_time + ) # Calculate elapsed time + if elapsed_time >= self.timeout: + raise TimeoutError( + f"Expected to finish in {self.timeout=} seconds but took {elapsed_time=} seconds" + ) diff --git a/python/cudf/cudf/tests/test_repr.py b/python/cudf/cudf/tests/test_repr.py index 681b467f66c..95e19fae501 100644 --- a/python/cudf/cudf/tests/test_repr.py +++ b/python/cudf/cudf/tests/test_repr.py @@ -1488,7 +1488,7 @@ def test_large_unique_categories_repr(): pi = pd.CategoricalIndex(range(100_000_000)) gi = cudf.CategoricalIndex(range(100_000_000)) expected_repr = repr(pi) - with utils.cudf_timeout(2, timeout_message="Failed to repr fast enough"): + with utils.cudf_timeout(6): actual_repr = repr(gi) assert expected_repr == actual_repr diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 0827602852d..505d5d0b9cc 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1664,7 +1664,7 @@ def test_notebook_slow_repr(): nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor( - timeout=20, kernel_name=jupyter_client.KernelManager().kernel_name + timeout=30, kernel_name=jupyter_client.KernelManager().kernel_name ) try: From 53f488ba2db10bead273b1e5eff5f1a07703a7ae Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:09:14 -1000 Subject: [PATCH 149/270] Add type annotations to Index classes, utilize _from_column more (#16695) * Add more type annotations to `index.py` * More consistently use `Index._from_column` where appropriate * Remove single used `Index._indices_of` in favor of just accessing the `Column._indicies_of` method Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16695 --- python/cudf/cudf/core/index.py | 165 +++++++++++++++----------------- python/cudf/cudf/core/series.py | 5 +- 2 files changed, 83 insertions(+), 87 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 241a276ebe2..66d03682de4 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -46,7 +46,6 @@ from cudf.core.column.column import as_column, concat_columns from cudf.core.column.string import StringMethods as StringMethods from cudf.core.dtypes import IntervalDtype -from cudf.core.frame import Frame from cudf.core.join._join_helpers import _match_join_keys from cudf.core.mixins import BinaryOperand from cudf.core.single_column_frame import SingleColumnFrame @@ -63,6 +62,8 @@ from collections.abc import Generator, Iterable from datetime import tzinfo + from cudf.core.frame import Frame + def ensure_index(index_like: Any) -> BaseIndex: """ @@ -316,7 +317,7 @@ def _num_rows(self) -> int: @cached_property # type: ignore @_performance_tracking - def _values(self): + def _values(self) -> ColumnBase: if len(self) > 0: return column.as_column(self._range, dtype=self.dtype) else: @@ -582,7 +583,7 @@ def __rmul__(self, other): return self.__mul__(other) @_performance_tracking - def _as_int_index(self): + def _as_int_index(self) -> Index: # Convert self to an integer index. This method is used to perform ops # that are not defined directly on RangeIndex. return cudf.Index._from_data(self._data) @@ -870,12 +871,12 @@ def join( @property # type: ignore @_performance_tracking - def _column(self): + def _column(self) -> ColumnBase: return self._as_int_index()._column @property # type: ignore @_performance_tracking - def _columns(self): + def _columns(self) -> list[ColumnBase]: return self._as_int_index()._columns @property # type: ignore @@ -937,7 +938,7 @@ def notna(self) -> cupy.ndarray: notnull = isna @_performance_tracking - def _minmax(self, meth: str): + def _minmax(self, meth: str) -> int | float: no_steps = len(self) - 1 if no_steps == -1: return np.nan @@ -948,10 +949,10 @@ def _minmax(self, meth: str): return self.start + self.step * no_steps - def min(self): + def min(self) -> int | float: return self._minmax("min") - def max(self): + def max(self) -> int | float: return self._minmax("max") @property @@ -1115,7 +1116,7 @@ def _from_data_like_self( @classmethod @_performance_tracking - def from_arrow(cls, obj): + def from_arrow(cls, obj) -> Index | cudf.MultiIndex: """Create from PyArrow Array/ChunkedArray. Parameters @@ -1145,11 +1146,11 @@ def from_arrow(cls, obj): return cudf.MultiIndex.from_arrow(obj) @cached_property - def is_monotonic_increasing(self): + def is_monotonic_increasing(self) -> bool: return super().is_monotonic_increasing @cached_property - def is_monotonic_decreasing(self): + def is_monotonic_decreasing(self) -> bool: return super().is_monotonic_decreasing def _binaryop( @@ -1191,7 +1192,7 @@ def _binaryop( @property # type: ignore @_performance_tracking - def _values(self): + def _values(self) -> ColumnBase: return self._column @classmethod @@ -1239,12 +1240,12 @@ def _concat(cls, objs): return result @_performance_tracking - def memory_usage(self, deep=False): + def memory_usage(self, deep: bool = False) -> int: return self._column.memory_usage @cached_property # type: ignore @_performance_tracking - def is_unique(self): + def is_unique(self) -> bool: return self._column.is_unique @_performance_tracking @@ -1271,7 +1272,7 @@ def equals(self, other) -> bool: return False @_performance_tracking - def copy(self, name=None, deep=False): + def copy(self, name: Hashable = None, deep: bool = False) -> Self: """ Make a copy of this object. @@ -1288,13 +1289,11 @@ def copy(self, name=None, deep=False): New index instance. """ name = self.name if name is None else name - - return _index_from_data( - {name: self._values.copy(True) if deep else self._values} - ) + col = self._column.copy(deep=True) if deep else self._column + return type(self)._from_column(col, name=name) @_performance_tracking - def astype(self, dtype, copy: bool = True): + def astype(self, dtype, copy: bool = True) -> Index: return super().astype({self.name: dtype}, copy) @_performance_tracking @@ -1405,7 +1404,7 @@ def get_loc(self, key) -> int | slice | cupy.ndarray: return mask @_performance_tracking - def __repr__(self): + def __repr__(self) -> str: max_seq_items = pd.get_option("max_seq_items") or len(self) mr = 0 if 2 * max_seq_items < len(self): @@ -1501,8 +1500,8 @@ def __repr__(self): keywords.append( f"freq={self._freq._maybe_as_fast_pandas_offset().freqstr!r}" ) - keywords = ", ".join(keywords) - lines.append(f"{prior_to_dtype} {keywords})") + joined_keywords = ", ".join(keywords) + lines.append(f"{prior_to_dtype} {joined_keywords})") return "\n".join(lines) @_performance_tracking @@ -1518,47 +1517,47 @@ def dtype(self): """ `dtype` of the underlying values in Index. """ - return self._values.dtype + return self._column.dtype @_performance_tracking - def isna(self): + def isna(self) -> cupy.ndarray: return self._column.isnull().values isnull = isna @_performance_tracking - def notna(self): + def notna(self) -> cupy.ndarray: return self._column.notnull().values notnull = notna - def _is_numeric(self): + def _is_numeric(self) -> bool: return ( isinstance(self._values, cudf.core.column.NumericalColumn) and self.dtype.kind != "b" ) - def _is_boolean(self): + def _is_boolean(self) -> bool: return self.dtype.kind == "b" - def _is_integer(self): + def _is_integer(self) -> bool: return self.dtype.kind in "iu" - def _is_floating(self): + def _is_floating(self) -> bool: return self.dtype.kind == "f" - def _is_object(self): - return isinstance(self._values, cudf.core.column.StringColumn) + def _is_object(self) -> bool: + return isinstance(self._column, cudf.core.column.StringColumn) - def _is_categorical(self): + def _is_categorical(self) -> bool: return False - def _is_interval(self): + def _is_interval(self) -> bool: return False @property # type: ignore @_performance_tracking - def hasnans(self): + def hasnans(self) -> bool: return self._column.has_nulls(include_nan=True) @_performance_tracking @@ -1600,13 +1599,13 @@ def argsort( na_position=na_position, ) - def repeat(self, repeats, axis=None): - return self._from_columns_like_self( - Frame._repeat([*self._columns], repeats, axis), self._column_names - ) + def repeat(self, repeats, axis=None) -> Self: + result = super()._repeat([self._column], repeats, axis)[0] + result = result._with_type_metadata(self.dtype) + return type(self)._from_column(result, name=self.name) @_performance_tracking - def where(self, cond, other=None, inplace=False): + def where(self, cond, other=None, inplace=False) -> Index: result_col = super().where(cond, other, inplace) return self._mimic_inplace( _index_from_data({self.name: result_col}), @@ -1614,14 +1613,14 @@ def where(self, cond, other=None, inplace=False): ) @property - def values(self): + def values(self) -> cupy.ndarray: return self._column.values - def __contains__(self, item): + def __contains__(self, item) -> bool: hash(item) - return item in self._values + return item in self._column - def _clean_nulls_from_index(self): + def _clean_nulls_from_index(self) -> Index: if self._values.has_nulls(): fill_value = ( str(cudf.NaT) @@ -1635,8 +1634,8 @@ def _clean_nulls_from_index(self): return self - def any(self): - return self._values.any() + def any(self) -> bool: + return self._column.any() def to_pandas( self, *, nullable: bool = False, arrow_type: bool = False @@ -1691,11 +1690,9 @@ def unique(self, level: int | None = None) -> Self: raise IndexError( f"Too many levels: Index has only 1 level, not {level + 1}" ) - return cudf.core.index._index_from_data( - {self.name: self._values.unique()}, name=self.name - ) + return type(self)._from_column(self._column.unique(), name=self.name) - def isin(self, values, level=None): + def isin(self, values, level=None) -> cupy.ndarray: if level is not None and level > 0: raise IndexError( f"Too many levels: Index has only 1 level, not {level + 1}" @@ -1706,11 +1703,7 @@ def isin(self, values, level=None): f"to isin(), you passed a {type(values).__name__}" ) - return self._values.isin(values).values - - def _indices_of(self, value): - """Return indices of value in index""" - return self._column.indices_of(value) + return self._column.isin(values).values @copy_docstring(StringMethods) # type: ignore @property @@ -2130,7 +2123,7 @@ def day_of_week(self) -> Index: @property # type: ignore @_performance_tracking - def year(self): + def year(self) -> Index: """ The year of the datetime. @@ -2149,7 +2142,7 @@ def year(self): @property # type: ignore @_performance_tracking - def month(self): + def month(self) -> Index: """ The month as January=1, December=12. @@ -2168,7 +2161,7 @@ def month(self): @property # type: ignore @_performance_tracking - def day(self): + def day(self) -> Index: """ The day of the datetime. @@ -2187,7 +2180,7 @@ def day(self): @property # type: ignore @_performance_tracking - def hour(self): + def hour(self) -> Index: """ The hours of the datetime. @@ -2208,7 +2201,7 @@ def hour(self): @property # type: ignore @_performance_tracking - def minute(self): + def minute(self) -> Index: """ The minutes of the datetime. @@ -2229,7 +2222,7 @@ def minute(self): @property # type: ignore @_performance_tracking - def second(self): + def second(self) -> Index: """ The seconds of the datetime. @@ -2250,7 +2243,7 @@ def second(self): @property # type: ignore @_performance_tracking - def microsecond(self): + def microsecond(self) -> Index: """ The microseconds of the datetime. @@ -2281,7 +2274,7 @@ def microsecond(self): @property # type: ignore @_performance_tracking - def nanosecond(self): + def nanosecond(self) -> Index: """ The nanoseconds of the datetime. @@ -2303,7 +2296,7 @@ def nanosecond(self): @property # type: ignore @_performance_tracking - def weekday(self): + def weekday(self) -> Index: """ The day of the week with Monday=0, Sunday=6. @@ -2325,7 +2318,7 @@ def weekday(self): @property # type: ignore @_performance_tracking - def dayofweek(self): + def dayofweek(self) -> Index: """ The day of the week with Monday=0, Sunday=6. @@ -2347,7 +2340,7 @@ def dayofweek(self): @property # type: ignore @_performance_tracking - def dayofyear(self): + def dayofyear(self) -> Index: """ The day of the year, from 1-365 in non-leap years and from 1-366 in leap years. @@ -2370,7 +2363,7 @@ def dayofyear(self): @property # type: ignore @_performance_tracking - def day_of_year(self): + def day_of_year(self) -> Index: """ The day of the year, from 1-365 in non-leap years and from 1-366 in leap years. @@ -2412,7 +2405,7 @@ def is_leap_year(self) -> cupy.ndarray: @property # type: ignore @_performance_tracking - def quarter(self): + def quarter(self) -> Index: """ Integer indicator for which quarter of the year the date belongs in. @@ -2523,11 +2516,11 @@ def _get_dt_field(self, field: str) -> Index: ) return Index._from_column(out_column, name=self.name) - def _is_boolean(self): + def _is_boolean(self) -> bool: return False @_performance_tracking - def ceil(self, freq): + def ceil(self, freq: str) -> Self: """ Perform ceil operation on the data to the specified freq. @@ -2558,7 +2551,7 @@ def ceil(self, freq): return type(self)._from_column(self._column.ceil(freq), name=self.name) @_performance_tracking - def floor(self, freq): + def floor(self, freq: str) -> Self: """ Perform floor operation on the data to the specified freq. @@ -2591,7 +2584,7 @@ def floor(self, freq): ) @_performance_tracking - def round(self, freq): + def round(self, freq: str) -> Self: """ Perform round operation on the data to the specified freq. @@ -2635,7 +2628,7 @@ def tz_localize( tz: str | None, ambiguous: Literal["NaT"] = "NaT", nonexistent: Literal["NaT"] = "NaT", - ): + ) -> Self: """ Localize timezone-naive data to timezone-aware data. @@ -2682,7 +2675,7 @@ def tz_localize( result_col, name=self.name, freq=self._freq ) - def tz_convert(self, tz: str | None): + def tz_convert(self, tz: str | None) -> Self: """ Convert tz-aware datetimes from one time zone to another. @@ -2717,7 +2710,7 @@ def tz_convert(self, tz: str | None): result_col = self._column.tz_convert(tz) return DatetimeIndex._from_column(result_col, name=self.name) - def repeat(self, repeats, axis=None): + def repeat(self, repeats, axis=None) -> Self: res = super().repeat(repeats, axis=axis) res._freq = None return res @@ -2982,7 +2975,7 @@ def nanoseconds(self) -> cudf.Index: @property # type: ignore @_performance_tracking - def components(self): + def components(self) -> cudf.DataFrame: """ Return a dataframe of the components (days, hours, minutes, seconds, milliseconds, microseconds, nanoseconds) of the Timedeltas. @@ -3003,7 +2996,7 @@ def inferred_freq(self): """ raise NotImplementedError("inferred_freq is not yet supported") - def _is_boolean(self): + def _is_boolean(self) -> bool: return False @@ -3122,16 +3115,16 @@ def codes(self) -> cudf.Index: @property # type: ignore @_performance_tracking - def categories(self): + def categories(self) -> cudf.Index: """ The categories of this categorical. """ return self.dtype.categories - def _is_boolean(self): + def _is_boolean(self) -> bool: return False - def _is_categorical(self): + def _is_categorical(self) -> bool: return True def add_categories(self, new_categories) -> Self: @@ -3440,7 +3433,7 @@ def __init__( super().__init__(interval_col, name=name) @property - def closed(self): + def closed(self) -> Literal["left", "right", "neither", "both"]: return self.dtype.closed @classmethod @@ -3461,7 +3454,7 @@ def from_breaks( name=None, copy: bool = False, dtype=None, - ): + ) -> Self: """ Construct an IntervalIndex from an array of splits. @@ -3533,7 +3526,7 @@ def from_tuples( name=None, copy: bool = False, dtype=None, - ) -> IntervalIndex: + ) -> Self: piidx = pd.IntervalIndex.from_tuples( data, closed=closed, name=name, copy=copy, dtype=dtype ) @@ -3544,13 +3537,13 @@ def __getitem__(self, index): "Getting a scalar from an IntervalIndex is not yet supported" ) - def _is_interval(self): + def _is_interval(self) -> bool: return True - def _is_boolean(self): + def _is_boolean(self) -> bool: return False - def _clean_nulls_from_index(self): + def _clean_nulls_from_index(self) -> Self: return self @property diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index a831a798772..837c6872258 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -377,7 +377,10 @@ def _loc_to_iloc(self, arg): warnings.warn(warn_msg, FutureWarning) return arg try: - indices = self._frame.index._indices_of(arg) + if isinstance(self._frame.index, RangeIndex): + indices = self._frame.index._indices_of(arg) + else: + indices = self._frame.index._column.indices_of(arg) if (n := len(indices)) == 0: raise KeyError("Label scalar is out of bounds") elif n == 1: From 8f2d68750f839326343db00debb5735fe14075d3 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:40:20 -0700 Subject: [PATCH 150/270] Refactor dictionary encoding in PQ writer to migrate to the new `cuco::static_map` (#16541) Part of #12261. This PR refactors the dictionary encoding in Parquet writers to migrate from `cuco::legacy::static_map` to `cuco::static_map` to build the dictionaries. ### Performance Results The changes result in +0.08% average speed improvement and +16.22% average memory footprint increase (stems from the adjusted sizes by `cuco::make_window_extent` due to [prime gap](https://en.wikipedia.org/wiki/Prime_gap)) across the benchmark cases extended from #16591 Currently, we do see a roughly 8% speed improvement in map insert and find kernels which is counteracted by the map init and map collect kernels as they have to process 16.22% more slots. With a cuco bump, the average speed improvement will increase from 0.08% to +3% and the memory footprint change will go back from 16.22% to +0%. ### Hardware used for benchmarking ``` `NVIDIA RTX 5880 Ada Generation` * SM Version: 890 (PTX Version: 860) * Number of SMs: 110 * SM Default Clock Rate: 18446744071874 MHz * Global Memory: 23879 MiB Free / 48632 MiB Total * Global Memory Bus Peak: 960 GB/sec (384-bit DDR @10001MHz) * Max Shared Memory: 100 KiB/SM, 48 KiB/Block * L2 Cache Size: 98304 KiB * Maximum Active Blocks: 24/SM * Maximum Active Threads: 1536/SM, 1024/Block * Available Registers: 65536/SM, 65536/Block * ECC Enabled: No ``` Authors: - Muhammad Haseeb (https://github.com/mhaseeb123) Approvers: - Yunsong Wang (https://github.com/PointKernel) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/16541 --- cpp/src/io/parquet/chunk_dict.cu | 370 ++++++++++++++++------------- cpp/src/io/parquet/parquet_gpu.cuh | 73 +++++- cpp/src/io/parquet/parquet_gpu.hpp | 44 +--- cpp/src/io/parquet/writer_impl.cu | 42 ++-- 4 files changed, 295 insertions(+), 234 deletions(-) diff --git a/cpp/src/io/parquet/chunk_dict.cu b/cpp/src/io/parquet/chunk_dict.cu index a43c6d4cbb6..17ccb73c0a8 100644 --- a/cpp/src/io/parquet/chunk_dict.cu +++ b/cpp/src/io/parquet/chunk_dict.cu @@ -22,6 +22,7 @@ #include +#include #include namespace cudf::io::parquet::detail { @@ -30,28 +31,14 @@ namespace { constexpr int DEFAULT_BLOCK_SIZE = 256; } -template -CUDF_KERNEL void __launch_bounds__(block_size) - initialize_chunk_hash_maps_kernel(device_span chunks) -{ - auto const chunk = chunks[blockIdx.x]; - auto const t = threadIdx.x; - // fut: Now that per-chunk dict is same size as ck.num_values, try to not use one block per chunk - for (thread_index_type i = 0; i < chunk.dict_map_size; i += block_size) { - if (t + i < chunk.dict_map_size) { - new (&chunk.dict_map_slots[t + i].first) map_type::atomic_key_type{KEY_SENTINEL}; - new (&chunk.dict_map_slots[t + i].second) map_type::atomic_mapped_type{VALUE_SENTINEL}; - } - } -} - template struct equality_functor { column_device_view const& col; - __device__ bool operator()(size_type lhs_idx, size_type rhs_idx) + __device__ bool operator()(key_type lhs_idx, key_type rhs_idx) const { - // We don't call this for nulls so this is fine - auto const equal = cudf::experimental::row::equality::nan_equal_physical_equality_comparator{}; + // We don't call this for nulls so this is fine. + auto constexpr equal = + cudf::experimental::row::equality::nan_equal_physical_equality_comparator{}; return equal(col.element(lhs_idx), col.element(rhs_idx)); } }; @@ -59,38 +46,167 @@ struct equality_functor { template struct hash_functor { column_device_view const& col; - __device__ auto operator()(size_type idx) const + uint32_t const seed = 0; + __device__ auto operator()(key_type idx) const { - return cudf::hashing::detail::MurmurHash3_x86_32{}(col.element(idx)); + return cudf::hashing::detail::MurmurHash3_x86_32{seed}(col.element(idx)); } }; +template struct map_insert_fn { - map_type::device_mutable_view& map; + storage_ref_type const& storage_ref; + EncColumnChunk* const& chunk; template - __device__ bool operator()(column_device_view const& col, size_type i) + __device__ void operator()(size_type const s_start_value_idx, size_type const end_value_idx) { if constexpr (column_device_view::has_element_accessor()) { - auto hash_fn = hash_functor{col}; - auto equality_fn = equality_functor{col}; - return map.insert(std::pair(i, i), hash_fn, equality_fn); + using block_reduce = cub::BlockReduce; + __shared__ typename block_reduce::TempStorage reduce_storage; + + auto const col = chunk->col_desc; + column_device_view const& data_col = *col->leaf_column; + __shared__ size_type total_num_dict_entries; + + using equality_fn_type = equality_functor; + using hash_fn_type = hash_functor; + // Choosing `linear_probing` over `double_hashing` for slighhhtly better performance seen in + // benchmarks. + using probing_scheme_type = cuco::linear_probing; + + // Make a view of the hash map. + auto hash_map_ref = cuco::static_map_ref{cuco::empty_key{KEY_SENTINEL}, + cuco::empty_value{VALUE_SENTINEL}, + equality_fn_type{data_col}, + probing_scheme_type{hash_fn_type{data_col}}, + cuco::thread_scope_block, + storage_ref}; + + // Create a map ref with `cuco::insert` operator + auto map_insert_ref = hash_map_ref.with_operators(cuco::insert); + auto const t = threadIdx.x; + + // Create atomic refs to the current chunk's num_dict_entries and uniq_data_size + cuda::atomic_ref const chunk_num_dict_entries{chunk->num_dict_entries}; + cuda::atomic_ref const chunk_uniq_data_size{chunk->uniq_data_size}; + + // Note: Adjust the following loop to use `cg::tile` if needed in the future. + for (thread_index_type val_idx = s_start_value_idx + t; val_idx - t < end_value_idx; + val_idx += block_size) { + size_type is_unique = 0; + size_type uniq_elem_size = 0; + + // Check if this index is valid. + auto const is_valid = + val_idx < end_value_idx and val_idx < data_col.size() and data_col.is_valid(val_idx); + + // Insert tile_val_idx to hash map and count successful insertions. + if (is_valid) { + // Insert the keys using a single thread for best performance for now. + is_unique = map_insert_ref.insert(cuco::pair{val_idx, val_idx}); + uniq_elem_size = [&]() -> size_type { + if (not is_unique) { return 0; } + switch (col->physical_type) { + case Type::INT32: return 4; + case Type::INT64: return 8; + case Type::INT96: return 12; + case Type::FLOAT: return 4; + case Type::DOUBLE: return 8; + case Type::BYTE_ARRAY: { + auto const col_type = data_col.type().id(); + if (col_type == type_id::STRING) { + // Strings are stored as 4 byte length + string bytes + return 4 + data_col.element(val_idx).size_bytes(); + } else if (col_type == type_id::LIST) { + // Binary is stored as 4 byte length + bytes + return 4 + + get_element(data_col, val_idx).size_bytes(); + } + CUDF_UNREACHABLE( + "Byte array only supports string and list column types for dictionary " + "encoding!"); + } + case Type::FIXED_LEN_BYTE_ARRAY: + if (data_col.type().id() == type_id::DECIMAL128) { return sizeof(__int128_t); } + CUDF_UNREACHABLE( + "Fixed length byte array only supports decimal 128 column types for dictionary " + "encoding!"); + default: CUDF_UNREACHABLE("Unsupported type for dictionary encoding"); + } + }(); + } + // Reduce num_unique and uniq_data_size from all tiles. + auto num_unique = block_reduce(reduce_storage).Sum(is_unique); + __syncthreads(); + auto uniq_data_size = block_reduce(reduce_storage).Sum(uniq_elem_size); + // The first thread in the block atomically updates total num_unique and uniq_data_size + if (t == 0) { + total_num_dict_entries = + chunk_num_dict_entries.fetch_add(num_unique, cuda::std::memory_order_relaxed); + total_num_dict_entries += num_unique; + chunk_uniq_data_size.fetch_add(uniq_data_size, cuda::std::memory_order_relaxed); + } + __syncthreads(); + + // Check if the num unique values in chunk has already exceeded max dict size and early exit + if (total_num_dict_entries > MAX_DICT_SIZE) { return; } + } // for loop } else { CUDF_UNREACHABLE("Unsupported type to insert in map"); } } }; +template struct map_find_fn { - map_type::device_view& map; - + storage_ref_type const& storage_ref; + EncColumnChunk* const& chunk; template - __device__ map_type::device_view::iterator operator()(column_device_view const& col, size_type i) + __device__ void operator()(size_type const s_start_value_idx, + size_type const end_value_idx, + size_type const s_ck_start_val_idx) { if constexpr (column_device_view::has_element_accessor()) { - auto hash_fn = hash_functor{col}; - auto equality_fn = equality_functor{col}; - return map.find(i, hash_fn, equality_fn); + auto const col = chunk->col_desc; + column_device_view const& data_col = *col->leaf_column; + + using equality_fn_type = equality_functor; + using hash_fn_type = hash_functor; + // Choosing `linear_probing` over `double_hashing` for slighhhtly better performance seen in + // benchmarks. + using probing_scheme_type = cuco::linear_probing; + + // Make a view of the hash map. + auto hash_map_ref = cuco::static_map_ref{cuco::empty_key{KEY_SENTINEL}, + cuco::empty_value{VALUE_SENTINEL}, + equality_fn_type{data_col}, + probing_scheme_type{hash_fn_type{data_col}}, + cuco::thread_scope_block, + storage_ref}; + + // Create a map ref with `cuco::find` operator + auto const map_find_ref = hash_map_ref.with_operators(cuco::find); + auto const t = threadIdx.x; + + // Note: Adjust the following loop to use `cg::tiles` if needed in the future. + for (thread_index_type val_idx = s_start_value_idx + t; val_idx < end_value_idx; + val_idx += block_size) { + // Find the key using a single thread for best performance for now. + if (data_col.is_valid(val_idx)) { + // No need for atomic as this is not going to be modified by any other thread. + chunk->dict_index[val_idx - s_ck_start_val_idx] = [&]() { + auto const found_slot = map_find_ref.find(val_idx); + + // Fail if we didn't find the previously inserted key. + cudf_assert(found_slot != map_find_ref.end() && + "Unable to find value in map in dictionary index construction"); + + // Return the found value. + return found_slot->second; + }(); + } + } } else { CUDF_UNREACHABLE("Unsupported type to find in map"); } @@ -99,124 +215,61 @@ struct map_find_fn { template CUDF_KERNEL void __launch_bounds__(block_size) - populate_chunk_hash_maps_kernel(cudf::detail::device_2dspan frags) + populate_chunk_hash_maps_kernel(device_span const map_storage, + cudf::detail::device_2dspan frags) { - auto col_idx = blockIdx.y; - auto block_x = blockIdx.x; - auto t = threadIdx.x; - auto frag = frags[col_idx][block_x]; - auto chunk = frag.chunk; - auto col = chunk->col_desc; + auto const col_idx = blockIdx.y; + auto const block_x = blockIdx.x; + auto const frag = frags[col_idx][block_x]; + auto chunk = frag.chunk; + auto col = chunk->col_desc; if (not chunk->use_dictionary) { return; } - using block_reduce = cub::BlockReduce; - __shared__ typename block_reduce::TempStorage reduce_storage; - size_type start_row = frag.start_row; size_type end_row = frag.start_row + frag.num_rows; - // Find the bounds of values in leaf column to be inserted into the map for current chunk + // Find the bounds of values in leaf column to be inserted into the map for current chunk. size_type const s_start_value_idx = row_to_value_idx(start_row, *col); size_type const end_value_idx = row_to_value_idx(end_row, *col); column_device_view const& data_col = *col->leaf_column; - - // Make a view of the hash map - auto hash_map_mutable = map_type::device_mutable_view(chunk->dict_map_slots, - chunk->dict_map_size, - cuco::empty_key{KEY_SENTINEL}, - cuco::empty_value{VALUE_SENTINEL}); - - __shared__ size_type total_num_dict_entries; - thread_index_type val_idx = s_start_value_idx + t; - while (val_idx - block_size < end_value_idx) { - auto const is_valid = - val_idx < end_value_idx and val_idx < data_col.size() and data_col.is_valid(val_idx); - - // insert element at val_idx to hash map and count successful insertions - size_type is_unique = 0; - size_type uniq_elem_size = 0; - if (is_valid) { - is_unique = - type_dispatcher(data_col.type(), map_insert_fn{hash_map_mutable}, data_col, val_idx); - uniq_elem_size = [&]() -> size_type { - if (not is_unique) { return 0; } - switch (col->physical_type) { - case Type::INT32: return 4; - case Type::INT64: return 8; - case Type::INT96: return 12; - case Type::FLOAT: return 4; - case Type::DOUBLE: return 8; - case Type::BYTE_ARRAY: { - auto const col_type = data_col.type().id(); - if (col_type == type_id::STRING) { - // Strings are stored as 4 byte length + string bytes - return 4 + data_col.element(val_idx).size_bytes(); - } else if (col_type == type_id::LIST) { - // Binary is stored as 4 byte length + bytes - return 4 + get_element(data_col, val_idx).size_bytes(); - } - CUDF_UNREACHABLE( - "Byte array only supports string and list column types for dictionary " - "encoding!"); - } - case Type::FIXED_LEN_BYTE_ARRAY: - if (data_col.type().id() == type_id::DECIMAL128) { return sizeof(__int128_t); } - CUDF_UNREACHABLE( - "Fixed length byte array only supports decimal 128 column types for dictionary " - "encoding!"); - default: CUDF_UNREACHABLE("Unsupported type for dictionary encoding"); - } - }(); - } - - auto num_unique = block_reduce(reduce_storage).Sum(is_unique); - __syncthreads(); - auto uniq_data_size = block_reduce(reduce_storage).Sum(uniq_elem_size); - if (t == 0) { - total_num_dict_entries = atomicAdd(&chunk->num_dict_entries, num_unique); - total_num_dict_entries += num_unique; - atomicAdd(&chunk->uniq_data_size, uniq_data_size); - } - __syncthreads(); - - // Check if the num unique values in chunk has already exceeded max dict size and early exit - if (total_num_dict_entries > MAX_DICT_SIZE) { return; } - - val_idx += block_size; - } // while + storage_ref_type const storage_ref{chunk->dict_map_size, + map_storage.data() + chunk->dict_map_offset}; + type_dispatcher(data_col.type(), + map_insert_fn{storage_ref, chunk}, + s_start_value_idx, + end_value_idx); } template CUDF_KERNEL void __launch_bounds__(block_size) - collect_map_entries_kernel(device_span chunks) + collect_map_entries_kernel(device_span const map_storage, + device_span chunks) { auto& chunk = chunks[blockIdx.x]; if (not chunk.use_dictionary) { return; } - auto t = threadIdx.x; - auto map = map_type::device_view(chunk.dict_map_slots, - chunk.dict_map_size, - cuco::empty_key{KEY_SENTINEL}, - cuco::empty_value{VALUE_SENTINEL}); - - __shared__ cuda::atomic counter; + auto t = threadIdx.x; + __shared__ cuda::atomic counter; using cuda::std::memory_order_relaxed; - if (t == 0) { new (&counter) cuda::atomic{0}; } + if (t == 0) { new (&counter) cuda::atomic{0}; } __syncthreads(); - for (size_type i = 0; i < chunk.dict_map_size; i += block_size) { - if (t + i < chunk.dict_map_size) { - auto* slot = reinterpret_cast(map.begin_slot() + t + i); - auto key = slot->first; + + // Iterate over all windows in the map. + for (; t < chunk.dict_map_size; t += block_size) { + auto window = map_storage.data() + chunk.dict_map_offset + t; + // Collect all slots from each window. + for (auto& slot : *window) { + auto const key = slot.first; if (key != KEY_SENTINEL) { - auto loc = counter.fetch_add(1, memory_order_relaxed); + auto const loc = counter.fetch_add(1, memory_order_relaxed); cudf_assert(loc < MAX_DICT_SIZE && "Number of filled slots exceeds max dict size"); chunk.dict_data[loc] = key; - // If sorting dict page ever becomes a hard requirement, enable the following statement and - // add a dict sorting step before storing into the slot's second field. - // chunk.dict_data_idx[loc] = t + i; - slot->second = loc; + // If sorting dict page ever becomes a hard requirement, enable the following statement + // and add a dict sorting step before storing into the slot's second field. + // chunk.dict_data_idx[loc] = idx; + slot.second = loc; } } } @@ -224,75 +277,60 @@ CUDF_KERNEL void __launch_bounds__(block_size) template CUDF_KERNEL void __launch_bounds__(block_size) - get_dictionary_indices_kernel(cudf::detail::device_2dspan frags) + get_dictionary_indices_kernel(device_span const map_storage, + cudf::detail::device_2dspan frags) { - auto col_idx = blockIdx.y; - auto block_x = blockIdx.x; - auto t = threadIdx.x; - auto frag = frags[col_idx][block_x]; - auto chunk = frag.chunk; - auto col = chunk->col_desc; + auto const col_idx = blockIdx.y; + auto const block_x = blockIdx.x; + auto const frag = frags[col_idx][block_x]; + auto chunk = frag.chunk; if (not chunk->use_dictionary) { return; } size_type start_row = frag.start_row; size_type end_row = frag.start_row + frag.num_rows; + auto const col = chunk->col_desc; // Find the bounds of values in leaf column to be searched in the map for current chunk auto const s_start_value_idx = row_to_value_idx(start_row, *col); auto const s_ck_start_val_idx = row_to_value_idx(chunk->start_row, *col); auto const end_value_idx = row_to_value_idx(end_row, *col); column_device_view const& data_col = *col->leaf_column; - - auto map = map_type::device_view(chunk->dict_map_slots, - chunk->dict_map_size, - cuco::empty_key{KEY_SENTINEL}, - cuco::empty_value{VALUE_SENTINEL}); - - thread_index_type val_idx = s_start_value_idx + t; - while (val_idx < end_value_idx) { - if (data_col.is_valid(val_idx)) { - auto found_slot = type_dispatcher(data_col.type(), map_find_fn{map}, data_col, val_idx); - cudf_assert(found_slot != map.end() && - "Unable to find value in map in dictionary index construction"); - if (found_slot != map.end()) { - // No need for atomic as this is not going to be modified by any other thread - auto* val_ptr = reinterpret_cast(&found_slot->second); - chunk->dict_index[val_idx - s_ck_start_val_idx] = *val_ptr; - } - } - - val_idx += block_size; - } -} - -void initialize_chunk_hash_maps(device_span chunks, rmm::cuda_stream_view stream) -{ - constexpr int block_size = 1024; - initialize_chunk_hash_maps_kernel - <<>>(chunks); + storage_ref_type const storage_ref{chunk->dict_map_size, + map_storage.data() + chunk->dict_map_offset}; + + type_dispatcher(data_col.type(), + map_find_fn{storage_ref, chunk}, + s_start_value_idx, + end_value_idx, + s_ck_start_val_idx); } -void populate_chunk_hash_maps(cudf::detail::device_2dspan frags, +void populate_chunk_hash_maps(device_span const map_storage, + cudf::detail::device_2dspan frags, rmm::cuda_stream_view stream) { dim3 const dim_grid(frags.size().second, frags.size().first); populate_chunk_hash_maps_kernel - <<>>(frags); + <<>>(map_storage, frags); } -void collect_map_entries(device_span chunks, rmm::cuda_stream_view stream) +void collect_map_entries(device_span const map_storage, + device_span chunks, + rmm::cuda_stream_view stream) { constexpr int block_size = 1024; - collect_map_entries_kernel<<>>(chunks); + collect_map_entries_kernel + <<>>(map_storage, chunks); } -void get_dictionary_indices(cudf::detail::device_2dspan frags, +void get_dictionary_indices(device_span const map_storage, + cudf::detail::device_2dspan frags, rmm::cuda_stream_view stream) { dim3 const dim_grid(frags.size().second, frags.size().first); get_dictionary_indices_kernel - <<>>(frags); + <<>>(map_storage, frags); } } // namespace cudf::io::parquet::detail diff --git a/cpp/src/io/parquet/parquet_gpu.cuh b/cpp/src/io/parquet/parquet_gpu.cuh index e3c44c78898..7c09764da2d 100644 --- a/cpp/src/io/parquet/parquet_gpu.cuh +++ b/cpp/src/io/parquet/parquet_gpu.cuh @@ -18,25 +18,37 @@ #include "parquet_gpu.hpp" +#include #include #include -#include +#include +#include namespace cudf::io::parquet::detail { -auto constexpr KEY_SENTINEL = size_type{-1}; -auto constexpr VALUE_SENTINEL = size_type{-1}; +using key_type = size_type; +using mapped_type = size_type; +using slot_type = cuco::pair; -using map_type = cuco::legacy::static_map; +auto constexpr map_cg_size = + 1; ///< A CUDA Cooperative Group of 1 thread (set for best performance) to handle each subset. + ///< Note: Adjust insert and find loops to use `cg::tile` if increasing this. +auto constexpr window_size = + 1; ///< Number of concurrent slots (set for best performance) handled by each thread. +auto constexpr occupancy_factor = 1.43f; ///< cuCollections suggests using a hash map of size + ///< N * (1/0.7) = 1.43 to target a 70% occupancy factor. -/** - * @brief The alias of `map_type::pair_atomic_type` class. - * - * Declare this struct by trivial subclassing instead of type aliasing so we can have forward - * declaration of this struct somewhere else. - */ -struct slot_type : public map_type::pair_atomic_type {}; +auto constexpr KEY_SENTINEL = key_type{-1}; +auto constexpr VALUE_SENTINEL = mapped_type{-1}; +auto constexpr SCOPE = cuda::thread_scope_block; + +using storage_type = cuco::aow_storage, + cudf::detail::cuco_allocator>; +using storage_ref_type = typename storage_type::ref_type; +using window_type = typename storage_type::window_type; /** * @brief Return the byte length of parquet dtypes that are physically represented by INT32 @@ -81,4 +93,43 @@ inline size_type __device__ row_to_value_idx(size_type idx, return idx; } +/** + * @brief Insert chunk values into their respective hash maps + * + * @param map_storage Bulk hashmap storage + * @param frags Column fragments + * @param stream CUDA stream to use + */ +void populate_chunk_hash_maps(device_span const map_storage, + cudf::detail::device_2dspan frags, + rmm::cuda_stream_view stream); + +/** + * @brief Compact dictionary hash map entries into chunk.dict_data + * + * @param map_storage Bulk hashmap storage + * @param chunks Flat span of chunks to compact hash maps for + * @param stream CUDA stream to use + */ +void collect_map_entries(device_span const map_storage, + device_span chunks, + rmm::cuda_stream_view stream); + +/** + * @brief Get the Dictionary Indices for each row + * + * For each row of a chunk, gets the indices into chunk.dict_data which contains the value otherwise + * stored in input column [row]. Stores these indices into chunk.dict_index. + * + * Since dict_data itself contains indices into the original cudf column, this means that + * col[row] == col[dict_data[dict_index[row - chunk.start_row]]] + * + * @param map_storage Bulk hashmap storage + * @param frags Column fragments + * @param stream CUDA stream to use + */ +void get_dictionary_indices(device_span const map_storage, + cudf::detail::device_2dspan frags, + rmm::cuda_stream_view stream); + } // namespace cudf::io::parquet::detail diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 8f52f073833..125d35f6499 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -514,7 +514,6 @@ constexpr unsigned int kDictHashBits = 16; constexpr size_t kDictScratchSize = (1 << kDictHashBits) * sizeof(uint32_t); struct EncPage; -struct slot_type; // convert Encoding to a mask value constexpr uint32_t encoding_to_mask(Encoding encoding) @@ -560,7 +559,8 @@ struct EncColumnChunk { uint8_t is_compressed; //!< Nonzero if the chunk uses compression uint32_t dictionary_size; //!< Size of dictionary page including header uint32_t ck_stat_size; //!< Size of chunk-level statistics (included in 1st page header) - slot_type* dict_map_slots; //!< Hash map storage for calculating dict encoding for this chunk + uint32_t dict_map_offset; //!< Offset of the hash map storage for calculating dict encoding for + //!< this chunk size_type dict_map_size; //!< Size of dict_map_slots size_type num_dict_entries; //!< Total number of entries in dictionary size_type @@ -1001,46 +1001,6 @@ void InitFragmentStatistics(device_span groups, device_span fragments, rmm::cuda_stream_view stream); -/** - * @brief Initialize per-chunk hash maps used for dictionary with sentinel values - * - * @param chunks Flat span of chunks to initialize hash maps for - * @param stream CUDA stream to use - */ -void initialize_chunk_hash_maps(device_span chunks, rmm::cuda_stream_view stream); - -/** - * @brief Insert chunk values into their respective hash maps - * - * @param frags Column fragments - * @param stream CUDA stream to use - */ -void populate_chunk_hash_maps(cudf::detail::device_2dspan frags, - rmm::cuda_stream_view stream); - -/** - * @brief Compact dictionary hash map entries into chunk.dict_data - * - * @param chunks Flat span of chunks to compact hash maps for - * @param stream CUDA stream to use - */ -void collect_map_entries(device_span chunks, rmm::cuda_stream_view stream); - -/** - * @brief Get the Dictionary Indices for each row - * - * For each row of a chunk, gets the indices into chunk.dict_data which contains the value otherwise - * stored in input column [row]. Stores these indices into chunk.dict_index. - * - * Since dict_data itself contains indices into the original cudf column, this means that - * col[row] == col[dict_data[dict_index[row - chunk.start_row]]] - * - * @param frags Column fragments - * @param stream CUDA stream to use - */ -void get_dictionary_indices(cudf::detail::device_2dspan frags, - rmm::cuda_stream_view stream); - /** * @brief Launches kernel for initializing encoder data pages * diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 74992aa733f..46c3151c731 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -1285,10 +1285,10 @@ build_chunk_dictionaries(hostdevice_2dvector& chunks, return std::pair(std::move(dict_data), std::move(dict_index)); } - // Allocate slots for each chunk - std::vector> hash_maps_storage; - hash_maps_storage.reserve(h_chunks.size()); - for (auto& chunk : h_chunks) { + // Variable to keep track of the current total map storage size + size_t total_map_storage_size = 0; + // Populate dict offsets and sizes for each chunk that need to build a dictionary. + std::for_each(h_chunks.begin(), h_chunks.end(), [&](auto& chunk) { auto const& chunk_col_desc = col_desc[chunk.col_desc_id]; auto const is_requested_non_dict = chunk_col_desc.requested_encoding != column_encoding::USE_DEFAULT && @@ -1300,19 +1300,31 @@ build_chunk_dictionaries(hostdevice_2dvector& chunks, chunk.use_dictionary = false; } else { chunk.use_dictionary = true; - // cuCollections suggests using a hash map of size N * (1/0.7) = num_values * 1.43 - // https://github.com/NVIDIA/cuCollections/blob/3a49fc71/include/cuco/static_map.cuh#L190-L193 - auto& inserted_map = hash_maps_storage.emplace_back(chunk.num_values * 1.43, stream); - chunk.dict_map_slots = inserted_map.data(); - chunk.dict_map_size = inserted_map.size(); + chunk.dict_map_size = + static_cast(cuco::make_window_extent( + static_cast(occupancy_factor * chunk.num_values))); + chunk.dict_map_offset = total_map_storage_size; + total_map_storage_size += chunk.dict_map_size; } - } + }); - chunks.host_to_device_async(stream); + // No chunk needs to create a dictionary, exit early + if (total_map_storage_size == 0) { return {std::move(dict_data), std::move(dict_index)}; } - initialize_chunk_hash_maps(chunks.device_view().flat_view(), stream); - populate_chunk_hash_maps(frags, stream); + // Create a single bulk storage used by all sub-dictionaries + auto map_storage = storage_type{ + total_map_storage_size, + cudf::detail::cuco_allocator{rmm::mr::polymorphic_allocator{}, stream}}; + // Create a span of non-const map_storage as map_storage_ref takes in a non-const pointer. + device_span const map_storage_data{map_storage.data(), total_map_storage_size}; + // Synchronize + chunks.host_to_device_async(stream); + // Initialize storage with the given sentinel + map_storage.initialize_async({KEY_SENTINEL, VALUE_SENTINEL}, {stream.value()}); + // Populate the hash map for each chunk + populate_chunk_hash_maps(map_storage_data, frags, stream); + // Synchronize again chunks.device_to_host_sync(stream); // Make decision about which chunks have dictionary @@ -1372,8 +1384,8 @@ build_chunk_dictionaries(hostdevice_2dvector& chunks, chunk.dict_index = inserted_dict_index.data(); } chunks.host_to_device_async(stream); - collect_map_entries(chunks.device_view().flat_view(), stream); - get_dictionary_indices(frags, stream); + collect_map_entries(map_storage_data, chunks.device_view().flat_view(), stream); + get_dictionary_indices(map_storage_data, frags, stream); return std::pair(std::move(dict_data), std::move(dict_index)); } From f932bf9c62f73aabee2ac094180036399ce88dcf Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:28:37 -1000 Subject: [PATCH 151/270] Fix Series.to_frame(name=None) setting a None name (#16698) In pandas 2.0, `to_frame(name=None)` allowed the resulting column name to be `None` https://github.com/pandas-dev/pandas/pull/45523 Looks like based on the current default of `cudf.Series.to_frame`, this behavior was not reflected. Additionally, created a `SingleColumnFrame._to_frame` to more easily share the logic between `Series.to_frame` and `Index.to_frame` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16698 --- python/cudf/cudf/core/_base_index.py | 58 -------------------- python/cudf/cudf/core/index.py | 57 +++++++++++++++++++ python/cudf/cudf/core/series.py | 12 +--- python/cudf/cudf/core/single_column_frame.py | 11 ++++ python/cudf/cudf/tests/test_series.py | 7 +++ 5 files changed, 77 insertions(+), 68 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index a224e0ce0d0..ff114474aa4 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -798,64 +798,6 @@ def fillna(self, value, downcast=None): return super().fillna(value=value) - def to_frame(self, index=True, name=no_default): - """Create a DataFrame with a column containing this Index - - Parameters - ---------- - index : boolean, default True - Set the index of the returned DataFrame as the original Index - name : object, defaults to index.name - The passed name should substitute for the index name (if it has - one). - - Returns - ------- - DataFrame - DataFrame containing the original Index data. - - See Also - -------- - Index.to_series : Convert an Index to a Series. - Series.to_frame : Convert Series to DataFrame. - - Examples - -------- - >>> import cudf - >>> idx = cudf.Index(['Ant', 'Bear', 'Cow'], name='animal') - >>> idx.to_frame() - animal - animal - Ant Ant - Bear Bear - Cow Cow - - By default, the original Index is reused. To enforce a new Index: - - >>> idx.to_frame(index=False) - animal - 0 Ant - 1 Bear - 2 Cow - - To override the name of the resulting column, specify `name`: - - >>> idx.to_frame(index=False, name='zoo') - zoo - 0 Ant - 1 Bear - 2 Cow - """ - - if name is no_default: - col_name = 0 if self.name is None else self.name - else: - col_name = name - - return cudf.DataFrame( - {col_name: self._values}, index=self if index else None - ) - def to_arrow(self): """Convert to a suitable Arrow object.""" raise NotImplementedError diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 66d03682de4..b2bd20c4982 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -529,6 +529,11 @@ def to_pandas( name=self.name, ) + def to_frame( + self, index: bool = True, name: Hashable = no_default + ) -> cudf.DataFrame: + return self._as_int_index().to_frame(index=index, name=name) + @property def is_unique(self) -> bool: return True @@ -1646,6 +1651,58 @@ def to_pandas( result.name = self.name return result + def to_frame( + self, index: bool = True, name: Hashable = no_default + ) -> cudf.DataFrame: + """Create a DataFrame with a column containing this Index + + Parameters + ---------- + index : boolean, default True + Set the index of the returned DataFrame as the original Index + name : object, defaults to index.name + The passed name should substitute for the index name (if it has + one). + + Returns + ------- + DataFrame + DataFrame containing the original Index data. + + See Also + -------- + Index.to_series : Convert an Index to a Series. + Series.to_frame : Convert Series to DataFrame. + + Examples + -------- + >>> import cudf + >>> idx = cudf.Index(['Ant', 'Bear', 'Cow'], name='animal') + >>> idx.to_frame() + animal + animal + Ant Ant + Bear Bear + Cow Cow + + By default, the original Index is reused. To enforce a new Index: + + >>> idx.to_frame(index=False) + animal + 0 Ant + 1 Bear + 2 Cow + + To override the name of the resulting column, specify `name`: + + >>> idx.to_frame(index=False, name='zoo') + zoo + 0 Ant + 1 Bear + 2 Cow + """ + return self._to_frame(name=name, index=self if index else None) + def append(self, other): if is_list_like(other): to_concat = [self] diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 837c6872258..aadbd80f4b4 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -1160,7 +1160,7 @@ def reset_index( ) @_performance_tracking - def to_frame(self, name=None): + def to_frame(self, name: abc.Hashable = no_default) -> cudf.DataFrame: """Convert Series into a DataFrame Parameters @@ -1192,15 +1192,7 @@ def to_frame(self, name=None): 13 15 d """ # noqa: E501 - - if name is not None: - col = name - elif self.name is None: - col = 0 - else: - col = self.name - - return cudf.DataFrame({col: self._column}, index=self.index) + return self._to_frame(name=name, index=self.index) @_performance_tracking def memory_usage(self, index=True, deep=False): diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index 55dda34a576..0e66f383ca0 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -158,6 +158,17 @@ def to_arrow(self) -> pa.Array: """ return self._column.to_arrow() + def _to_frame( + self, name: Hashable, index: cudf.Index | None + ) -> cudf.DataFrame: + """Helper function for Series.to_frame, Index.to_frame""" + if name is no_default: + col_name = 0 if self.name is None else self.name + else: + col_name = name + ca = ColumnAccessor({col_name: self._column}, verify=False) + return cudf.DataFrame._from_data(ca, index=index) + @property # type: ignore @_performance_tracking def is_unique(self) -> bool: diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 8d673e23ab2..a24002dc38e 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2557,6 +2557,13 @@ def test_series_arrow_list_types_roundtrip(): cudf.from_pandas(pdf) +@pytest.mark.parametrize("base_name", [None, "a"]) +def test_series_to_frame_none_name(base_name): + result = cudf.Series(range(1), name=base_name).to_frame(name=None) + expected = pd.Series(range(1), name=base_name).to_frame(name=None) + assert_eq(result, expected) + + @pytest.mark.parametrize("klass", [cudf.Index, cudf.Series]) @pytest.mark.parametrize( "data", [pa.array([float("nan")]), pa.chunked_array([[float("nan")]])] From 62a53b34f6c5c9145e908403d674cc6c16bab7f2 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Thu, 29 Aug 2024 21:54:02 -0400 Subject: [PATCH 152/270] [FEA] Add third-party library integration testing of cudf.pandas to cudf (#16645) Closes #16580 Authors: - Matthew Murray (https://github.com/Matt711) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16645 --- .github/workflows/test.yaml | 11 + ci/cudf_pandas_scripts/run_tests.sh | 3 + .../ci_run_library_tests.sh | 56 +++ .../third-party-integration/test.sh | 83 ++++ .../dependencies.yaml | 276 +++++++++++++ .../tests/conftest.py | 173 +++++++++ .../tests/pytest.ini | 7 + .../tests/test_cugraph.py | 94 +++++ .../tests/test_cuml.py | 152 ++++++++ .../tests/test_dask.py | 10 + .../tests/test_featureengine.py | 47 +++ .../tests/test_holoviews.py | 79 ++++ .../tests/test_hvplot.py | 72 ++++ .../tests/test_ibis.py | 169 ++++++++ .../tests/test_matplotlib.py | 70 ++++ .../tests/test_numpy.py | 59 +++ .../tests/test_plotly.py | 67 ++++ .../tests/test_pytorch.py | 128 ++++++ .../tests/test_scipy.py | 65 ++++ .../tests/test_seaborn.py | 60 +++ .../tests/test_sklearn.py | 82 ++++ .../tests/test_stumpy.py | 94 +++++ .../tests/test_stumpy_distributed.py | 48 +++ .../tests/test_tensorflow.py | 367 ++++++++++++++++++ .../tests/test_xgboost.py | 135 +++++++ 25 files changed, 2407 insertions(+) create mode 100755 ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh create mode 100755 ci/cudf_pandas_scripts/third-party-integration/test.sh create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/conftest.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cugraph.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cuml.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_dask.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_featureengine.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_holoviews.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_hvplot.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_ibis.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_matplotlib.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_numpy.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_plotly.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_pytorch.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_scipy.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_seaborn.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_sklearn.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy_distributed.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_tensorflow.py create mode 100644 python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_xgboost.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9feea050b19..2c68f2861bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -124,3 +124,14 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/cudf_pandas_scripts/run_tests.sh + third-party-integration-tests-cudf-pandas: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + container_image: "rapidsai/ci-conda:latest" + run_script: | + ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 52964496b36..8b85695c861 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -64,7 +64,9 @@ fi python -m pip install ipykernel python -m ipykernel install --user --name python3 +# The third-party integration tests are ignored because they are run nightly in seperate CI job python -m pytest -p cudf.pandas \ + --ignore=./python/cudf/cudf_pandas_tests/third_party_integration_tests/ \ --cov-config=./python/cudf/.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-pandas-coverage.xml" \ @@ -80,6 +82,7 @@ for version in "${versions[@]}"; do echo "Installing pandas version: ${version}" python -m pip install "numpy>=1.23,<2.0a0" "pandas==${version}" python -m pytest -p cudf.pandas \ + --ignore=./python/cudf/cudf_pandas_tests/third_party_integration_tests/ \ --cov-config=./python/cudf/.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-pandas-coverage.xml" \ diff --git a/ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh b/ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh new file mode 100755 index 00000000000..54a56508cdc --- /dev/null +++ b/ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cleanup() { + rm ${TEST_DIR}/results-*.pickle +} + +trap cleanup EXIT + +runtest_gold() { + local lib=$1 + + pytest \ + -v \ + --continue-on-collection-errors \ + --cache-clear \ + --numprocesses=${NUM_PROCESSES} \ + --dist=worksteal \ + ${TEST_DIR}/test_${lib}*.py +} + +runtest_cudf_pandas() { + local lib=$1 + + pytest \ + -p cudf.pandas \ + -v \ + --continue-on-collection-errors \ + --cache-clear \ + --numprocesses=${NUM_PROCESSES} \ + --dist=worksteal \ + ${TEST_DIR}/test_${lib}*.py +} + +main() { + local lib=$1 + + # generation phase + runtest_gold ${lib} + runtest_cudf_pandas ${lib} + + # assertion phase + pytest \ + --compare \ + -p cudf.pandas \ + -v \ + --continue-on-collection-errors \ + --cache-clear \ + --numprocesses=${NUM_PROCESSES} \ + --dist=worksteal \ + ${TEST_DIR}/test_${lib}*.py +} + +main $@ diff --git a/ci/cudf_pandas_scripts/third-party-integration/test.sh b/ci/cudf_pandas_scripts/third-party-integration/test.sh new file mode 100755 index 00000000000..89b28c30e39 --- /dev/null +++ b/ci/cudf_pandas_scripts/third-party-integration/test.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +# Common setup steps shared by Python test jobs + +set -euo pipefail + +write_output() { + local key="$1" + local value="$2" + echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}" +} + +extract_lib_from_dependencies_yaml() { + local file=$1 + # Parse all keys in dependencies.yaml under the "files" section, + # extract all the keys that start with "test_", and extract the rest + local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries(select(.key | contains("test_"))) | keys | map(sub("^test_"; ""))')" + echo $extracted_libs +} + +main() { + local dependencies_yaml="$1" + + LIBS=$(extract_lib_from_dependencies_yaml "$dependencies_yaml") + LIBS=${LIBS#[} + LIBS=${LIBS%]} + + for lib in ${LIBS//,/ }; do + lib=$(echo "$lib" | tr -d '""') + echo "Running tests for library $lib" + + CUDA_MAJOR=$(if [ "$lib" = "tensorflow" ]; then echo "11"; else echo "12"; fi) + + . /opt/conda/etc/profile.d/conda.sh + + rapids-logger "Generate Python testing dependencies" + rapids-dependency-file-generator \ + --config "$dependencies_yaml" \ + --output conda \ + --file-key test_${lib} \ + --matrix "cuda=${CUDA_MAJOR};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + + rapids-mamba-retry env create --yes -f env.yaml -n test + + # Temporarily allow unbound variables for conda activation. + set +u + conda activate test + set -u + + repo_root=$(git rev-parse --show-toplevel) + TEST_DIR=${repo_root}/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests + + rapids-print-env + + rapids-logger "Check GPU usage" + nvidia-smi + + EXITCODE=0 + trap "EXITCODE=1" ERR + set +e + + rapids-logger "pytest ${lib}" + + NUM_PROCESSES=8 + serial_libraries=( + "tensorflow" + ) + for serial_library in "${serial_libraries[@]}"; do + if [ "${lib}" = "${serial_library}" ]; then + NUM_PROCESSES=1 + fi + done + + TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${lib} + + rapids-logger "Test script exiting with value: ${EXITCODE}" + done + + exit ${EXITCODE} +} + +main "$@" diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml b/python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml new file mode 100644 index 00000000000..05e1d8178d5 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml @@ -0,0 +1,276 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Dependency list for https://github.com/rapidsai/dependency-file-generator +files: + checks: + output: none + includes: + - develop + - py_version + test_dask: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_dask + test_matplotlib: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_matplotlib + test_numpy: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_numpy + test_pytorch: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_pytorch + test_seaborn: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_seaborn + test_scipy: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_scipy + test_sklearn: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_sklearn + test_stumpy: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_stumpy + test_tensorflow: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_tensorflow + test_xgboost: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_xgboost + test_cuml: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_cuml + test_cugraph: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_cugraph + test_ibis: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_ibis + test_hvplot: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_hvplot + test_holoviews: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_holoviews + test_plotly: + output: none + includes: + - cuda_version + - py_version + - test_base + - test_plotly + +channels: + - rapidsai-nightly + - rapidsai + - conda-forge + - nvidia + +dependencies: + develop: + common: + - output_types: conda + packages: + - pre-commit + cuda_version: + specific: + - output_types: conda + matrices: + - matrix: + cuda: "11" + packages: + - cuda-version=11.8 + - matrix: + cuda: "11.8" + packages: + - cuda-version=11.8 + - matrix: + cuda: "12.0" + packages: + - cuda-version=12.0 + - matrix: + cuda: "12.2" + packages: + - cuda-version=12.2 + - matrix: + cuda: "12.5" + packages: + - cuda-version=12.5 + - matrix: + cuda: "12" + packages: + - cuda-version=12.5 + py_version: + specific: + - output_types: conda + matrices: + - matrix: + py: "3.10" + packages: + - python=3.10 + - matrix: + py: "3.11" + packages: + - python=3.11 + - matrix: + packages: + - python>=3.10,<3.12 + test_base: + common: + - output_types: conda + packages: + - cudf==24.10.*,>=0.0.0a0 + - pandas + - pytest + - pytest-xdist + test_dask: + common: + - output_types: conda + packages: + - dask + test_matplotlib: + common: + - output_types: conda + packages: + - matplotlib-base + test_numpy: + common: + - output_types: conda + packages: + - numpy + test_pytorch: + common: + - output_types: conda + packages: + - numpy + - pytorch>=2.1.0 + test_seaborn: + common: + - output_types: conda + packages: + - seaborn + test_scipy: + common: + - output_types: conda + packages: + - scipy + test_sklearn: + common: + - output_types: conda + packages: + - scikit-learn + test_stumpy: + common: + - output_types: conda + packages: + - dask + - stumpy + test_tensorflow: + common: + - output_types: conda + packages: + - tensorflow + test_xgboost: + common: + - output_types: conda + packages: + - hypothesis + - numpy + - scipy + - scikit-learn + - pip + - pip: + - xgboost>=2.0.1 + test_cuml: + common: + - output_types: conda + packages: + - cuml==24.10.*,>=0.0.0a0 + - scikit-learn + test_cugraph: + common: + - output_types: conda + packages: + - cugraph==24.10.*,>=0.0.0a0 + - networkx + test_ibis: + common: + - output_types: conda + packages: + - pip + - pip: + - ibis-framework[pandas] + test_hvplot: + common: + - output_types: conda + packages: + - hvplot + test_holoviews: + common: + - output_types: conda + packages: + - holoviews + test_plotly: + common: + - output_types: conda + packages: + - plotly diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/conftest.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/conftest.py new file mode 100644 index 00000000000..33b6ffdbd5c --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/conftest.py @@ -0,0 +1,173 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +from __future__ import annotations + +import os +import pickle +from typing import TYPE_CHECKING, BinaryIO + +import _pytest +import _pytest.config +import _pytest.nodes +import pytest + +if TYPE_CHECKING: + import _pytest.python + +from _pytest.stash import StashKey + +from cudf.pandas.module_accelerator import disable_module_accelerator + +file_handle_key = StashKey[BinaryIO]() +basename_key = StashKey[str]() +test_folder_key = StashKey[str]() +results = StashKey[tuple[dict, dict]]() + + +def pytest_addoption(parser): + parser.addoption( + "--compare", + action="store_true", + default=False, + help="Run comparison step?", + ) + + +def read_results(f): + while True: + try: + yield pickle.load(f) + except EOFError: + return + + +def pytest_collection_modifyitems( + session, config: _pytest.config.Config, items: list[_pytest.nodes.Item] +): + if config.getoption("--compare"): + current_pass = "compare" + elif "cudf.pandas" in config.option.plugins: + current_pass = "cudf_pandas" + else: + current_pass = "gold" + + def swap_xfail(item: _pytest.nodes.Item, name: str): + """Replace custom `xfail_**` mark with a `xfail` mark having the same kwargs.""" + + old_mark = item.keywords[name] + new_mark = pytest.mark.xfail(**old_mark.kwargs) + + # Replace all "xfail_**" mark in the node chain with the "xfail" mark + # if not found, the node chain is not modified. + for node, mark in item.iter_markers_with_node(name): + idx = node.own_markers.index(mark) + node.own_markers[idx] = new_mark + + for item in items: + if current_pass == "gold" and "xfail_gold" in item.keywords: + swap_xfail(item, "xfail_gold") + elif ( + current_pass == "cudf_pandas" + and "xfail_cudf_pandas" in item.keywords + ): + swap_xfail(item, "xfail_cudf_pandas") + elif current_pass == "compare" and "xfail_compare" in item.keywords: + swap_xfail(item, "xfail_compare") + + +def pytest_configure(config: _pytest.config.Config): + gold_basename = "results-gold" + cudf_basename = "results-cudf-pandas" + test_folder = os.path.join(os.path.dirname(__file__)) + + if config.getoption("--compare"): + # Everyone reads everything + gold_path = os.path.join(test_folder, f"{gold_basename}.pickle") + cudf_path = os.path.join(test_folder, f"{cudf_basename}.pickle") + with disable_module_accelerator(): + with open(gold_path, "rb") as f: + gold_results = dict(read_results(f)) + with open(cudf_path, "rb") as f: + cudf_results = dict(read_results(f)) + config.stash[results] = (gold_results, cudf_results) + else: + if "cudf.pandas" in config.option.plugins: + basename = cudf_basename + else: + basename = gold_basename + + if hasattr(config, "workerinput"): + # If we're on an xdist worker, open a worker-unique pickle file. + worker = config.workerinput["workerid"] + filename = f"{basename}-{worker}.pickle" + else: + filename = f"{basename}.pickle" + + pickle_path = os.path.join(test_folder, filename) + config.stash[file_handle_key] = open(pickle_path, "wb") + config.stash[test_folder_key] = test_folder + config.stash[basename_key] = basename + + +def pytest_pyfunc_call(pyfuncitem: _pytest.python.Function): + if pyfuncitem.config.getoption("--compare"): + gold_results, cudf_results = pyfuncitem.config.stash[results] + key = pyfuncitem.nodeid + try: + gold = gold_results[key] + except KeyError: + assert False, "pickled gold result is not available" + try: + cudf = cudf_results[key] + except KeyError: + assert False, "pickled cudf result is not available" + if gold is None and cudf is None: + raise ValueError(f"Integration test {key} did not return a value") + asserter = pyfuncitem.get_closest_marker("assert_eq") + if asserter is None: + assert gold == cudf, "Test failed" + else: + asserter.kwargs["fn"](gold, cudf) + else: + # Replace default call of test function with one that captures the + # result + testfunction = pyfuncitem.obj + funcargs = pyfuncitem.funcargs + testargs = { + arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames + } + result = testfunction(**testargs) + # Tuple-based key-value pairs, key is the node-id + try: + pickle.dump( + (pyfuncitem.nodeid, result), + pyfuncitem.config.stash[file_handle_key], + ) + except pickle.PicklingError: + pass + return True + + +def pytest_unconfigure(config): + if config.getoption("--compare"): + return + if file_handle_key not in config.stash: + # We didn't open a pickle file + return + if not hasattr(config, "workerinput"): + # If we're the controlling process + if ( + hasattr(config.option, "numprocesses") + and config.option.numprocesses is not None + ): + # Concat the worker partial pickle results and remove them + for i in range(config.option.numprocesses): + worker_result = os.path.join( + config.stash[test_folder_key], + f"{config.stash[basename_key]}-gw{i}.pickle", + ) + with open(worker_result, "rb") as f: + config.stash[file_handle_key].write(f.read()) + os.remove(worker_result) + # Close our file + del config.stash[file_handle_key] diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini new file mode 100644 index 00000000000..817d98e6ba2 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +xfail_strict=true +markers= + assert_eq: custom binary asserter for a test + xfail_gold: this test is expected to fail in the gold pass + xfail_cudf_pandas: this test is expected to fail in the cudf_pandas pass + xfail_compare: this test is expected to fail in the comparison pass diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cugraph.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cugraph.py new file mode 100644 index 00000000000..7acc8672063 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cugraph.py @@ -0,0 +1,94 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import cugraph +import cupy as cp +import networkx as nx +import numpy as np +import pandas as pd +import pytest + +cugraph_algos = [ + "betweenness_centrality", + "degree_centrality", + "katz_centrality", + "sorensen_coefficient", + "jaccard_coefficient", +] + +nx_algos = [ + "betweenness_centrality", + "degree_centrality", + "katz_centrality", +] + + +def assert_cugraph_equal(expect, got): + if isinstance(expect, cp.ndarray): + expect = expect.get() + if isinstance(got, cp.ndarray): + got = got.get() + elif isinstance(expect, np.ndarray) and isinstance(got, np.ndarray): + assert np.array_equal(expect, got) + else: + assert expect == got + + +pytestmark = pytest.mark.assert_eq(fn=assert_cugraph_equal) + + +@pytest.fixture(scope="session") +def df(): + return pd.DataFrame({"source": [0, 1, 2], "destination": [1, 2, 3]}) + + +@pytest.fixture(scope="session") +def adjacency_matrix(): + data = { + "A": [0, 1, 1, 0], + "B": [1, 0, 0, 1], + "C": [1, 0, 0, 1], + "D": [0, 1, 1, 0], + } + df = pd.DataFrame(data, index=["A", "B", "C", "D"]) + return df + + +@pytest.mark.parametrize("algo", cugraph_algos) +def test_cugraph_from_pandas_edgelist(df, algo): + G = cugraph.Graph() + G.from_pandas_edgelist(df) + return getattr(cugraph, algo)(G).to_pandas().values + + +@pytest.mark.parametrize("algo", cugraph_algos) +def test_cugraph_from_pandas_adjacency(adjacency_matrix, algo): + G = cugraph.Graph() + G.from_pandas_adjacency(adjacency_matrix) + res = getattr(cugraph, algo)(G).to_pandas() + return res.sort_values(list(res.columns)).values + + +@pytest.mark.parametrize("algo", cugraph_algos) +def test_cugraph_from_numpy_array(df, algo): + G = cugraph.Graph() + G.from_numpy_array(df.values) + return getattr(cugraph, algo)(G).to_pandas().values + + +@pytest.mark.parametrize("algo", nx_algos) +def test_networkx_from_pandas_edgelist(df, algo): + G = nx.from_pandas_edgelist( + df, "source", "destination", ["source", "destination"] + ) + return getattr(nx, algo)(G) + + +@pytest.mark.parametrize("algo", nx_algos) +def test_networkx_from_pandas_adjacency(adjacency_matrix, algo): + G = nx.from_pandas_adjacency(adjacency_matrix) + return getattr(nx, algo)(G) + + +@pytest.mark.parametrize("algo", nx_algos) +def test_networkx_from_numpy_array(adjacency_matrix, algo): + G = nx.from_numpy_array(adjacency_matrix.values) + return getattr(nx, algo)(G) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cuml.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cuml.py new file mode 100644 index 00000000000..892d0886596 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cuml.py @@ -0,0 +1,152 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import cupy as cp +import numpy as np +import pandas as pd +import pytest +from cuml.cluster import KMeans +from cuml.decomposition import PCA +from cuml.ensemble import RandomForestClassifier +from cuml.linear_model import LinearRegression, LogisticRegression +from cuml.metrics import accuracy_score +from cuml.model_selection import train_test_split +from cuml.pipeline import Pipeline +from cuml.preprocessing import StandardScaler + + +def assert_cuml_equal(expect, got): + # Coerce GPU arrays to CPU + if isinstance(expect, cp.ndarray): + expect = expect.get() + if isinstance(got, cp.ndarray): + got = got.get() + + # Handle equality + if isinstance(expect, KMeans) and isinstance(got, KMeans): + # same clusters + np.testing.assert_allclose( + expect.cluster_centers_, got.cluster_centers_ + ) + elif isinstance(expect, np.ndarray) and isinstance(got, np.ndarray): + np.testing.assert_allclose(expect, got) + elif isinstance(expect, tuple) and isinstance(got, tuple): + assert len(expect) == len(got) + for e, g in zip(expect, got): + assert_cuml_equal(e, g) + elif isinstance(expect, pd.DataFrame): + assert pd.testing.assert_frame_equal(expect, got) + elif isinstance(expect, pd.Series): + assert pd.testing.assert_series_equal(expect, got) + else: + assert expect == got + + +pytestmark = pytest.mark.assert_eq(fn=assert_cuml_equal) + + +@pytest.fixture +def binary_classification_data(): + data = { + "feature1": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], + "feature2": [2.0, 4.0, 1.0, 3.0, 5.0, 7.0, 6.0, 8.0, 10.0, 9.0], + "target": [0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + } + df = pd.DataFrame(data) + return df + + +def test_linear_regression(): + lr = LinearRegression(fit_intercept=True, normalize=False, algorithm="eig") + X = pd.DataFrame() + X["col1"] = np.array([1, 1, 2, 2], dtype=np.float32) + X["col2"] = np.array([1, 2, 2, 3], dtype=np.float32) + y = pd.Series(np.array([6.0, 8.0, 9.0, 11.0], dtype=np.float32)) + lr.fit(X, y) + + X_new = pd.DataFrame() + X_new["col1"] = np.array([3, 2], dtype=np.float32) + X_new["col2"] = np.array([5, 5], dtype=np.float32) + preds = lr.predict(X_new) + return preds.values + + +def test_logistic_regression(binary_classification_data): + X = binary_classification_data[["feature1", "feature2"]] + y = binary_classification_data["target"] + + (X_train, X_test, y_train, y_test) = train_test_split( + X, y, test_size=0.2, random_state=42 + ) + + model = LogisticRegression() + model.fit(X_train, y_train) + + y_pred = model.predict(X_test) + accuracy = accuracy_score(y_test, y_pred) + + return accuracy + + +def test_random_forest(binary_classification_data): + X = binary_classification_data[["feature1", "feature2"]] + y = binary_classification_data["target"] + + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=42 + ) + model = RandomForestClassifier(n_estimators=100) + model.fit(X_train, y_train) + preds = model.predict(X_test) + return preds.values + + +def test_clustering(): + rng = np.random.default_rng(42) + nsamps = 300 + X = rng.random((nsamps, 2)) + data = pd.DataFrame(X, columns=["x", "y"]) + + kmeans = KMeans(n_clusters=3, random_state=42) + kmeans.fit(data) + return kmeans + + +def test_data_scaling(): + data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + scaler = StandardScaler() + + scaled_data = scaler.fit_transform(data.values.reshape(-1, 1)) + return scaled_data + + +def test_pipeline(binary_classification_data): + X = binary_classification_data[["feature1", "feature2"]] + y = binary_classification_data["target"] + + pipe = Pipeline( + [ + ("scaler", StandardScaler()), + ("pca", PCA()), + ("random_forest", LogisticRegression()), + ] + ) + + pipe.fit(X, y) + results = pipe.predict(X) + return results.values + + +@pytest.mark.parametrize( + "X, y", + [ + (pd.DataFrame({"a": range(10), "b": range(10)}), pd.Series(range(10))), + ( + pd.DataFrame({"a": range(10), "b": range(10)}).values, + pd.Series(range(10)).values, + ), # cudf.pandas wrapped numpy arrays + ], +) +def test_train_test_split(X, y): + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + + # Compare only the size of the data splits + return len(X_train), len(X_test), len(y_train), len(y_test) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_dask.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_dask.py new file mode 100644 index 00000000000..c34778dfded --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_dask.py @@ -0,0 +1,10 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import pandas as pd + +import dask.dataframe as dd + + +def test_sum(): + data = {"x": range(1, 11)} + ddf = dd.from_pandas(pd.DataFrame(data), npartitions=2) + return ddf["x"].sum().compute() diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_featureengine.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_featureengine.py new file mode 100644 index 00000000000..3e247291fad --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_featureengine.py @@ -0,0 +1,47 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import numpy as np +import pandas as pd +from feature_engine.imputation import DropMissingData +from feature_engine.preprocessing import MatchVariables + + +def test_drop_missing_data(): + data = { + "x": [np.nan, 1, 1, 0, np.nan], + "y": ["a", np.nan, "b", np.nan, "a"], + } + df = pd.DataFrame(data) + + dmd = DropMissingData() + dmd.fit(df) + dmd.transform(df) + + return dmd + + +def test_match_variables(): + train = pd.DataFrame( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + } + ) + + test = pd.DataFrame( + { + "Name": ["tom", "sam", "nick"], + "Age": [20, 22, 23], + "Marks": [0.9, 0.7, 0.6], + "Hobbies": ["tennis", "rugby", "football"], + } + ) + + match_columns = MatchVariables() + + match_columns.fit(train) + + df_transformed = match_columns.transform(test) + + return df_transformed diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_holoviews.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_holoviews.py new file mode 100644 index 00000000000..bef02c86355 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_holoviews.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import holoviews as hv +import numpy as np +import pandas as pd +import pytest + +nsamps = 1000 +hv.extension("bokeh") # load holoviews extension + + +def assert_holoviews_equal(expect, got): + expect_data, expect_ndims, expect_kdims, expect_vdims, expect_shape = ( + expect + ) + got_data, got_ndims, got_kdims, got_vdims, got_shape = got + + if isinstance(expect_data, dict): + np.testing.assert_allclose(expect_data["x"], got_data["x"]) + np.testing.assert_allclose( + expect_data["Frequency"], got_data["Frequency"] + ) + else: + pd._testing.assert_frame_equal(expect_data, got_data) + assert expect_ndims == got_ndims + assert expect_kdims == got_kdims + assert expect_vdims == got_vdims + assert expect_shape == got_shape + + +pytestmark = pytest.mark.assert_eq(fn=assert_holoviews_equal) + + +@pytest.fixture(scope="module") +def df(): + rng = np.random.default_rng(42) + return pd.DataFrame( + { + "x": rng.random(nsamps), + "y": rng.random(nsamps), + "category": rng.integers(0, 10, nsamps), + "category2": rng.integers(0, 10, nsamps), + } + ) + + +def get_plot_info(plot): + return ( + plot.data, + plot.ndims, + plot.kdims, + plot.vdims, + plot.shape, + ) + + +def test_holoviews_barplot(df): + return get_plot_info(hv.Bars(df, kdims="category", vdims="y")) + + +def test_holoviews_scatterplot(df): + return get_plot_info(hv.Scatter(df, kdims="x", vdims="y")) + + +def test_holoviews_curve(df): + return get_plot_info(hv.Curve(df, kdims="category", vdims="y")) + + +def test_holoviews_heatmap(df): + return get_plot_info( + hv.HeatMap(df, kdims=["category", "category2"], vdims="y") + ) + + +def test_holoviews_histogram(df): + return get_plot_info(hv.Histogram(df.values)) + + +def test_holoviews_hexbin(df): + return get_plot_info(hv.HexTiles(df, kdims=["x", "y"], vdims="y")) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_hvplot.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_hvplot.py new file mode 100644 index 00000000000..0f0d2f8bcbd --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_hvplot.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import hvplot.pandas # noqa: F401, needs to monkey patch pandas with this. +import numpy as np +import pandas as pd +import pytest + +nsamps = 1000 + + +def assert_hvplot_equal(expect, got): + expect_data, expect_ndims, expect_kdims, expect_vdims, expect_shape = ( + expect + ) + got_data, got_ndims, got_kdims, got_vdims, got_shape = got + + if isinstance(expect_data, dict): + np.testing.assert_allclose(expect_data["x"], got_data["x"]) + np.testing.assert_allclose( + expect_data["Frequency"], got_data["Frequency"] + ) + else: + pd._testing.assert_frame_equal(expect_data, got_data) + assert expect_ndims == got_ndims + assert expect_kdims == got_kdims + assert expect_vdims == got_vdims + assert expect_shape == got_shape + + +pytestmark = pytest.mark.assert_eq(fn=assert_hvplot_equal) + + +@pytest.fixture(scope="module") +def df(): + rng = np.random.default_rng(42) + return pd.DataFrame( + { + "x": rng.random(nsamps), + "y": rng.random(nsamps), + "category": rng.integers(0, 10, nsamps), + "category2": rng.integers(0, 10, nsamps), + } + ) + + +def get_plot_info(plot): + return ( + plot.data, + plot.ndims, + plot.kdims, + plot.vdims, + plot.shape, + ) + + +def test_hvplot_barplot(df): + return get_plot_info(df.hvplot.bar(x="category", y="y")) + + +def test_hvplot_scatterplot(df): + return get_plot_info(df.hvplot.scatter(x="x", y="y")) + + +def test_hvplot_lineplot(df): + return get_plot_info(df.hvplot.line(x="x", y="y")) + + +def test_hvplot_heatmap(df): + return get_plot_info(df.hvplot.heatmap(x="x", y="y", C="y")) + + +def test_hvplot_hexbin(df): + return get_plot_info(df.hvplot.hexbin(x="x", y="y", C="y")) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_ibis.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_ibis.py new file mode 100644 index 00000000000..2a8cf7c6ac2 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_ibis.py @@ -0,0 +1,169 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +import ibis +import numpy as np +import pandas as pd +import pytest + +ibis.set_backend("pandas") +ibis.options.interactive = False + + +def ibis_assert_equal(expect, got, rtol: float = 1e-7, atol: float = 0.0): + pd._testing.assert_almost_equal(expect, got, rtol=rtol, atol=atol) + + +pytestmark = pytest.mark.assert_eq(fn=ibis_assert_equal) + + +COLUMN_REDUCTIONS = ["sum", "min", "max", "mean", "var", "std"] +ELEMENTWISE_UFUNCS = [ + "sin", + "cos", + "atan", + "exp", + "log", + "abs", +] +STRING_UNARY_FUNCS = [ + "lower", + "upper", + "capitalize", + "reverse", +] + + +@pytest.fixture +def ibis_table_num_str(): + N = 1000 + K = 5 + rng = np.random.default_rng(42) + + df = pd.DataFrame( + rng.integers(0, 100, (N, K)), columns=[f"col{x}" for x in np.arange(K)] + ) + df["key"] = rng.choice(np.arange(10), N) + df["str_col"] = rng.choice(["Hello", "World", "It's", "Me", "Again"], N) + table = ibis.memtable(df, name="t") + return table + + +@pytest.fixture +def ibis_table_num(): + N = 100 + K = 2 + rng = np.random.default_rng(42) + + df = pd.DataFrame( + rng.integers(0, 100, (N, K)), columns=[f"val{x}" for x in np.arange(K)] + ) + df["key"] = rng.choice(np.arange(10), N) + table = ibis.memtable(df, name="t") + return table + + +@pytest.mark.parametrize("op", COLUMN_REDUCTIONS) +def test_column_reductions(ibis_table_num_str, op): + t = ibis_table_num_str + return getattr(t.col1, op)().to_pandas() + + +@pytest.mark.parametrize("op", ["mean", "sum", "min", "max"]) +def test_groupby_reductions(ibis_table_num_str, op): + t = ibis_table_num_str + return getattr(t.group_by("key").col1, op)().to_pandas() + + +@pytest.mark.parametrize("op", ELEMENTWISE_UFUNCS) +def test_mutate_ufunc(ibis_table_num_str, op): + t = ibis_table_num_str + expr = getattr(t.col1, op)() + return t.mutate(col1_sin=expr).to_pandas() + + +@pytest.mark.parametrize("op", STRING_UNARY_FUNCS) +def test_string_unary(ibis_table_num_str, op): + t = ibis_table_num_str + return getattr(t.str_col, op)().to_pandas() + + +def test_nunique(ibis_table_num_str): + t = ibis_table_num_str + return t.col1.nunique().to_pandas() + + +def test_count(ibis_table_num_str): + t = ibis_table_num_str + return t.col1.count().to_pandas() + + +def test_select(ibis_table_num_str): + t = ibis_table_num_str + return t.select("col0", "col1").to_pandas() + + +def test_between(ibis_table_num_str): + t = ibis_table_num_str + return t.key.between(4, 8).to_pandas() + + +def test_notin(ibis_table_num_str): + t = ibis_table_num_str + return t.key.notin([0, 1, 8, 3]).to_pandas() + + +def test_window(ibis_table_num_str): + t = ibis_table_num_str + return ( + t.group_by("key").mutate(demeaned=t.col1 - t.col1.mean()).to_pandas() + ) + + +def test_limit(ibis_table_num_str): + t = ibis_table_num_str + return t.limit(5).to_pandas() + + +def test_filter(ibis_table_num_str): + t = ibis_table_num_str + return t.filter([t.key == 4, t.col0 > 15]).to_pandas() + + +@pytest.mark.skip(reason="Join ordering not currently guaranteed, i.e., flaky") +@pytest.mark.parametrize("join_type", ["inner", "left", "right"]) +def test_join_exact_ordering(ibis_table_num_str, ibis_table_num, join_type): + t1 = ibis_table_num_str + t2 = ibis_table_num + res = t1.join(t2, "key", how=join_type).to_pandas() + return res + + +@pytest.mark.parametrize("join_type", ["inner", "left", "right"]) +def test_join_sort_correctness(ibis_table_num_str, ibis_table_num, join_type): + """ + While we don't currently guarantee exact row ordering + we can still test join correctness with ex-post sorting. + """ + t1 = ibis_table_num_str + t2 = ibis_table_num + res = t1.join(t2, "key", how=join_type).to_pandas() + + res_sorted = res.sort_values(by=res.columns.tolist()).reset_index( + drop=True + ) + return res_sorted + + +def test_order_by(ibis_table_num_str): + t = ibis_table_num_str + return t.order_by(ibis.desc("col1")).to_pandas() + + +def test_aggregate_having(ibis_table_num_str): + t = ibis_table_num_str + return t.aggregate( + by=["key"], + sum_c0=t.col0.sum(), + avg_c0=t.col0.mean(), + having=t.col1.mean() > 50, + ).to_pandas() diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_matplotlib.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_matplotlib.py new file mode 100644 index 00000000000..665b9d6fb08 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_matplotlib.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pytest +from matplotlib.axes import Axes +from matplotlib.collections import PathCollection +from matplotlib.lines import Line2D +from matplotlib.patches import Rectangle +from pandas._testing import assert_equal + + +def assert_plots_equal(expect, got): + if isinstance(expect, Axes) and isinstance(got, Axes): + for expect_ch, got_ch in zip( + expect.get_children(), got.get_children() + ): + assert type(expect_ch) == type(got_ch) + if isinstance(expect_ch, Line2D): + assert_equal(expect_ch.get_xdata(), got_ch.get_xdata()) + assert_equal(expect_ch.get_ydata(), got_ch.get_ydata()) + elif isinstance(expect_ch, Rectangle): + assert expect_ch.get_height() == got_ch.get_height() + elif isinstance(expect, PathCollection) and isinstance( + got, PathCollection + ): + assert_equal(expect.get_offsets()[:, 0], got.get_offsets()[:, 0]) + assert_equal(expect.get_offsets()[:, 1], got.get_offsets()[:, 1]) + else: + assert_equal(expect, got) + + +pytestmark = pytest.mark.assert_eq(fn=assert_plots_equal) + + +def test_line(): + df = pd.DataFrame({"x": [1, 2, 3, 4, 5], "y": [2, 4, 6, 8, 10]}) + (data,) = plt.plot(df["x"], df["y"], marker="o", linestyle="-") + + return plt.gca() + + +def test_bar(): + data = pd.Series([1, 2, 3, 4, 5], index=["a", "b", "c", "d", "e"]) + ax = data.plot(kind="bar") + return ax + + +def test_scatter(): + df = pd.DataFrame({"x": [1, 2, 3, 4, 5], "y": [5, 4, 3, 2, 1]}) + + fig, ax = plt.subplots(figsize=(8, 6)) + ax.scatter(df["x"], df["y"]) + + return plt.gca() + + +def test_dataframe_plot(): + rng = np.random.default_rng(42) + df = pd.DataFrame(rng.random((10, 5)), columns=["a", "b", "c", "d", "e"]) + ax = df.plot() + + return ax + + +def test_series_plot(): + sr = pd.Series([1, 2, 3, 4, 5]) + ax = sr.plot() + + return ax diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_numpy.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_numpy.py new file mode 100644 index 00000000000..472f1889354 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_numpy.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +import numpy as np +import pandas as pd +import pytest + +nsamps = 1000 +reductions = ["sum", "min", "max", "mean", "var", "std"] + + +pytestmark = pytest.mark.assert_eq(fn=np.testing.assert_allclose) + + +@pytest.fixture(scope="module") +def sr(): + rng = np.random.default_rng(42) + return pd.Series(rng.random(nsamps)) + + +@pytest.mark.parametrize("op", reductions) +def test_numpy_series_reductions(sr, op): + return getattr(np, op)(sr) + + +@pytest.fixture(scope="module") +def df(): + rng = np.random.default_rng(42) + return pd.DataFrame({"A": rng.random(nsamps), "B": rng.random(nsamps)}) + + +@pytest.mark.parametrize("op", reductions) +def test_numpy_dataframe_reductions(df, op): + return getattr(np, op)(df) + + +def test_numpy_dot(df): + return np.dot(df, df.T) + + +def test_numpy_fft(sr): + fft = np.fft.fft(sr) + return fft + + +def test_numpy_sort(df): + return np.sort(df) + + +@pytest.mark.parametrize("percentile", [0, 25, 50, 75, 100]) +def test_numpy_percentile(df, percentile): + return np.percentile(df, percentile) + + +def test_numpy_unique(df): + return np.unique(df) + + +def test_numpy_transpose(df): + return np.transpose(df) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_plotly.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_plotly.py new file mode 100644 index 00000000000..27d9df83476 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_plotly.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import numpy as np +import pandas as pd +import plotly.express as px +import pytest + +nsamps = 100 + + +def assert_plotly_equal(expect, got): + assert type(expect) == type(got) + if isinstance(expect, dict): + assert expect.keys() == got.keys() + for k in expect.keys(): + assert_plotly_equal(expect[k], got[k]) + elif isinstance(got, list): + assert len(expect) == len(got) + for i in range(len(expect)): + assert_plotly_equal(expect[i], got[i]) + elif isinstance(expect, np.ndarray): + np.testing.assert_allclose(expect, got) + else: + assert expect == got + + +pytestmark = pytest.mark.assert_eq(fn=assert_plotly_equal) + + +@pytest.fixture(scope="module") +def df(): + rng = np.random.default_rng(42) + return pd.DataFrame( + { + "x": rng.random(nsamps), + "y": rng.random(nsamps), + "category": rng.integers(0, 10, nsamps), + "category2": rng.integers(0, 10, nsamps), + } + ) + + +def test_plotly_scatterplot(df): + return px.scatter(df, x="x", y="y").to_plotly_json() + + +def test_plotly_lineplot(df): + return px.line(df, x="category", y="y").to_plotly_json() + + +def test_plotly_barplot(df): + return px.bar(df, x="category", y="y").to_plotly_json() + + +def test_plotly_histogram(df): + return px.histogram(df, x="category").to_plotly_json() + + +def test_plotly_pie(df): + return px.pie(df, values="category", names="category2").to_plotly_json() + + +def test_plotly_heatmap(df): + return px.density_heatmap(df, x="category", y="category2").to_plotly_json() + + +def test_plotly_boxplot(df): + return px.box(df, x="category", y="y").to_plotly_json() diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_pytorch.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_pytorch.py new file mode 100644 index 00000000000..ae9db3836a6 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_pytorch.py @@ -0,0 +1,128 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +import numpy as np +import pandas as pd +import pytest +import torch + +pytestmark = pytest.mark.assert_eq(fn=torch.testing.assert_close) + + +@pytest.fixture +def data(): + rng = np.random.default_rng(0) + x1 = rng.random(100, dtype=np.float32) + x2 = rng.random(100, dtype=np.float32) + y = np.zeros(100).astype(np.int64) + + y[(x1 > x2) & (x1 > 0)] = 0 + y[(x1 < x2) & (x1 > 0)] = 1 + y[(x1 > x2) & (x1 < 0)] = 2 + y[(x1 < x2) & (x1 < 0)] = 3 + + return x1, x2, y + + +class Dataset(torch.utils.data.Dataset): + def __init__(self, x1, x2, y): + self.x1 = x1 + self.x2 = x2 + self.y = y + + def __getitem__(self, idx): + x1 = self.x1[idx] + x2 = self.x2[idx] + y = self.y[idx] + return (x1, x2), y + + def __len__(self): + return len(self.x1) + + +def test_dataloader_auto_batching(data): + x1, x2, y = (pd.Series(i) for i in data) + + dataset = Dataset(x1, x2, y) + + # default collate_fn + dataloader = torch.utils.data.DataLoader(dataset, batch_size=10) + + (x1, x2), y = next(iter(dataloader)) + return x1, x2, y + + +def test_dataloader_manual_batching(data): + x1, x2, y = (pd.Series(i) for i in data) + + dataset = Dataset(x1, x2, y) + + # default collate_fn + dataloader = torch.utils.data.DataLoader(dataset, batch_size=None) + + (x1, x2), y = next(iter(dataloader)) + return x1, x2, y + + +class Model(torch.nn.Module): + def __init__(self): + super().__init__() + self.fc1 = torch.nn.Linear(2, 10) + self.relu1 = torch.nn.ReLU() + self.fc2 = torch.nn.Linear(10, 10) + self.relu2 = torch.nn.ReLU() + self.output = torch.nn.Linear(10, 4) + + def forward(self, x1, x2): + x = torch.stack([x1, x2], dim=0).T + x = self.fc1(x) + x = self.relu1(x) + x = self.fc2(x) + x = self.relu2(x) + return torch.nn.functional.softmax(x, dim=1) + + +def train(model, dataloader, optimizer, criterion): + model.train() + for (x1, x2), y in dataloader: + x1 = x1.to("cuda") + x2 = x2.to("cuda") + y = y.to("cuda") + + optimizer.zero_grad() + y_pred = model(x1, x2) + loss = criterion(y_pred, y) + loss.backward() + optimizer.step() + + +def test_torch_train(data): + torch.manual_seed(0) + + x1, x2, y = (pd.Series(i) for i in data) + dataset = Dataset(x1, x2, y) + # default collate_fn + dataloader = torch.utils.data.DataLoader(dataset, batch_size=10) + + model = Model().to("cuda") + optimizer = torch.optim.SGD(model.parameters(), lr=0.001) + criterion = torch.nn.CrossEntropyLoss() + + train(model, dataloader, optimizer, criterion) + + test_x1, test_x2 = next(iter(dataloader))[0] + test_x1 = test_x1.to("cuda") + test_x2 = test_x2.to("cuda") + + return model(test_x1, test_x2) + + +def test_torch_tensor_ctor(): + s = pd.Series(range(5)) + return torch.tensor(s.values) + + +@pytest.mark.xfail_cudf_pandas(reason="Known failure, see xdf/#210") +@pytest.mark.xfail_compare +def test_torch_tensor_from_numpy(): + s = pd.Series(range(5)) + return torch.from_numpy(s.values) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_scipy.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_scipy.py new file mode 100644 index 00000000000..963a8549000 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_scipy.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +import numpy as np +import pandas as pd +import pytest +import scipy + + +@pytest.mark.parametrize("func", ["hmean", "tvar", "gstd"]) +def test_scipy_stats(func): + rng = np.random.default_rng(42) + data = pd.Series(rng.random(1000)) + return getattr(scipy.stats, func)(data) + + +@pytest.mark.parametrize("func", ["norm"]) +def test_scipy_linalg(func): + rng = np.random.default_rng(42) + data = pd.Series(rng.random(1000)) + return getattr(scipy.linalg, func)(data) + + +pytestmark = pytest.mark.assert_eq(fn=pd._testing.assert_almost_equal) + + +def test_compute_pi(): + def circle(x): + return (1 - x**2) ** 0.5 + + x = pd.Series(np.linspace(0, 1, 100)) + y = pd.Series(circle(np.linspace(0, 1, 100))) + + result = scipy.integrate.trapezoid(y, x) + return result * 4 + + +def test_matrix_solve(): + A = pd.DataFrame([[2, 3], [1, 2]]) + b = pd.Series([1, 2]) + + return scipy.linalg.solve(A, b) + + +def test_correlation(): + data = pd.DataFrame({"A": [1, 2, 3, 4, 5], "B": [5, 4, 3, 2, 1]}) + + return scipy.stats.pearsonr(data["A"], data["B"]) + + +def test_optimization(): + x = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + + def rosen(x): # banana function from scipy tutorial + return sum( + 100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + (1 - x[:-1]) ** 2.0 + ) + + result = scipy.optimize.fmin(rosen, x) + return result + + +def test_regression(): + data = pd.DataFrame({"x": [1, 2, 3, 4, 5], "y": [2, 4, 5, 4, 5]}) + result = scipy.stats.linregress(data["y"], data["y"]) + return result.slope, result.intercept diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_seaborn.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_seaborn.py new file mode 100644 index 00000000000..4b272900acd --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_seaborn.py @@ -0,0 +1,60 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import pandas as pd +import pytest +import seaborn as sns +from matplotlib.axes import Axes +from matplotlib.collections import PathCollection +from matplotlib.lines import Line2D +from matplotlib.patches import Rectangle +from pandas._testing import assert_equal + + +def assert_plots_equal(expect, got): + if isinstance(expect, Axes) and isinstance(got, Axes): + for expect_ch, got_ch in zip( + expect.get_children(), got.get_children() + ): + assert type(expect_ch) == type(got_ch) + if isinstance(expect_ch, Line2D): + assert_equal(expect_ch.get_xdata(), got_ch.get_xdata()) + assert_equal(expect_ch.get_ydata(), got_ch.get_ydata()) + elif isinstance(expect_ch, Rectangle): + assert expect_ch.get_height() == got_ch.get_height() + elif isinstance(expect, PathCollection) and isinstance( + got, PathCollection + ): + assert_equal(expect.get_offsets()[:, 0], got.get_offsets()[:, 0]) + assert_equal(expect.get_offsets()[:, 1], got.get_offsets()[:, 1]) + else: + assert_equal(expect, got) + + +pytestmark = pytest.mark.assert_eq(fn=assert_plots_equal) + + +@pytest.fixture(scope="module") +def df(): + df = pd.DataFrame( + { + "x": [2, 3, 4, 5, 11], + "y": [4, 3, 2, 1, 15], + "hue": ["c", "a", "b", "b", "a"], + } + ) + return df + + +def test_bar(df): + ax = sns.barplot(data=df, x="x", y="y") + return ax + + +def test_scatter(df): + ax = sns.scatterplot(data=df, x="x", y="y", hue="hue") + return ax + + +def test_lineplot_with_sns_data(): + df = sns.load_dataset("flights") + ax = sns.lineplot(data=df, x="month", y="passengers") + return ax diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_sklearn.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_sklearn.py new file mode 100644 index 00000000000..1635fd3dcda --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_sklearn.py @@ -0,0 +1,82 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +import numpy as np +import pandas as pd +import pytest +from sklearn.cluster import KMeans +from sklearn.feature_selection import SelectKBest, f_classif +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import accuracy_score +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler + + +def test_regression(): + data = { + "feature1": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "feature2": [2, 4, 1, 3, 5, 7, 6, 8, 10, 9], + "target": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1], + } + df = pd.DataFrame(data) + + X = df[["feature1", "feature2"]] + y = df["target"] + + # Data Splitting + (X_train, X_test, y_train, y_test) = train_test_split( + X, y, test_size=0.2, random_state=42 + ) + + # Basic deterministic LR model + model = LogisticRegression() + model.fit(X_train, y_train) + + # predction phase + y_pred = model.predict(X_test) + accuracy = accuracy_score(y_test, y_pred) + + return accuracy + + +@pytest.mark.assert_eq(fn=np.testing.assert_allclose) +def test_clustering(): + rng = np.random.default_rng(42) + nsamps = 300 + X = rng.random((nsamps, 2)) + data = pd.DataFrame(X, columns=["x", "y"]) + + # Create and fit a KMeans clustering model + kmeans = KMeans(n_clusters=3, random_state=42) + kmeans.fit(data) + return kmeans.cluster_centers_ + + +def test_feature_selection(): + rng = np.random.default_rng(42) + n_samples = 100 + n_features = 10 + + X = rng.random((n_samples, n_features)) + y = rng.integers(0, 2, size=n_samples) + + data = pd.DataFrame( + X, columns=[f"feature{i}" for i in range(1, n_features + 1)] + ) + data["target"] = y + + # Select the top k features + k_best = SelectKBest(score_func=f_classif, k=5) + k_best.fit_transform(X, y) + + feat_inds = k_best.get_support(indices=True) + features = data.iloc[:, feat_inds] + + return sorted(features.columns.tolist()) + + +@pytest.mark.assert_eq(fn=np.testing.assert_allclose) +def test_data_scaling(): + data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + scaler = StandardScaler() + + scaled_data = scaler.fit_transform(data.values.reshape(-1, 1)) + return scaled_data diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy.py new file mode 100644 index 00000000000..69248002a58 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy.py @@ -0,0 +1,94 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +import numpy as np +import pandas as pd +import pytest +import stumpy +from numba import cuda +from pandas._testing import assert_equal + + +def stumpy_assert_equal(expected, got): + def as_float64(x): + if isinstance(x, (tuple, list)): + return [as_float64(y) for y in x] + else: + return x.astype(np.float64) + + assert_equal(as_float64(expected), as_float64(got)) + + +pytestmark = pytest.mark.assert_eq(fn=stumpy_assert_equal) + + +def test_1d_time_series(): + rng = np.random.default_rng(42) + ts = pd.Series(rng.random(10)) + m = 3 + + return stumpy.stump(ts, m) + + +def test_1d_gpu(): + rng = np.random.default_rng(42) + your_time_series = rng.random(10000) + window_size = ( + 50 # Approximately, how many data points might be found in a pattern + ) + all_gpu_devices = [ + device.id for device in cuda.list_devices() + ] # Get a list of all available GPU devices + + return stumpy.gpu_stump( + your_time_series, m=window_size, device_id=all_gpu_devices + ) + + +def test_multidimensional_timeseries(): + rng = np.random.default_rng(42) + # Each row represents data from a different dimension while each column represents + # data from the same dimension + your_time_series = rng.random((3, 1000)) + # Approximately, how many data points might be found in a pattern + window_size = 50 + + return stumpy.mstump(your_time_series, m=window_size) + + +def test_anchored_time_series_chains(): + rng = np.random.default_rng(42) + your_time_series = rng.random(10000) + window_size = ( + 50 # Approximately, how many data points might be found in a pattern + ) + + matrix_profile = stumpy.stump(your_time_series, m=window_size) + + left_matrix_profile_index = matrix_profile[:, 2] + right_matrix_profile_index = matrix_profile[:, 3] + idx = 10 # Subsequence index for which to retrieve the anchored time series chain for + + anchored_chain = stumpy.atsc( + left_matrix_profile_index, right_matrix_profile_index, idx + ) + + all_chain_set, longest_unanchored_chain = stumpy.allc( + left_matrix_profile_index, right_matrix_profile_index + ) + + return anchored_chain, all_chain_set, longest_unanchored_chain + + +def test_semantic_segmentation(): + rng = np.random.default_rng(42) + your_time_series = rng.random(10000) + window_size = ( + 50 # Approximately, how many data points might be found in a pattern + ) + + matrix_profile = stumpy.stump(your_time_series, m=window_size) + + subseq_len = 50 + return stumpy.fluss( + matrix_profile[:, 1], L=subseq_len, n_regimes=2, excl_factor=1 + ) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy_distributed.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy_distributed.py new file mode 100644 index 00000000000..37e3cc34856 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy_distributed.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +import numpy as np +import pandas as pd +import pytest +import stumpy +from pandas._testing import assert_equal + +from dask.distributed import Client, LocalCluster + + +def stumpy_assert_equal(expected, got): + def as_float64(x): + if isinstance(x, (tuple, list)): + return [as_float64(y) for y in x] + else: + return x.astype(np.float64) + + assert_equal(as_float64(expected), as_float64(got)) + + +pytestmark = pytest.mark.assert_eq(fn=stumpy_assert_equal) + + +# Shared dask client for all tests in this module +@pytest.fixture(scope="module") +def dask_client(): + with LocalCluster(n_workers=4, threads_per_worker=1) as cluster: + with Client(cluster) as dask_client: + yield dask_client + + +def test_1d_distributed(dask_client): + np.random.seed(42) + ts = pd.Series(np.random.rand(100)) + m = 10 + return stumpy.stumped(dask_client, ts, m) + + +def test_multidimensional_distributed_timeseries(dask_client): + np.random.seed(42) + # Each row represents data from a different dimension while each column represents + # data from the same dimension + your_time_series = np.random.rand(3, 1000) + # Approximately, how many data points might be found in a pattern + window_size = 50 + + return stumpy.mstumped(dask_client, your_time_series, m=window_size) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_tensorflow.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_tensorflow.py new file mode 100644 index 00000000000..ba1f518cbfd --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_tensorflow.py @@ -0,0 +1,367 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +import numpy as np +import pandas as pd +import pytest +import tensorflow as tf + +SHUFFLE_BUFFER = 500 +BATCH_SIZE = 2 + +pytestmark = pytest.mark.assert_eq(fn=pd._testing.assert_equal) + + +@pytest.fixture(scope="module") +def df(): + rng = np.random.RandomState(42) + + nrows = 303 + columns = { + "age": rng.randint(29, 78, size=(nrows,), dtype="int64"), + "sex": rng.randint(0, 2, size=(nrows,), dtype="int64"), + "cp": rng.randint(0, 5, size=(nrows,), dtype="int64"), + "trestbps": rng.randint(94, 201, size=(nrows,), dtype="int64"), + "chol": rng.randint(126, 565, size=(nrows,), dtype="int64"), + "fbs": rng.randint(0, 2, size=(nrows,), dtype="int64"), + "restecg": rng.randint(0, 3, size=(nrows,), dtype="int64"), + "thalach": rng.randint(71, 203, size=(nrows,), dtype="int64"), + "exang": rng.randint(0, 2, size=(nrows,), dtype="int64"), + "oldpeak": rng.uniform(0.0, 6.2, size=(nrows,)), + "slope": rng.randint(1, 4, size=(nrows,), dtype="int64"), + "ca": rng.randint(0, 4, size=(nrows,), dtype="int64"), + "thal": rng.choice( + ["fixed", "normal", "reversible", "1", "2"], size=(nrows,) + ), + "target": rng.randint(0, 2, size=(nrows,), dtype="int64"), + } + + return pd.DataFrame(columns) + + +@pytest.fixture(scope="module") +def target(df): + return df.pop("target") + + +@pytest.fixture +def model_gen(): + def make_model(numeric_features): + normalizer = tf.keras.layers.Normalization(axis=-1) + normalizer.adapt(numeric_features) + model = tf.keras.Sequential( + [ + normalizer, + tf.keras.layers.Dense(10, activation="relu"), + tf.keras.layers.Dense(1), + ] + ) + + model.compile( + optimizer="adam", + loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), + metrics=["accuracy"], + ) + return model + + return make_model + + +def test_dataframe_as_array(model_gen, df, target): + tf.keras.utils.set_random_seed(42) + + numeric_feature_names = ["age", "thalach", "trestbps", "chol", "oldpeak"] + numeric_features = df[numeric_feature_names] + + numeric_features = tf.convert_to_tensor( + numeric_features.values, dtype=tf.float32 + ) + + model = model_gen(numeric_features) + model.fit(numeric_features, target, epochs=1, batch_size=BATCH_SIZE) + + test_data = numeric_features[:BATCH_SIZE] + return model.predict(test_data) + + +def test_dataframe_as_dataset(model_gen, df, target): + tf.keras.utils.set_random_seed(42) + + numeric_feature_names = ["age", "thalach", "trestbps", "chol", "oldpeak"] + numeric_features = df[numeric_feature_names] + + numeric_features = tf.convert_to_tensor( + numeric_features.values, dtype=tf.float32 + ) + + dataset = tf.data.Dataset.from_tensor_slices((numeric_features, target)) + dataset = dataset.shuffle(SHUFFLE_BUFFER).batch(BATCH_SIZE) + + model = model_gen(numeric_features) + model.fit(dataset, epochs=1) + + test_data = dataset.take(1) + return model.predict(test_data) + + +def stack_dict(inputs, func=tf.stack): + values = [] + for key in sorted(inputs.keys()): + values.append(CastLayer()(inputs[key])) + + class MyLayer(tf.keras.layers.Layer): + def call(self, val): + return func(val, axis=-1) + + return MyLayer()(values) + + +def test_dataframe_as_dictionary_with_keras_input_layer(df, target): + # ensure deterministic results + tf.keras.utils.set_random_seed(42) + + numeric_feature_names = ["age", "thalach", "trestbps", "chol", "oldpeak"] + numeric_features = df[numeric_feature_names] + + inputs = {} + for name in numeric_features: + inputs[name] = tf.keras.Input(shape=(1,), name=name, dtype=tf.float32) + + x = stack_dict(inputs, func=tf.concat) + + normalizer = tf.keras.layers.Normalization(axis=-1) + normalizer.adapt(stack_dict(dict(numeric_features))) + + x = normalizer(x) + x = tf.keras.layers.Dense(10, activation="relu")(x) + x = tf.keras.layers.Dense(1)(x) + + model = tf.keras.Model(inputs, x) + + model.compile( + optimizer="adam", + loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), + metrics=["accuracy"], + run_eagerly=True, + ) + + # Train with dictionary of columns as input: + model.fit(dict(numeric_features), target, epochs=1, batch_size=BATCH_SIZE) + + # Train with a dataset of dictionary-elements + numeric_dict_ds = tf.data.Dataset.from_tensor_slices( + (dict(numeric_features), target) + ) + numeric_dict_batches = numeric_dict_ds.shuffle(SHUFFLE_BUFFER).batch( + BATCH_SIZE + ) + model.fit(numeric_dict_batches, epochs=1) + + # Predict + return model.predict(numeric_dict_batches.take(1)) + + +def test_full_example_train_with_ds(df, target): + # https://www.tensorflow.org/tutorials/load_data/pandas_dataframe#full_example + # Inputs are converted to tf.dataset and then batched + + # ensure deterministic results + tf.keras.utils.set_random_seed(42) + + numeric_feature_names = ["age", "thalach", "trestbps", "chol", "oldpeak"] + binary_feature_names = ["sex", "fbs", "exang"] + categorical_feature_names = ["cp", "restecg", "slope", "thal", "ca"] + + numeric_features = df[numeric_feature_names] + + inputs = {} + for name, column in df.items(): + if isinstance(column[0], str): + dtype = tf.string + elif name in categorical_feature_names or name in binary_feature_names: + dtype = tf.int64 + else: + dtype = tf.float32 + + inputs[name] = tf.keras.Input(shape=(), name=name, dtype=dtype) + + preprocessed = [] + + # Process binary features + for name in binary_feature_names: + inp = inputs[name] + inp = inp[:, tf.newaxis] + float_value = CastLayer()(inp) + preprocessed.append(float_value) + + normalizer = tf.keras.layers.Normalization(axis=-1) + normalizer.adapt(stack_dict(dict(numeric_features))) + + # Process numeric features + numeric_inputs = {} + for name in numeric_feature_names: + numeric_inputs[name] = inputs[name] + + numeric_inputs = stack_dict(numeric_inputs) + numeric_normalized = normalizer(numeric_inputs) + + preprocessed.append(numeric_normalized) + + # Process categorical features + for name in categorical_feature_names: + vocab = sorted(set(df[name])) + print(f"name: {name}") + print(f"vocab: {vocab}\n") + + if isinstance(vocab[0], str): + lookup = tf.keras.layers.StringLookup( + vocabulary=vocab, output_mode="one_hot" + ) + else: + lookup = tf.keras.layers.IntegerLookup( + vocabulary=vocab, output_mode="one_hot" + ) + + x = inputs[name][:, tf.newaxis] + x = lookup(x) + preprocessed.append(x) + + # Concatenate all tensors + preprocesssed_result = MyConcatLayer()(preprocessed) + + preprocessor = tf.keras.Model(inputs, preprocesssed_result) + + # Create the model + body = tf.keras.Sequential( + [ + tf.keras.layers.Dense(10, activation="relu"), + tf.keras.layers.Dense(10, activation="relu"), + tf.keras.layers.Dense(1), + ] + ) + + x = preprocessor(inputs) + result = body(x) + + model = tf.keras.Model(inputs, result) + + model.compile( + optimizer="adam", + loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), + metrics=["accuracy"], + ) + + ds = tf.data.Dataset.from_tensor_slices((dict(df), target)) + ds = ds.batch(BATCH_SIZE) + model.fit(ds, epochs=1) + + return model.predict(ds.take(1)) + + +class CastLayer(tf.keras.layers.Layer): + def __init__(self, **kwargs): + super(CastLayer, self).__init__(**kwargs) + + def call(self, inp): + return tf.cast(inp, tf.float32) + + +class MyConcatLayer(tf.keras.layers.Layer): + def call(self, values): + values = [tf.cast(v, tf.float32) for v in values] + return tf.concat(values, axis=-1) + + +def test_full_example_train_with_df(df, target): + # https://www.tensorflow.org/tutorials/load_data/pandas_dataframe#full_example + # Inputs are directly passed as dictionary of series + + # ensure deterministic results + tf.keras.utils.set_random_seed(42) + + numeric_feature_names = ["age", "thalach", "trestbps", "chol", "oldpeak"] + binary_feature_names = ["sex", "fbs", "exang"] + categorical_feature_names = ["cp", "restecg", "slope", "thal", "ca"] + + numeric_features = df[numeric_feature_names] + + inputs = {} + + for name, column in df.items(): + if isinstance(column[0], str): + dtype = tf.string + elif name in categorical_feature_names or name in binary_feature_names: + dtype = tf.int64 + else: + dtype = tf.float32 + + inputs[name] = tf.keras.Input(shape=(), name=name, dtype=dtype) + + preprocessed = [] + + # Process binary features + for name in binary_feature_names: + inp = inputs[name] + inp = inp[:, tf.newaxis] + float_value = CastLayer()(inp) + preprocessed.append(float_value) + + normalizer = tf.keras.layers.Normalization(axis=-1) + normalizer.adapt(stack_dict(dict(numeric_features))) + + # Process numeric features + numeric_inputs = {} + for name in numeric_feature_names: + numeric_inputs[name] = inputs[name] + + numeric_inputs = stack_dict(numeric_inputs) + numeric_normalized = normalizer(numeric_inputs) + + preprocessed.append(numeric_normalized) + + # Process categorical features + for name in categorical_feature_names: + vocab = sorted(set(df[name])) + print(f"name: {name}") + print(f"vocab: {vocab}\n") + + if isinstance(vocab[0], str): + lookup = tf.keras.layers.StringLookup( + vocabulary=vocab, output_mode="one_hot" + ) + else: + lookup = tf.keras.layers.IntegerLookup( + vocabulary=vocab, output_mode="one_hot" + ) + + x = inputs[name][:, tf.newaxis] + x = lookup(x) + preprocessed.append(x) + + # Concatenate all tensors + preprocesssed_result = MyConcatLayer()(preprocessed) + + preprocessor = tf.keras.Model(inputs, preprocesssed_result) + + # Create the model + body = tf.keras.Sequential( + [ + tf.keras.layers.Dense(10, activation="relu"), + tf.keras.layers.Dense(10, activation="relu"), + tf.keras.layers.Dense(1), + ] + ) + + x = preprocessor(inputs) + result = body(x) + + model = tf.keras.Model(inputs, result) + + model.compile( + optimizer="adam", + loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), + metrics=["accuracy"], + ) + + model.fit(dict(df), target, epochs=1, batch_size=BATCH_SIZE) + + return model.predict(dict(df[:BATCH_SIZE])) diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_xgboost.py b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_xgboost.py new file mode 100644 index 00000000000..70f1e6a4250 --- /dev/null +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_xgboost.py @@ -0,0 +1,135 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +import scipy.sparse +import xgboost as xgb +from sklearn.datasets import make_regression +from xgboost.testing import IteratorForTest, make_categorical + +n_samples = 128 +n_features = 16 + + +def xgboost_assert_equal(expect, got, rtol: float = 1e-7, atol: float = 0.0): + if isinstance(expect, (tuple, list)): + assert len(expect) == len(got) + for e, g in zip(expect, got): + xgboost_assert_equal(e, g, rtol, atol) + elif isinstance(expect, scipy.sparse.csr_matrix): + np.testing.assert_allclose(expect.data, got.data, rtol=rtol, atol=atol) + np.testing.assert_equal(expect.indices, got.indices) + np.testing.assert_equal(expect.indptr, got.indptr) + else: + pd._testing.assert_almost_equal(expect, got, rtol=rtol, atol=atol) + + +pytestmark = pytest.mark.assert_eq(fn=xgboost_assert_equal) + + +@pytest.fixture +def reg_data() -> tuple[np.ndarray, np.ndarray]: + X, y = make_regression(n_samples, n_features, random_state=11) + return X, y + + +@pytest.fixture +def reg_batches_data() -> tuple[list[pd.DataFrame], list[pd.Series]]: + cov = [] + res = [] + for i in range(3): + X, y = make_regression(n_samples, n_features, random_state=i + 1) + cov.append(pd.DataFrame(X)) + res.append(pd.Series(y)) + return cov, res + + +def test_with_dmatrix( + reg_data: tuple[np.ndarray, np.ndarray], +) -> tuple[scipy.sparse.csr_matrix, scipy.sparse.csr_matrix]: + """DMatrix is the primary interface for XGBoost.""" + X, y = reg_data + X_df = pd.DataFrame(X) + y_ser = pd.Series(y) + Xy = xgb.DMatrix(X_df, y_ser) + assert Xy.feature_names == list(map(str, X_df.columns)) + csr_0 = Xy.get_data() + + Xc, yc = make_categorical( + n_samples, n_features, n_categories=13, onehot=False + ) + Xy = xgb.DMatrix(Xc, yc, enable_categorical=True) + csr_1 = Xy.get_data() + return csr_0, csr_1 + + +def test_with_quantile_dmatrix( + reg_data: tuple[np.ndarray, np.ndarray], +) -> tuple[scipy.sparse.csr_matrix, scipy.sparse.csr_matrix]: + """QuantileDMatrix is an optimization for the `hist` tree method for XGBoost.""" + from xgboost.testing.data import memory + + memory.clear(warn=False) + + X, y = reg_data + X_df = pd.DataFrame(X) + y_ser = pd.Series(y) + Xy = xgb.QuantileDMatrix(X_df, y_ser) + assert Xy.feature_names == list(map(str, X_df.columns)) + csr_0 = Xy.get_data() + + Xc, yc = make_categorical( + n_samples, n_features, n_categories=13, onehot=False + ) + Xy = xgb.QuantileDMatrix(Xc, yc, enable_categorical=True) + csr_1 = Xy.get_data() + return csr_0, csr_1 + + +def test_with_iter_quantile_dmatrix( + reg_batches_data: tuple[list[pd.DataFrame], list[pd.DataFrame]], +) -> scipy.sparse.csr_matrix: + """Using iterator to initialize QuantileDMatrix.""" + cov, res = reg_batches_data + it = IteratorForTest(cov, res, w=None, cache=None) + Xy = xgb.QuantileDMatrix(it) + csr = Xy.get_data() + return csr + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_with_external_memory( + device: str, + reg_batches_data: tuple[list[pd.DataFrame], list[pd.DataFrame]], +) -> np.ndarray: + """Test with iterator-based external memory.""" + cov, res = reg_batches_data + it = IteratorForTest(cov, res, w=None, cache="cache") + Xy = xgb.DMatrix(it) + predt = xgb.train({"device": device}, Xy, num_boost_round=1).predict(Xy) + return predt + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_predict(device: str) -> np.ndarray: + reg = xgb.XGBRegressor(n_estimators=2, device=device) + X, y = make_regression(n_samples, n_features, random_state=11) + X_df = pd.DataFrame(X) + reg.fit(X_df, y) + booster = reg.get_booster() + + predt0 = reg.predict(X_df) + + predt1 = booster.inplace_predict(X_df) + np.testing.assert_allclose(predt0, predt1) + + predt2 = booster.predict(xgb.DMatrix(X_df)) + np.testing.assert_allclose(predt0, predt2) + + predt3 = booster.inplace_predict(X) + np.testing.assert_allclose(predt0, predt3) + + return predt0 From 23fb31e7af5e768722f640601034a9d490c2e54c Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Thu, 29 Aug 2024 19:27:06 -0700 Subject: [PATCH 153/270] Add a libcudf/thrust-based TPC-H derived datagen (#16294) This PR adds a TPC-H (according to spec 3.0.1) inspired datagen written using `libcudf` and `thrust` ### Implementation Status - [x] lineitem - [x] orders - [x] region - [x] nation - [x] supplier - [x] customer - [x] part - [x] partsupp Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) - Karthikeyan (https://github.com/karthikeyann) Approvers: - Mark Harris (https://github.com/harrism) - Yunsong Wang (https://github.com/PointKernel) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/16294 --- cpp/benchmarks/CMakeLists.txt | 24 + .../random_column_generator.cu | 246 +++++ .../random_column_generator.hpp | 150 +++ .../tpch_data_generator/table_helpers.cpp | 386 +++++++ .../tpch_data_generator/table_helpers.hpp | 155 +++ .../tpch_data_generator.cpp | 987 ++++++++++++++++++ .../tpch_data_generator.hpp | 94 ++ 7 files changed, 2042 insertions(+) create mode 100644 cpp/benchmarks/common/tpch_data_generator/random_column_generator.cu create mode 100644 cpp/benchmarks/common/tpch_data_generator/random_column_generator.hpp create mode 100644 cpp/benchmarks/common/tpch_data_generator/table_helpers.cpp create mode 100644 cpp/benchmarks/common/tpch_data_generator/table_helpers.hpp create mode 100644 cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.cpp create mode 100644 cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.hpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 99ef9e2976f..d2c22b788cb 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -35,6 +35,30 @@ target_include_directories( "$" ) +add_library( + tpch_data_generator STATIC + common/tpch_data_generator/tpch_data_generator.cpp common/tpch_data_generator/table_helpers.cpp + common/tpch_data_generator/random_column_generator.cu +) +target_compile_features(tpch_data_generator PUBLIC cxx_std_17 cuda_std_17) + +target_compile_options( + tpch_data_generator PUBLIC "$<$:${CUDF_CXX_FLAGS}>" + "$<$:${CUDF_CUDA_FLAGS}>" +) + +target_link_libraries( + tpch_data_generator + PUBLIC cudf cudftestutil nvtx3::nvtx3-cpp + PRIVATE $ +) + +target_include_directories( + tpch_data_generator + PUBLIC "$" "$" + "$" +) + # ################################################################################################## # * compiler function ----------------------------------------------------------------------------- diff --git a/cpp/benchmarks/common/tpch_data_generator/random_column_generator.cu b/cpp/benchmarks/common/tpch_data_generator/random_column_generator.cu new file mode 100644 index 00000000000..4246bd1a83b --- /dev/null +++ b/cpp/benchmarks/common/tpch_data_generator/random_column_generator.cu @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "random_column_generator.hpp" + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include + +namespace cudf::datagen { + +namespace { + +// Functor for generating random strings +struct random_string_generator { + char* chars; + thrust::default_random_engine engine; + thrust::uniform_int_distribution char_dist; + + CUDF_HOST_DEVICE random_string_generator(char* c) : chars(c), char_dist(44, 122) {} + + __device__ void operator()(thrust::tuple str_begin_end) + { + auto begin = thrust::get<0>(str_begin_end); + auto end = thrust::get<1>(str_begin_end); + engine.discard(begin); + for (auto i = begin; i < end; ++i) { + auto ch = char_dist(engine); + if (i == end - 1 && ch >= '\x7F') ch = ' '; // last element ASCII only. + if (ch >= '\x7F') // x7F is at the top edge of ASCII + chars[i++] = '\xC4'; // these characters are assigned two bytes + chars[i] = static_cast(ch + (ch >= '\x7F')); + } + } +}; + +// Functor for generating random numbers +template +struct random_number_generator { + T lower; + T upper; + + CUDF_HOST_DEVICE random_number_generator(T lower, T upper) : lower(lower), upper(upper) {} + + __device__ T operator()(const int64_t idx) const + { + if constexpr (cudf::is_integral()) { + thrust::default_random_engine engine; + thrust::uniform_int_distribution dist(lower, upper); + engine.discard(idx); + return dist(engine); + } else { + thrust::default_random_engine engine; + thrust::uniform_real_distribution dist(lower, upper); + engine.discard(idx); + return dist(engine); + } + } +}; + +} // namespace + +std::unique_ptr generate_random_string_column(cudf::size_type lower, + cudf::size_type upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto offsets_begin = cudf::detail::make_counting_transform_iterator( + 0, random_number_generator(lower, upper)); + auto [offsets_column, computed_bytes] = cudf::strings::detail::make_offsets_child_column( + offsets_begin, offsets_begin + num_rows, stream, mr); + rmm::device_uvector chars(computed_bytes, stream); + + auto const offset_itr = + cudf::detail::offsetalator_factory::make_input_iterator(offsets_column->view()); + + // We generate the strings in parallel into the `chars` vector using the + // offsets vector generated above. + thrust::for_each_n(rmm::exec_policy(stream), + thrust::make_zip_iterator(offset_itr, offset_itr + 1), + num_rows, + random_string_generator(chars.data())); + + return cudf::make_strings_column( + num_rows, std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); +} + +template +std::unique_ptr generate_random_numeric_column(T lower, + T upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto col = cudf::make_numeric_column( + cudf::data_type{cudf::type_to_id()}, num_rows, cudf::mask_state::UNALLOCATED, stream, mr); + cudf::size_type begin = 0; + cudf::size_type end = num_rows; + thrust::transform(rmm::exec_policy(stream), + thrust::make_counting_iterator(begin), + thrust::make_counting_iterator(end), + col->mutable_view().begin(), + random_number_generator(lower, upper)); + return col; +} + +template std::unique_ptr generate_random_numeric_column( + int8_t lower, + int8_t upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + +template std::unique_ptr generate_random_numeric_column( + int16_t lower, + int16_t upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + +template std::unique_ptr generate_random_numeric_column( + cudf::size_type lower, + cudf::size_type upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + +template std::unique_ptr generate_random_numeric_column( + double lower, + double upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + +std::unique_ptr generate_primary_key_column(cudf::scalar const& start, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + return cudf::sequence(num_rows, start, stream, mr); +} + +std::unique_ptr generate_repeat_string_column(std::string const& value, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const scalar = cudf::string_scalar(value); + return cudf::make_column_from_scalar(scalar, num_rows, stream, mr); +} + +std::unique_ptr generate_random_string_column_from_set( + cudf::host_span set, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Build a gather map of random strings to choose from + // The size of the string sets always fits within 16-bit integers + auto const indices = + generate_primary_key_column(cudf::numeric_scalar(0), set.size(), stream, mr); + auto const keys = cudf::test::strings_column_wrapper(set.begin(), set.end()).release(); + auto const gather_map = cudf::table_view({indices->view(), keys->view()}); + + // Build a column of random keys to gather from the set + auto const gather_keys = + generate_random_numeric_column(0, set.size() - 1, num_rows, stream, mr); + + // Perform the gather operation + auto const gathered_table = cudf::gather( + gather_map, gather_keys->view(), cudf::out_of_bounds_policy::DONT_CHECK, stream, mr); + auto gathered_table_columns = gathered_table->release(); + return std::move(gathered_table_columns[1]); +} + +template +std::unique_ptr generate_repeat_sequence_column(T seq_length, + bool zero_indexed, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto pkey = + generate_primary_key_column(cudf::numeric_scalar(0), num_rows, stream, mr); + auto repeat_seq_zero_indexed = cudf::binary_operation(pkey->view(), + cudf::numeric_scalar(seq_length), + cudf::binary_operator::MOD, + cudf::data_type{cudf::type_to_id()}, + stream, + mr); + if (zero_indexed) { return repeat_seq_zero_indexed; } + return cudf::binary_operation(repeat_seq_zero_indexed->view(), + cudf::numeric_scalar(1), + cudf::binary_operator::ADD, + cudf::data_type{cudf::type_to_id()}, + stream, + mr); +} + +template std::unique_ptr generate_repeat_sequence_column( + int8_t seq_length, + bool zero_indexed, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + +template std::unique_ptr generate_repeat_sequence_column( + cudf::size_type seq_length, + bool zero_indexed, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + +} // namespace cudf::datagen diff --git a/cpp/benchmarks/common/tpch_data_generator/random_column_generator.hpp b/cpp/benchmarks/common/tpch_data_generator/random_column_generator.hpp new file mode 100644 index 00000000000..3e254f49805 --- /dev/null +++ b/cpp/benchmarks/common/tpch_data_generator/random_column_generator.hpp @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +namespace cudf::datagen { + +/** + * @brief Generate a column of random strings + * + * @param lower The lower bound of the length of the strings + * @param upper The upper bound of the length of the strings + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_random_string_column( + cudf::size_type lower, + cudf::size_type upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate a column of random numbers + * + * Example: + * + * lower = 10 + * upper = 15 + * num_rows = 10 + * result = [10, 11, 14, 14, 13, 12, 11, 11, 12, 14] + + * + * @param lower The lower bound of the random numbers + * @param upper The upper bound of the random numbers + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +template +std::unique_ptr generate_random_numeric_column( + T lower, + T upper, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate a primary key column + * + * Example: + * + * start = 1 + * num_rows = 10 + * result = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + * + * @param start The starting value of the primary key + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_primary_key_column( + cudf::scalar const& start, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate a column where all the rows have the same string value + * + * Example: + * + * value = "abc" + * num_rows = 5 + * result = ["abc", "abc", "abc", "abc", "abc"] + * + * @param value The string value to fill the column with + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_repeat_string_column( + std::string const& value, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate a column by randomly choosing from set of strings + * + * Example: + * + * set = {"s1", "s2", "s3"} + * num_rows = 10 + * result = ["s1", "s2", "s2", "s1", "s3", "s3", "s3", "s2", "s1", "s1"] + * + * @param set The set of strings to choose from + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_random_string_column_from_set( + cudf::host_span set, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate a column consisting of a repeating sequence of integers + * + * Example: + * + * seq_length = 3 + * zero_indexed = false + * num_rows = 10 + * result = [1, 2, 3, 1, 2, 3, 1, 2, 3, 1] + * + * @param seq_length The length of the repeating sequence + * @param zero_indexed Whether the sequence is zero or one indexed + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +template +std::unique_ptr generate_repeat_sequence_column( + T seq_length, + bool zero_indexed, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +} // namespace cudf::datagen diff --git a/cpp/benchmarks/common/tpch_data_generator/table_helpers.cpp b/cpp/benchmarks/common/tpch_data_generator/table_helpers.cpp new file mode 100644 index 00000000000..36bf9c49cea --- /dev/null +++ b/cpp/benchmarks/common/tpch_data_generator/table_helpers.cpp @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "table_helpers.hpp" + +#include "random_column_generator.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace cudf::datagen { + +/** + * @brief Add a column of days to a column of timestamp_days + * + * @param timestamp_days The column of timestamp_days + * @param days The column of days to add + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr add_calendrical_days(cudf::column_view const& timestamp_days, + cudf::column_view const& days, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const days_duration_type = cudf::cast(days, cudf::data_type{cudf::type_id::DURATION_DAYS}); + auto const data_type = cudf::data_type{cudf::type_id::TIMESTAMP_DAYS}; + return cudf::binary_operation( + timestamp_days, days_duration_type->view(), cudf::binary_operator::ADD, data_type, stream, mr); +} + +/** + * @brief Perform a left join operation between two tables + * + * @param left_input The left table + * @param right_input The right table + * @param left_on The indices of the columns to join on in the left table + * @param right_on The indices of the columns to join on in the right table + * @param compare_nulls The null equality comparison + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned table's device memory + */ +std::unique_ptr perform_left_join(cudf::table_view const& left_input, + cudf::table_view const& right_input, + std::vector const& left_on, + std::vector const& right_on, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + constexpr auto oob_policy = cudf::out_of_bounds_policy::NULLIFY; + auto const left_selected = left_input.select(left_on); + auto const right_selected = right_input.select(right_on); + auto const [left_join_indices, right_join_indices] = + cudf::left_join(left_selected, right_selected, cudf::null_equality::EQUAL, mr); + + auto const left_indices_span = cudf::device_span{*left_join_indices}; + auto const right_indices_span = cudf::device_span{*right_join_indices}; + + auto const left_indices_col = cudf::column_view{left_indices_span}; + auto const right_indices_col = cudf::column_view{right_indices_span}; + + auto const left_result = cudf::gather(left_input, left_indices_col, oob_policy, stream, mr); + auto const right_result = cudf::gather(right_input, right_indices_col, oob_policy, stream, mr); + + auto joined_cols = left_result->release(); + auto right_cols = right_result->release(); + joined_cols.insert(joined_cols.end(), + std::make_move_iterator(right_cols.begin()), + std::make_move_iterator(right_cols.end())); + return std::make_unique(std::move(joined_cols)); +} + +/** + * @brief Generate the `p_retailprice` column of the `part` table + * + * @param p_partkey The `p_partkey` column of the `part` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_p_retailprice( + cudf::column_view const& p_partkey, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Expression: (90000 + ((p_partkey/10) modulo 20001) + 100 * (p_partkey modulo 1000)) / 100 + auto table = cudf::table_view({p_partkey}); + auto p_partkey_col_ref = cudf::ast::column_reference(0); + + auto scalar_10 = cudf::numeric_scalar(10); + auto scalar_100 = cudf::numeric_scalar(100); + auto scalar_1000 = cudf::numeric_scalar(1000); + auto scalar_20001 = cudf::numeric_scalar(20001); + auto scalar_90000 = cudf::numeric_scalar(90000); + + auto literal_10 = cudf::ast::literal(scalar_10); + auto literal_100 = cudf::ast::literal(scalar_100); + auto literal_1000 = cudf::ast::literal(scalar_1000); + auto literal_20001 = cudf::ast::literal(scalar_20001); + auto literal_90000 = cudf::ast::literal(scalar_90000); + + auto expr_a = cudf::ast::operation(cudf::ast::ast_operator::DIV, p_partkey_col_ref, literal_10); + auto expr_b = cudf::ast::operation(cudf::ast::ast_operator::MOD, expr_a, literal_20001); + auto expr_c = cudf::ast::operation(cudf::ast::ast_operator::MOD, p_partkey_col_ref, literal_1000); + auto expr_d = cudf::ast::operation(cudf::ast::ast_operator::MUL, expr_c, literal_100); + auto expr_e = cudf::ast::operation(cudf::ast::ast_operator::ADD, expr_b, expr_d); + auto expr_f = cudf::ast::operation(cudf::ast::ast_operator::ADD, expr_e, literal_90000); + auto final_expr = cudf::ast::operation(cudf::ast::ast_operator::TRUE_DIV, expr_f, literal_100); + + // Execute the AST expression + return cudf::compute_column(table, final_expr, stream, mr); +} + +/** + * @brief Generate the `l_suppkey` column of the `lineitem` table + * + * @param l_partkey The `l_partkey` column of the `lineitem` table + * @param scale_factor The scale factor to use + * @param num_rows The number of rows in the `lineitem` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_l_suppkey(cudf::column_view const& l_partkey, + cudf::size_type scale_factor, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Expression: (l_partkey + (i * (s/4 + (int)(l_partkey - 1)/s))) % s + 1 + + // Generate the `s` col + auto s_empty = cudf::make_numeric_column( + cudf::data_type{cudf::type_id::INT32}, num_rows, cudf::mask_state::UNALLOCATED, stream); + + auto s = cudf::fill(s_empty->view(), + 0, + num_rows, + cudf::numeric_scalar(scale_factor * 10'000), + stream, + mr); + + // Generate the `i` col + auto i = generate_repeat_sequence_column(4, true, num_rows, stream, mr); + + // Create a table view out of `l_partkey`, `s`, and `i` + auto table = cudf::table_view({l_partkey, s->view(), i->view()}); + + // Create the AST expression + auto scalar_1 = cudf::numeric_scalar(1); + auto scalar_4 = cudf::numeric_scalar(4); + auto literal_1 = cudf::ast::literal(scalar_1); + auto literal_4 = cudf::ast::literal(scalar_4); + + auto l_partkey_col_ref = cudf::ast::column_reference(0); + auto s_col_ref = cudf::ast::column_reference(1); + auto i_col_ref = cudf::ast::column_reference(2); + + // (int)(l_partkey - 1)/s + auto expr_a = cudf::ast::operation(cudf::ast::ast_operator::SUB, l_partkey_col_ref, literal_1); + auto expr_b = cudf::ast::operation(cudf::ast::ast_operator::DIV, expr_a, s_col_ref); + + // s/4 + auto expr_c = cudf::ast::operation(cudf::ast::ast_operator::DIV, s_col_ref, literal_4); + + // (s/4 + (int)(l_partkey - 1)/s) + auto expr_d = cudf::ast::operation(cudf::ast::ast_operator::ADD, expr_c, expr_b); + + // (i * (s/4 + (int)(l_partkey - 1)/s)) + auto expr_e = cudf::ast::operation(cudf::ast::ast_operator::MUL, i_col_ref, expr_d); + + // (l_partkey + (i * (s/4 + (int)(l_partkey - 1)/s))) + auto expr_f = cudf::ast::operation(cudf::ast::ast_operator::ADD, l_partkey_col_ref, expr_e); + + // (l_partkey + (i * (s/4 + (int)(l_partkey - 1)/s))) % s + auto expr_g = cudf::ast::operation(cudf::ast::ast_operator::MOD, expr_f, s_col_ref); + + // (l_partkey + (i * (s/4 + (int)(l_partkey - 1)/s))) % s + 1 + auto final_expr = cudf::ast::operation(cudf::ast::ast_operator::ADD, expr_g, literal_1); + + // Execute the AST expression + return cudf::compute_column(table, final_expr, stream, mr); +} + +/** + * @brief Generate the `ps_suppkey` column of the `partsupp` table + * + * @param ps_partkey The `ps_partkey` column of the `partsupp` table + * @param scale_factor The scale factor to use + * @param num_rows The number of rows in the `partsupp` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_ps_suppkey( + cudf::column_view const& ps_partkey, + cudf::size_type scale_factor, + cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Expression: ps_suppkey = (ps_partkey + (i * (s/4 + (int)(ps_partkey - 1)/s))) % s + 1 + + // Generate the `s` col + auto s_empty = cudf::make_numeric_column( + cudf::data_type{cudf::type_id::INT32}, num_rows, cudf::mask_state::UNALLOCATED, stream); + + auto s = cudf::fill(s_empty->view(), + 0, + num_rows, + cudf::numeric_scalar(scale_factor * 10'000), + stream, + mr); + + // Generate the `i` col + auto i = generate_repeat_sequence_column(4, true, num_rows, stream, mr); + + // Create a table view out of `p_partkey`, `s`, and `i` + auto table = cudf::table_view({ps_partkey, s->view(), i->view()}); + + // Create the AST expression + auto scalar_1 = cudf::numeric_scalar(1); + auto scalar_4 = cudf::numeric_scalar(4); + auto literal_1 = cudf::ast::literal(scalar_1); + auto literal_4 = cudf::ast::literal(scalar_4); + + auto ps_partkey_col_ref = cudf::ast::column_reference(0); + auto s_col_ref = cudf::ast::column_reference(1); + auto i_col_ref = cudf::ast::column_reference(2); + + // (int)(ps_partkey - 1)/s + auto expr_a = cudf::ast::operation(cudf::ast::ast_operator::SUB, ps_partkey_col_ref, literal_1); + auto expr_b = cudf::ast::operation(cudf::ast::ast_operator::DIV, expr_a, s_col_ref); + + // s/4 + auto expr_c = cudf::ast::operation(cudf::ast::ast_operator::DIV, s_col_ref, literal_4); + + // (s/4 + (int)(ps_partkey - 1)/s) + auto expr_d = cudf::ast::operation(cudf::ast::ast_operator::ADD, expr_c, expr_b); + + // (i * (s/4 + (int)(ps_partkey - 1)/s)) + auto expr_e = cudf::ast::operation(cudf::ast::ast_operator::MUL, i_col_ref, expr_d); + + // (ps_partkey + (i * (s/4 + (int)(ps_partkey - 1)/s))) + auto expr_f = cudf::ast::operation(cudf::ast::ast_operator::ADD, ps_partkey_col_ref, expr_e); + + // (ps_partkey + (i * (s/4 + (int)(ps_partkey - 1)/s))) % s + auto expr_g = cudf::ast::operation(cudf::ast::ast_operator::MOD, expr_f, s_col_ref); + + // (ps_partkey + (i * (s/4 + (int)(ps_partkey - 1)/s))) % s + 1 + auto final_expr = cudf::ast::operation(cudf::ast::ast_operator::ADD, expr_g, literal_1); + + // Execute the AST expression + return cudf::compute_column(table, final_expr, stream, mr); +} + +/** + * @brief Calculate the cardinality of the `lineitem` table + * + * @param o_rep_freqs The frequency of each `o_orderkey` value in the `lineitem` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] cudf::size_type calculate_l_cardinality(cudf::column_view const& o_rep_freqs, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const sum_agg = cudf::make_sum_aggregation(); + auto const l_num_rows_scalar = + cudf::reduce(o_rep_freqs, *sum_agg, cudf::data_type{cudf::type_id::INT32}, stream, mr); + return reinterpret_cast*>(l_num_rows_scalar.get()) + ->value(stream); +} + +/** + * @brief Calculate the charge column for the `lineitem` table + * + * @param extendedprice The `l_extendedprice` column + * @param tax The `l_tax` column + * @param discount The `l_discount` column + * @param stream The CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_charge(cudf::column_view const& extendedprice, + cudf::column_view const& tax, + cudf::column_view const& discount, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const one = cudf::numeric_scalar(1); + auto const one_minus_discount = cudf::binary_operation( + one, discount, cudf::binary_operator::SUB, cudf::data_type{cudf::type_id::FLOAT64}, stream, mr); + auto disc_price = cudf::binary_operation(extendedprice, + one_minus_discount->view(), + cudf::binary_operator::MUL, + cudf::data_type{cudf::type_id::FLOAT64}, + stream, + mr); + auto const one_plus_tax = + cudf::binary_operation(one, tax, cudf::binary_operator::ADD, tax.type(), stream, mr); + return cudf::binary_operation(disc_price->view(), + one_plus_tax->view(), + cudf::binary_operator::MUL, + cudf::data_type{cudf::type_id::FLOAT64}, + stream, + mr); +} + +/** + * @brief Generate a column of random addresses according to TPC-H specification clause 4.2.2.7 + * + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr generate_address_column( + cudf::size_type num_rows, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + return generate_random_string_column(10, 40, num_rows, stream, mr); +} + +/** + * @brief Generate a phone number column according to TPC-H specification clause 4.2.2.9 + * + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr generate_phone_column(cudf::size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const part_a = cudf::strings::from_integers( + generate_random_numeric_column(10, 34, num_rows, stream, mr)->view()); + auto const part_b = cudf::strings::from_integers( + generate_random_numeric_column(100, 999, num_rows, stream, mr)->view()); + auto const part_c = cudf::strings::from_integers( + generate_random_numeric_column(100, 999, num_rows, stream, mr)->view()); + auto const part_d = cudf::strings::from_integers( + generate_random_numeric_column(1000, 9999, num_rows, stream, mr)->view()); + auto const phone_parts_table = + cudf::table_view({part_a->view(), part_b->view(), part_c->view(), part_d->view()}); + return cudf::strings::concatenate(phone_parts_table, + cudf::string_scalar("-"), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); +} + +} // namespace cudf::datagen diff --git a/cpp/benchmarks/common/tpch_data_generator/table_helpers.hpp b/cpp/benchmarks/common/tpch_data_generator/table_helpers.hpp new file mode 100644 index 00000000000..11091689469 --- /dev/null +++ b/cpp/benchmarks/common/tpch_data_generator/table_helpers.hpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include + +namespace cudf::datagen { + +/** + * @brief Add a column of days to a column of timestamp_days + * + * @param timestamp_days The column of timestamp_days + * @param days The column of days to add + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr add_calendrical_days( + cudf::column_view const& timestamp_days, + cudf::column_view const& days, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Perform a left join operation between two tables + * + * @param left_input The left table + * @param right_input The right table + * @param left_on The indices of the columns to join on in the left table + * @param right_on The indices of the columns to join on in the right table + * @param compare_nulls The null equality comparison + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned table's device memory + */ +std::unique_ptr perform_left_join( + cudf::table_view const& left_input, + cudf::table_view const& right_input, + std::vector const& left_on, + std::vector const& right_on, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `p_retailprice` column of the `part` table + * + * @param p_partkey The `p_partkey` column of the `part` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_p_retailprice( + cudf::column_view const& p_partkey, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `l_suppkey` column of the `lineitem` table + * + * @param l_partkey The `l_partkey` column of the `lineitem` table + * @param scale_factor The scale factor to use + * @param num_rows The number of rows in the `lineitem` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_l_suppkey( + cudf::column_view const& l_partkey, + cudf::size_type scale_factor, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `ps_suppkey` column of the `partsupp` table + * + * @param ps_partkey The `ps_partkey` column of the `partsupp` table + * @param scale_factor The scale factor to use + * @param num_rows The number of rows in the `partsupp` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_ps_suppkey( + cudf::column_view const& ps_partkey, + cudf::size_type scale_factor, + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); +/** + * @brief Calculate the cardinality of the `lineitem` table + * + * @param o_rep_freqs The frequency of each `o_orderkey` value in the `lineitem` table + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] cudf::size_type calculate_l_cardinality( + cudf::column_view const& o_rep_freqs, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); +/** + * @brief Calculate the charge column for the `lineitem` table + * + * @param extendedprice The `l_extendedprice` column + * @param tax The `l_tax` column + * @param discount The `l_discount` column + * @param stream The CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr calculate_charge( + cudf::column_view const& extendedprice, + cudf::column_view const& tax, + cudf::column_view const& discount, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate a column of random addresses according to TPC-H specification clause 4.2.2.7 + * + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr generate_address_column( + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate a phone number column according to TPC-H specification clause 4.2.2.9 + * + * @param num_rows The number of rows in the column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr generate_phone_column( + cudf::size_type num_rows, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +} // namespace cudf::datagen diff --git a/cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.cpp b/cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.cpp new file mode 100644 index 00000000000..9001c50c5a5 --- /dev/null +++ b/cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.cpp @@ -0,0 +1,987 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tpch_data_generator.hpp" + +#include "random_column_generator.hpp" +#include "table_helpers.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace cudf::datagen { + +namespace { +constexpr std::array nations{ + "ALGERIA", "ARGENTINA", "BRAZIL", "CANADA", "EGYPT", "ETHIOPIA", "FRANCE", + "GERMANY", "INDIA", "INDONESIA", "IRAN", "IRAQ", "JAPAN", "JORDAN", + "KENYA", "MOROCCO", "MOZAMBIQUE", "PERU", "CHINA", "ROMANIA", "SAUDI ARABIA", + "VIETNAM", "RUSSIA", "UNITED KINGDOM", "UNITED STATES"}; + +constexpr std::array years{"1992", "1993", "1994", "1995", "1996", "1997", "1998"}; +constexpr std::array months{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"}; +constexpr std::array days{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", + "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", + "23", "24", "25", "26", "27", "28", "29", "30", "31"}; + +constexpr std::array vocab_p_name{ + "almond", "antique", "aquamarine", "azure", "beige", "bisque", "black", + "blanched", "blue", "blush", "brown", "burlywood", "burnished", "chartreuse", + "chiffon", "chocolate", "coral", "cornflower", "cornsilk", "cream", "cyan", + "dark", "deep", "dim", "dodger", "drab", "firebrick", "floral", + "forest", "frosted", "gainsboro", "ghost", "goldenrod", "green", "grey", + "honeydew", "hot", "indian", "ivory", "khaki", "lace", "lavender", + "lawn", "lemon", "light", "lime", "linen", "magenta", "maroon", + "medium", "metallic", "midnight", "mint", "misty", "moccasin", "navajo", + "navy", "olive", "orange", "orchid", "pale", "papaya", "peach", + "peru", "pink", "plum", "powder", "puff", "purple", "red", + "rose", "rosy", "royal", "saddle", "salmon", "sandy", "seashell", + "sienna", "sky", "slate", "smoke", "snow", "spring", "steel", + "tan", "thistle", "tomato", "turquoise", "violet", "wheat", "white", + "yellow"}; + +constexpr std::array vocab_modes{"REG AIR", "AIR", "RAIL", "SHIP", "TRUCK", "MAIL", "FOB"}; + +constexpr std::array vocab_instructions{ + "DELIVER IN PERSON", "COLLECT COD", "NONE", "TAKE BACK RETURN"}; + +constexpr std::array vocab_priorities{"1-URGENT", "2-HIGH", "3-MEDIUM", "4-NOT SPECIFIED", "5-LOW"}; + +constexpr std::array vocab_segments{ + "AUTOMOBILE", "BUILDING", "FURNITURE", "MACHINERY", "HOUSEHOLD"}; + +constexpr std::array vocab_types{ + "STANDARD ANODIZED TIN", "STANDARD ANODIZED NICKEL", "STANDARD ANODIZED BRASS", + "STANDARD ANODIZED STEEL", "STANDARD ANODIZED COPPER", "STANDARD BURNISHED TIN", + "STANDARD BURNISHED NICKEL", "STANDARD BURNISHED BRASS", "STANDARD BURNISHED STEEL", + "STANDARD BURNISHED COPPER", "STANDARD PLATED TIN", "STANDARD PLATED NICKEL", + "STANDARD PLATED BRASS", "STANDARD PLATED STEEL", "STANDARD PLATED COPPER", + "STANDARD POLISHED TIN", "STANDARD POLISHED NICKEL", "STANDARD POLISHED BRASS", + "STANDARD POLISHED STEEL", "STANDARD POLISHED COPPER", "STANDARD BRUSHED TIN", + "STANDARD BRUSHED NICKEL", "STANDARD BRUSHED BRASS", "STANDARD BRUSHED STEEL", + "STANDARD BRUSHED COPPER", "SMALL ANODIZED TIN", "SMALL ANODIZED NICKEL", + "SMALL ANODIZED BRASS", "SMALL ANODIZED STEEL", "SMALL ANODIZED COPPER", + "SMALL BURNISHED TIN", "SMALL BURNISHED NICKEL", "SMALL BURNISHED BRASS", + "SMALL BURNISHED STEEL", "SMALL BURNISHED COPPER", "SMALL PLATED TIN", + "SMALL PLATED NICKEL", "SMALL PLATED BRASS", "SMALL PLATED STEEL", + "SMALL PLATED COPPER", "SMALL POLISHED TIN", "SMALL POLISHED NICKEL", + "SMALL POLISHED BRASS", "SMALL POLISHED STEEL", "SMALL POLISHED COPPER", + "SMALL BRUSHED TIN", "SMALL BRUSHED NICKEL", "SMALL BRUSHED BRASS", + "SMALL BRUSHED STEEL", "SMALL BRUSHED COPPER", "MEDIUM ANODIZED TIN", + "MEDIUM ANODIZED NICKEL", "MEDIUM ANODIZED BRASS", "MEDIUM ANODIZED STEEL", + "MEDIUM ANODIZED COPPER", "MEDIUM BURNISHED TIN", "MEDIUM BURNISHED NICKEL", + "MEDIUM BURNISHED BRASS", "MEDIUM BURNISHED STEEL", "MEDIUM BURNISHED COPPER", + "MEDIUM PLATED TIN", "MEDIUM PLATED NICKEL", "MEDIUM PLATED BRASS", + "MEDIUM PLATED STEEL", "MEDIUM PLATED COPPER", "MEDIUM POLISHED TIN", + "MEDIUM POLISHED NICKEL", "MEDIUM POLISHED BRASS", "MEDIUM POLISHED STEEL", + "MEDIUM POLISHED COPPER", "MEDIUM BRUSHED TIN", "MEDIUM BRUSHED NICKEL", + "MEDIUM BRUSHED BRASS", "MEDIUM BRUSHED STEEL", "MEDIUM BRUSHED COPPER", + "LARGE ANODIZED TIN", "LARGE ANODIZED NICKEL", "LARGE ANODIZED BRASS", + "LARGE ANODIZED STEEL", "LARGE ANODIZED COPPER", "LARGE BURNISHED TIN", + "LARGE BURNISHED NICKEL", "LARGE BURNISHED BRASS", "LARGE BURNISHED STEEL", + "LARGE BURNISHED COPPER", "LARGE PLATED TIN", "LARGE PLATED NICKEL", + "LARGE PLATED BRASS", "LARGE PLATED STEEL", "LARGE PLATED COPPER", + "LARGE POLISHED TIN", "LARGE POLISHED NICKEL", "LARGE POLISHED BRASS", + "LARGE POLISHED STEEL", "LARGE POLISHED COPPER", "LARGE BRUSHED TIN", + "LARGE BRUSHED NICKEL", "LARGE BRUSHED BRASS", "LARGE BRUSHED STEEL", + "LARGE BRUSHED COPPER", "ECONOMY ANODIZED TIN", "ECONOMY ANODIZED NICKEL", + "ECONOMY ANODIZED BRASS", "ECONOMY ANODIZED STEEL", "ECONOMY ANODIZED COPPER", + "ECONOMY BURNISHED TIN", "ECONOMY BURNISHED NICKEL", "ECONOMY BURNISHED BRASS", + "ECONOMY BURNISHED STEEL", "ECONOMY BURNISHED COPPER", "ECONOMY PLATED TIN", + "ECONOMY PLATED NICKEL", "ECONOMY PLATED BRASS", "ECONOMY PLATED STEEL", + "ECONOMY PLATED COPPER", "ECONOMY POLISHED TIN", "ECONOMY POLISHED NICKEL", + "ECONOMY POLISHED BRASS", "ECONOMY POLISHED STEEL", "ECONOMY POLISHED COPPER", + "ECONOMY BRUSHED TIN", "ECONOMY BRUSHED NICKEL", "ECONOMY BRUSHED BRASS", + "ECONOMY BRUSHED STEEL", "ECONOMY BRUSHED COPPER", "PROMO ANODIZED TIN", + "PROMO ANODIZED NICKEL", "PROMO ANODIZED BRASS", "PROMO ANODIZED STEEL", + "PROMO ANODIZED COPPER", "PROMO BURNISHED TIN", "PROMO BURNISHED NICKEL", + "PROMO BURNISHED BRASS", "PROMO BURNISHED STEEL", "PROMO BURNISHED COPPER", + "PROMO PLATED TIN", "PROMO PLATED NICKEL", "PROMO PLATED BRASS", + "PROMO PLATED STEEL", "PROMO PLATED COPPER", "PROMO POLISHED TIN", + "PROMO POLISHED NICKEL", "PROMO POLISHED BRASS", "PROMO POLISHED STEEL", + "PROMO POLISHED COPPER", "PROMO BRUSHED TIN", "PROMO BRUSHED NICKEL", + "PROMO BRUSHED BRASS", "PROMO BRUSHED STEEL", "PROMO BRUSHED COPPER"}; + +constexpr std::array vocab_containers{ + "SM CASE", "SM BOX", "SM BAG", "SM JAR", "SM PKG", "SM PACK", "SM CAN", + "SM DRUM", "LG CASE", "LG BOX", "LG BAG", "LG JAR", "LG PKG", "LG PACK", + "LG CAN", "LG DRUM", "MED CASE", "MED BOX", "MED BAG", "MED JAR", "MED PKG", + "MED PACK", "MED CAN", "MED DRUM", "JUMBO CASE", "JUMBO BOX", "JUMBO BAG", "JUMBO JAR", + "JUMBO PKG", "JUMBO PACK", "JUMBO CAN", "JUMBO DRUM", "WRAP CASE", "WRAP BOX", "WRAP BAG", + "WRAP JAR", "WRAP PKG", "WRAP PACK", "WRAP CAN", "WRAP DRUM"}; + +} // namespace + +/** + * @brief Generate a table out of the independent columns of the `orders` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_orders_independent(double scale_factor, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + cudf::size_type const o_num_rows = scale_factor * 1'500'000; + + // Generate the `o_orderkey` column + auto o_orderkey = [&]() { + auto const o_orderkey_candidates = generate_primary_key_column( + cudf::numeric_scalar(1), 4 * o_num_rows, stream, mr); + auto const o_orderkey_unsorted = cudf::sample(cudf::table_view({o_orderkey_candidates->view()}), + o_num_rows, + cudf::sample_with_replacement::FALSE, + 0, + stream, + mr); + auto const sort_result = + cudf::sort_by_key(o_orderkey_unsorted->view(), + cudf::table_view({o_orderkey_unsorted->view().column(0)}), + {}, + {}, + stream, + mr); + return std::move(sort_result->release()[0]); + }(); + + // Generate the `o_custkey` column + auto o_custkey = [&]() { + auto const col = generate_random_numeric_column( + 1, scale_factor * 49'000, o_num_rows, stream, mr); + auto const col_mul_3 = cudf::binary_operation(col->view(), + cudf::numeric_scalar(3), + cudf::binary_operator::MUL, + cudf::data_type{cudf::type_id::INT32}, + stream, + mr); + return cudf::binary_operation(col_mul_3->view(), + cudf::numeric_scalar(1), + cudf::binary_operator::ADD, + cudf::data_type{cudf::type_id::INT32}, + stream, + mr); + }(); + + // Generate the `o_orderdate` column + auto o_orderdate_ts = [&]() { + auto const o_orderdate_year = generate_random_string_column_from_set( + cudf::host_span(years.data(), years.size()), o_num_rows, stream, mr); + auto const o_orderdate_month = generate_random_string_column_from_set( + cudf::host_span(months.data(), months.size()), o_num_rows, stream, mr); + auto const o_orderdate_day = generate_random_string_column_from_set( + cudf::host_span(days.data(), days.size()), o_num_rows, stream, mr); + auto const o_orderdate_str = cudf::strings::concatenate( + cudf::table_view( + {o_orderdate_year->view(), o_orderdate_month->view(), o_orderdate_day->view()}), + cudf::string_scalar("-"), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); + + return cudf::strings::to_timestamps(o_orderdate_str->view(), + cudf::data_type{cudf::type_id::TIMESTAMP_DAYS}, + std::string("%Y-%m-%d"), + stream, + mr); + }(); + + // Generate the `o_orderpriority` column + auto o_orderpriority = generate_random_string_column_from_set( + cudf::host_span(vocab_priorities.data(), vocab_priorities.size()), + o_num_rows, + stream, + mr); + + // Generate the `o_clerk` column + auto o_clerk = [&]() { + auto const clerk_repeat = generate_repeat_string_column("Clerk#", o_num_rows, stream, mr); + auto const random_c = generate_random_numeric_column( + 1, scale_factor * 1'000, o_num_rows, stream, mr); + auto const random_c_str = cudf::strings::from_integers(random_c->view(), stream, mr); + auto const random_c_str_padded = cudf::strings::zfill(random_c_str->view(), 9, stream, mr); + return cudf::strings::concatenate( + cudf::table_view({clerk_repeat->view(), random_c_str_padded->view()}), + cudf::string_scalar(""), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); + }(); + + // Generate the `o_shippriority` column + auto o_shippriority = [&]() { + auto const empty = cudf::make_numeric_column( + cudf::data_type{cudf::type_id::INT8}, o_num_rows, cudf::mask_state::UNALLOCATED, stream); + return cudf::fill(empty->view(), 0, o_num_rows, cudf::numeric_scalar(0), stream, mr); + }(); + + // Generate the `o_comment` column + // NOTE: This column is not compliant with clause 4.2.2.10 of the TPC-H specification + auto o_comment = generate_random_string_column(19, 78, o_num_rows, stream, mr); + + // Generate the `orders_independent` table + std::vector> columns; + columns.push_back(std::move(o_orderkey)); + columns.push_back(std::move(o_custkey)); + columns.push_back(std::move(o_orderdate_ts)); + columns.push_back(std::move(o_orderpriority)); + columns.push_back(std::move(o_clerk)); + columns.push_back(std::move(o_shippriority)); + columns.push_back(std::move(o_comment)); + return std::make_unique(std::move(columns)); +} + +/** + * @brief Generate the `lineitem` table partially + * + * @param orders_independent Table with the independent columns of the `orders` table + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_lineitem_partial(cudf::table_view const& orders_independent, + double scale_factor, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const o_num_rows = orders_independent.num_rows(); + // Generate the `lineitem` table. For each row in the `orders` table, + // we have a random number (between 1 and 7) of rows in the `lineitem` table + + // For each `o_orderkey`, generate a random number (between 1 and 7), + // which will be the number of rows in the `lineitem` table that will + // have the same `l_orderkey` + auto const o_rep_freqs = generate_random_numeric_column(1, 7, o_num_rows, stream, mr); + + // Sum up the `o_rep_freqs` to get the number of rows in the + // `lineitem` table. This is required to generate the independent columns + // in the `lineitem` table + auto const l_num_rows = calculate_l_cardinality(o_rep_freqs->view(), stream, mr); + + // We create a table out of `o_orderkey` and `o_orderdate_ts` by repeating + // the rows of `orders` according to the frequencies in `o_rep_freqs` + auto const o_orderkey = orders_independent.column(0); + auto const o_orderdate_ts = orders_independent.column(2); + auto const l_base = + cudf::repeat(cudf::table_view({o_orderkey, o_orderdate_ts}), o_rep_freqs->view(), stream, mr); + auto l_base_columns = l_base->release(); + + // Generate the `l_orderkey` column + auto l_orderkey = std::move(l_base_columns[0]); + + // Generate the `l_partkey` column + auto l_partkey = generate_random_numeric_column( + 1, scale_factor * 200'000, l_num_rows, stream, mr); + + // Generate the `l_suppkey` column + auto l_suppkey = calculate_l_suppkey(l_partkey->view(), scale_factor, l_num_rows, stream, mr); + + // Generate the `l_linenumber` column + auto l_linenumber = generate_repeat_sequence_column(7, false, l_num_rows, stream, mr); + + // Generate the `l_quantity` column + auto l_quantity = generate_random_numeric_column(1, 50, l_num_rows, stream, mr); + + // Generate the `l_discount` column + auto l_discount = [&]() { + auto const col = generate_random_numeric_column(0.00, 0.10, l_num_rows, stream, mr); + return cudf::round(col->view(), 2); + }(); + + // Generate the `l_tax` column + auto l_tax = [&]() { + auto const col = generate_random_numeric_column(0.00, 0.08, l_num_rows, stream, mr); + return cudf::round(col->view(), 2); + }(); + + // Get the orderdate column from the `l_base` table + auto const ol_orderdate_ts = std::move(l_base_columns[1]); + + // Generate the `l_shipdate` column + auto l_shipdate_ts = [&]() { + auto const l_shipdate_rand_add_days = + generate_random_numeric_column(1, 121, l_num_rows, stream, mr); + return add_calendrical_days( + ol_orderdate_ts->view(), l_shipdate_rand_add_days->view(), stream, mr); + }(); + + // Generate the `l_commitdate` column + auto l_commitdate_ts = [&]() { + auto const l_commitdate_rand_add_days = + generate_random_numeric_column(30, 90, l_num_rows, stream, mr); + return add_calendrical_days( + ol_orderdate_ts->view(), l_commitdate_rand_add_days->view(), stream, mr); + }(); + + // Generate the `l_receiptdate` column + auto l_receiptdate_ts = [&]() { + auto const l_receiptdate_rand_add_days = + generate_random_numeric_column(1, 30, l_num_rows, stream, mr); + return add_calendrical_days( + l_shipdate_ts->view(), l_receiptdate_rand_add_days->view(), stream, mr); + }(); + + // Define the current date as per clause 4.2.2.12 of the TPC-H specification + constexpr cudf::size_type current_date_days_since_epoch = 9'298; + auto current_date = + cudf::timestamp_scalar(current_date_days_since_epoch, true); + auto current_date_literal = cudf::ast::literal(current_date); + + // Generate the `l_returnflag` column + // if `l_receiptdate` <= current_date then "R" or "A" else "N" + auto l_returnflag = [&]() { + auto const col_ref = cudf::ast::column_reference(0); + auto const pred = + cudf::ast::operation(cudf::ast::ast_operator::LESS_EQUAL, col_ref, current_date_literal); + auto const binary_mask = + cudf::compute_column(cudf::table_view({l_receiptdate_ts->view()}), pred, stream, mr); + + auto const multiplier = + generate_repeat_sequence_column(2, false, l_num_rows, stream, mr); + auto const ternary_mask = cudf::binary_operation(binary_mask->view(), + multiplier->view(), + cudf::binary_operator::MUL, + cudf::data_type{cudf::type_id::INT8}, + stream, + mr); + auto const indices = cudf::test::fixed_width_column_wrapper({0, 1, 2}).release(); + auto const keys = cudf::test::strings_column_wrapper({"N", "A", "R"}).release(); + auto const gather_map = cudf::table_view({indices->view(), keys->view()}); + auto const gathered_table = cudf::gather( + gather_map, ternary_mask->view(), cudf::out_of_bounds_policy::DONT_CHECK, stream, mr); + return std::move(gathered_table->release()[1]); + }(); + + // Generate the `l_linestatus` column + // if `l_shipdate` > current_date then "F" else "O" + auto [l_linestatus, l_linestatus_mask] = [&]() { + auto const col_ref = cudf::ast::column_reference(0); + auto const pred = + cudf::ast::operation(cudf::ast::ast_operator::GREATER, col_ref, current_date_literal); + auto mask = cudf::compute_column(cudf::table_view({l_shipdate_ts->view()}), pred, stream, mr); + auto mask_index_type = cudf::cast(mask->view(), cudf::data_type{cudf::type_id::INT8}); + auto const indices = cudf::test::fixed_width_column_wrapper({0, 1}).release(); + auto const keys = cudf::test::strings_column_wrapper({"O", "F"}).release(); + auto const gather_map = cudf::table_view({indices->view(), keys->view()}); + auto const gathered_table = cudf::gather( + gather_map, mask_index_type->view(), cudf::out_of_bounds_policy::DONT_CHECK, stream, mr); + return std::make_tuple(std::move(gathered_table->release()[1]), std::move(mask_index_type)); + }(); + + // Generate the `l_shipinstruct` column + auto l_shipinstruct = generate_random_string_column_from_set( + cudf::host_span(vocab_instructions.data(), vocab_instructions.size()), + l_num_rows, + stream, + mr); + + // Generate the `l_shipmode` column + auto l_shipmode = generate_random_string_column_from_set( + cudf::host_span(vocab_modes.data(), vocab_modes.size()), + l_num_rows, + stream, + mr); + + // Generate the `l_comment` column + // NOTE: This column is not compliant with + // clause 4.2.2.10 of the TPC-H specification + auto l_comment = generate_random_string_column(10, 43, l_num_rows, stream, mr); + + // Generate the `lineitem_partial` table + std::vector> columns; + columns.push_back(std::move(l_linestatus_mask)); + columns.push_back(std::move(l_orderkey)); + columns.push_back(std::move(l_partkey)); + columns.push_back(std::move(l_suppkey)); + columns.push_back(std::move(l_linenumber)); + columns.push_back(std::move(l_quantity)); + columns.push_back(std::move(l_discount)); + columns.push_back(std::move(l_tax)); + columns.push_back(std::move(l_shipdate_ts)); + columns.push_back(std::move(l_commitdate_ts)); + columns.push_back(std::move(l_receiptdate_ts)); + columns.push_back(std::move(l_returnflag)); + columns.push_back(std::move(l_linestatus)); + columns.push_back(std::move(l_shipinstruct)); + columns.push_back(std::move(l_shipmode)); + columns.push_back(std::move(l_comment)); + return std::make_unique(std::move(columns)); +} + +std::unique_ptr generate_orders_dependent(cudf::table_view const& lineitem, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const l_linestatus_mask = lineitem.column(0); + auto const l_orderkey = lineitem.column(1); + auto const l_discount = lineitem.column(6); + auto const l_tax = lineitem.column(7); + auto const l_extendedprice = lineitem.column(16); + + std::vector> orders_dependent_columns; + + // Generate the `o_totalprice` column + // We calculate the `charge` column, which is a function of `l_extendedprice`, + // `l_tax`, and `l_discount` and then group by `l_orderkey` and sum the `charge` + auto const l_charge = calculate_charge(l_extendedprice, l_tax, l_discount, stream, mr); + auto o_totalprice = [&]() { + auto const keys = cudf::table_view({l_orderkey}); + cudf::groupby::groupby gb(keys); + std::vector requests; + requests.push_back(cudf::groupby::aggregation_request()); + requests[0].aggregations.push_back(cudf::make_sum_aggregation()); + requests[0].values = l_charge->view(); + auto agg_result = gb.aggregate(requests); + return cudf::round(agg_result.second[0].results[0]->view(), 2); + }(); + orders_dependent_columns.push_back(std::move(o_totalprice)); + + // Generate the `o_orderstatus` column + auto o_orderstatus = [&]() { + auto const keys = cudf::table_view({l_orderkey}); + cudf::groupby::groupby gb(keys); + std::vector requests; + + // Perform a `count` aggregation on `l_orderkey` + requests.push_back(cudf::groupby::aggregation_request()); + requests[0].aggregations.push_back(cudf::make_count_aggregation()); + requests[0].values = l_orderkey; + + // Perform a `sum` aggregation on `l_linestatus_mask` + requests.push_back(cudf::groupby::aggregation_request()); + requests[1].aggregations.push_back(cudf::make_sum_aggregation()); + requests[1].values = l_linestatus_mask; + + // Perform the aggregations + auto agg_result = gb.aggregate(requests); + + // Create a `table_view` out of the `l_orderkey`, `count`, and `sum` columns + auto const count = std::move(agg_result.second[0].results[0]); + auto const sum = cudf::cast( + agg_result.second[1].results[0]->view(), cudf::data_type{cudf::type_id::INT32}, stream, mr); + + auto const table = + cudf::table_view({agg_result.first->get_column(0).view(), count->view(), sum->view()}); + + // Now on this table, + // if `sum` == `count` then "O", + // if `sum` == 0, then "F", + // else "P" + + // So, we first evaluate an expression `sum == count` and generate a boolean mask + auto const count_ref = cudf::ast::column_reference(1); + auto const sum_ref = cudf::ast::column_reference(2); + auto const expr_a = cudf::ast::operation(cudf::ast::ast_operator::EQUAL, sum_ref, count_ref); + auto const mask_a = cudf::compute_column(table, expr_a); + auto const o_orderstatus_intermediate = + cudf::copy_if_else(cudf::string_scalar("O"), cudf::string_scalar("F"), mask_a->view()); + + // Then, we evaluate an expression `sum == 0` and generate a boolean mask + auto zero_scalar = cudf::numeric_scalar(0); + auto const zero_literal = cudf::ast::literal(zero_scalar); + auto const expr_b_left = + cudf::ast::operation(cudf::ast::ast_operator::NOT_EQUAL, sum_ref, count_ref); + auto const expr_b_right = + cudf::ast::operation(cudf::ast::ast_operator::NOT_EQUAL, sum_ref, zero_literal); + auto const expr_b = + cudf::ast::operation(cudf::ast::ast_operator::LOGICAL_AND, expr_b_left, expr_b_right); + auto const mask_b = cudf::compute_column(table, expr_b); + return cudf::copy_if_else( + cudf::string_scalar("P"), o_orderstatus_intermediate->view(), mask_b->view()); + }(); + orders_dependent_columns.push_back(std::move(o_orderstatus)); + return std::make_unique(std::move(orders_dependent_columns)); +} + +/** + * @brief Generate the `partsupp` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_partsupp(double scale_factor, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Define the number of rows in the `part` and `partsupp` tables + cudf::size_type const p_num_rows = scale_factor * 200'000; + cudf::size_type const ps_num_rows = scale_factor * 800'000; + + // Generate the `ps_partkey` column + auto ps_partkey = [&]() { + auto const p_partkey = + generate_primary_key_column(cudf::numeric_scalar(1), p_num_rows, stream, mr); + auto const rep_table = cudf::repeat(cudf::table_view({p_partkey->view()}), 4, stream, mr); + return std::move(rep_table->release()[0]); + }(); + + // Generate the `ps_suppkey` column + auto ps_suppkey = calculate_ps_suppkey(ps_partkey->view(), scale_factor, ps_num_rows, stream, mr); + + // Generate the `ps_availqty` column + auto ps_availqty = generate_random_numeric_column(1, 9999, ps_num_rows, stream, mr); + + // Generate the `ps_supplycost` column + auto ps_supplycost = [&]() { + auto const col = generate_random_numeric_column(1.00, 1000.00, ps_num_rows, stream, mr); + return cudf::round(col->view(), 2); + }(); + + // Generate the `ps_comment` column + // NOTE: This column is not compliant with clause 4.2.2.10 of the TPC-H specification + auto ps_comment = generate_random_string_column(49, 198, ps_num_rows, stream, mr); + + // Create the `partsupp` table + std::vector> columns; + columns.push_back(std::move(ps_partkey)); + columns.push_back(std::move(ps_suppkey)); + columns.push_back(std::move(ps_availqty)); + columns.push_back(std::move(ps_supplycost)); + columns.push_back(std::move(ps_comment)); + return std::make_unique(std::move(columns)); +} + +/** + * @brief Generate the `part` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_part(double scale_factor, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + cudf::size_type const num_rows = scale_factor * 200'000; + + // Generate the `p_partkey` column + auto p_partkey = + generate_primary_key_column(cudf::numeric_scalar(1), num_rows, stream, mr); + + // Generate the `p_name` column + auto p_name = [&]() { + auto const p_name_a = generate_random_string_column_from_set( + cudf::host_span(vocab_p_name.data(), vocab_p_name.size()), + num_rows, + stream, + mr); + auto const p_name_b = generate_random_string_column_from_set( + cudf::host_span(vocab_p_name.data(), vocab_p_name.size()), + num_rows, + stream, + mr); + auto const p_name_c = generate_random_string_column_from_set( + cudf::host_span(vocab_p_name.data(), vocab_p_name.size()), + num_rows, + stream, + mr); + auto const p_name_d = generate_random_string_column_from_set( + cudf::host_span(vocab_p_name.data(), vocab_p_name.size()), + num_rows, + stream, + mr); + auto const p_name_e = generate_random_string_column_from_set( + cudf::host_span(vocab_p_name.data(), vocab_p_name.size()), + num_rows, + stream, + mr); + return cudf::strings::concatenate( + cudf::table_view( + {p_name_a->view(), p_name_b->view(), p_name_c->view(), p_name_d->view(), p_name_e->view()}), + cudf::string_scalar(" "), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); + }(); + + // Generate the `p_mfgr` and `p_brand` columns + auto const random_values_m = generate_random_numeric_column(1, 5, num_rows, stream, mr); + auto const random_values_m_str = + cudf::strings::from_integers(random_values_m->view(), stream, mr); + + auto const random_values_n = generate_random_numeric_column(1, 5, num_rows, stream, mr); + auto const random_values_n_str = + cudf::strings::from_integers(random_values_n->view(), stream, mr); + + auto p_mfgr = [&]() { + auto const mfgr_repeat = generate_repeat_string_column("Manufacturer#", num_rows, stream, mr); + return cudf::strings::concatenate( + cudf::table_view({mfgr_repeat->view(), random_values_m_str->view()}), + cudf::string_scalar(""), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); + }(); + + auto p_brand = [&]() { + auto const brand_repeat = generate_repeat_string_column("Brand#", num_rows, stream, mr); + return cudf::strings::concatenate( + cudf::table_view( + {brand_repeat->view(), random_values_m_str->view(), random_values_n_str->view()}), + cudf::string_scalar(""), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); + }(); + + // Generate the `p_type` column + auto p_type = generate_random_string_column_from_set( + cudf::host_span(vocab_types.data(), vocab_types.size()), + num_rows, + stream, + mr); + + // Generate the `p_size` column + auto p_size = generate_random_numeric_column(1, 50, num_rows, stream, mr); + + // Generate the `p_container` column + auto p_container = generate_random_string_column_from_set( + cudf::host_span(vocab_containers.data(), vocab_containers.size()), + num_rows, + stream, + mr); + + // Generate the `p_retailprice` column + auto p_retailprice = calculate_p_retailprice(p_partkey->view(), stream, mr); + + // Generate the `p_comment` column + // NOTE: This column is not compliant with clause 4.2.2.10 of the TPC-H specification + auto p_comment = generate_random_string_column(5, 22, num_rows, stream, mr); + + // Create the `part` table + std::vector> columns; + columns.push_back(std::move(p_partkey)); + columns.push_back(std::move(p_name)); + columns.push_back(std::move(p_mfgr)); + columns.push_back(std::move(p_brand)); + columns.push_back(std::move(p_type)); + columns.push_back(std::move(p_size)); + columns.push_back(std::move(p_container)); + columns.push_back(std::move(p_retailprice)); + columns.push_back(std::move(p_comment)); + return std::make_unique(std::move(columns)); +} + +/** + * @brief Generate the `orders`, `lineitem`, and `part` tables + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::tuple, std::unique_ptr, std::unique_ptr> +generate_orders_lineitem_part(double scale_factor, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Generate a table with the independent columns of the `orders` table + auto orders_independent = generate_orders_independent(scale_factor, stream, mr); + + // Generate the `lineitem` table partially + auto lineitem_partial = + generate_lineitem_partial(orders_independent->view(), scale_factor, stream, mr); + + // Generate the `part` table + auto part = generate_part(scale_factor, stream, mr); + + // Join the `part` and partial `lineitem` tables, then calculate the `l_extendedprice` column, + // add the column to the `lineitem` table, and write the `lineitem` table to a parquet file + + auto l_extendedprice = [&]() { + auto const left = cudf::table_view( + {lineitem_partial->get_column(2).view(), lineitem_partial->get_column(5).view()}); + auto const right = cudf::table_view({part->get_column(0).view(), part->get_column(7).view()}); + auto const joined_table = perform_left_join(left, right, {0}, {0}, stream, mr); + auto joined_table_columns = joined_table->release(); + auto const l_quantity = std::move(joined_table_columns[1]); + auto const l_quantity_fp = + cudf::cast(l_quantity->view(), cudf::data_type{cudf::type_id::FLOAT64}); + auto const p_retailprice = std::move(joined_table_columns[3]); + auto const col = cudf::binary_operation(l_quantity_fp->view(), + p_retailprice->view(), + cudf::binary_operator::MUL, + cudf::data_type{cudf::type_id::FLOAT64}, + stream, + mr); + return cudf::round(col->view(), 2); + }(); + + auto lineitem_partial_columns = lineitem_partial->release(); + lineitem_partial_columns.push_back(std::move(l_extendedprice)); + auto lineitem_temp = std::make_unique(std::move(lineitem_partial_columns)); + + // Generate the dependent columns of the `orders` table + // and merge them with the independent columns + auto orders_dependent = generate_orders_dependent(lineitem_temp->view(), stream, mr); + + auto orders_independent_columns = orders_independent->release(); + auto orders_dependent_columns = orders_dependent->release(); + orders_independent_columns.insert(orders_independent_columns.end(), + std::make_move_iterator(orders_dependent_columns.begin()), + std::make_move_iterator(orders_dependent_columns.end())); + + // Create the `orders` table + auto orders = std::make_unique(std::move(orders_independent_columns)); + + // Create the `lineitem` table + auto lineitem_temp_columns = lineitem_temp->release(); + lineitem_temp_columns.erase(lineitem_temp_columns.begin()); + auto lineitem = std::make_unique(std::move(lineitem_temp_columns)); + + return std::make_tuple(std::move(orders), std::move(lineitem), std::move(part)); +} + +/** + * @brief Generate the `supplier` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_supplier(double scale_factor, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Calculate the number of rows based on the scale factor + cudf::size_type const num_rows = scale_factor * 10'000; + + // Generate the `s_suppkey` column + auto s_suppkey = + generate_primary_key_column(cudf::numeric_scalar(1), num_rows, stream, mr); + + // Generate the `s_name` column + auto s_name = [&]() { + auto const supplier_repeat = generate_repeat_string_column("Supplier#", num_rows, stream, mr); + auto const s_suppkey_str = cudf::strings::from_integers(s_suppkey->view(), stream, mr); + auto const s_suppkey_str_padded = cudf::strings::zfill(s_suppkey_str->view(), 9, stream, mr); + return cudf::strings::concatenate( + cudf::table_view({supplier_repeat->view(), s_suppkey_str_padded->view()}), + cudf::string_scalar(""), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); + }(); + + // Generate the `s_address` column + auto s_address = generate_address_column(num_rows, stream, mr); + + // Generate the `s_nationkey` column + auto s_nationkey = generate_random_numeric_column(0, 24, num_rows, stream, mr); + + // Generate the `s_phone` column + auto s_phone = generate_phone_column(num_rows, stream, mr); + + // Generate the `s_acctbal` column + auto s_acctbal = [&]() { + auto const col = generate_random_numeric_column(-999.99, 9999.99, num_rows, stream, mr); + return cudf::round(col->view(), 2); + }(); + + // Generate the `s_comment` column + // NOTE: This column is not compliant with clause 4.2.2.10 of the TPC-H specification + auto s_comment = generate_random_string_column(25, 100, num_rows, stream, mr); + + // Create the `supplier` table + std::vector> columns; + columns.push_back(std::move(s_suppkey)); + columns.push_back(std::move(s_name)); + columns.push_back(std::move(s_address)); + columns.push_back(std::move(s_nationkey)); + columns.push_back(std::move(s_phone)); + columns.push_back(std::move(s_acctbal)); + columns.push_back(std::move(s_comment)); + return std::make_unique(std::move(columns)); +} + +/** + * @brief Generate the `customer` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_customer(double scale_factor, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Calculate the number of rows based on the scale factor + cudf::size_type const num_rows = scale_factor * 150'000; + + // Generate the `c_custkey` column + auto c_custkey = + generate_primary_key_column(cudf::numeric_scalar(1), num_rows, stream, mr); + + // Generate the `c_name` column + auto c_name = [&]() { + auto const customer_repeat = generate_repeat_string_column("Customer#", num_rows, stream, mr); + auto const c_custkey_str = cudf::strings::from_integers(c_custkey->view(), stream, mr); + auto const c_custkey_str_padded = cudf::strings::zfill(c_custkey_str->view(), 9, stream, mr); + return cudf::strings::concatenate( + cudf::table_view({customer_repeat->view(), c_custkey_str_padded->view()}), + cudf::string_scalar(""), + cudf::string_scalar("", false), + cudf::strings::separator_on_nulls::NO, + stream, + mr); + }(); + + // Generate the `c_address` column + auto c_address = generate_address_column(num_rows, stream, mr); + + // Generate the `c_nationkey` column + auto c_nationkey = generate_random_numeric_column(0, 24, num_rows, stream, mr); + + // Generate the `c_phone` column + auto c_phone = generate_phone_column(num_rows, stream, mr); + + // Generate the `c_acctbal` column + auto c_acctbal = [&]() { + auto const col = generate_random_numeric_column(-999.99, 9999.99, num_rows, stream, mr); + return cudf::round(col->view(), 2); + }(); + + // Generate the `c_mktsegment` column + auto c_mktsegment = generate_random_string_column_from_set( + cudf::host_span(vocab_segments.data(), vocab_segments.size()), + num_rows, + stream, + mr); + + // Generate the `c_comment` column + // NOTE: This column is not compliant with clause 4.2.2.10 of the TPC-H specification + auto c_comment = generate_random_string_column(29, 116, num_rows, stream, mr); + + // Create the `customer` table + std::vector> columns; + columns.push_back(std::move(c_custkey)); + columns.push_back(std::move(c_name)); + columns.push_back(std::move(c_address)); + columns.push_back(std::move(c_nationkey)); + columns.push_back(std::move(c_phone)); + columns.push_back(std::move(c_acctbal)); + columns.push_back(std::move(c_mktsegment)); + columns.push_back(std::move(c_comment)); + return std::make_unique(std::move(columns)); +} + +/** + * @brief Generate the `nation` table + * + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_nation(rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Define the number of rows + constexpr cudf::size_type num_rows = 25; + + // Generate the `n_nationkey` column + auto n_nationkey = + generate_primary_key_column(cudf::numeric_scalar(0), num_rows, stream, mr); + + // Generate the `n_name` column + auto n_name = cudf::test::strings_column_wrapper(nations.begin(), nations.end()).release(); + + // Generate the `n_regionkey` column + std::vector region_keys{0, 1, 1, 1, 4, 0, 3, 3, 2, 2, 4, 4, 2, + 4, 0, 0, 0, 1, 2, 3, 4, 2, 3, 3, 1}; + auto n_regionkey = + cudf::test::fixed_width_column_wrapper(region_keys.begin(), region_keys.end()) + .release(); + + // Generate the `n_comment` column + // NOTE: This column is not compliant with clause 4.2.2.10 of the TPC-H specification + auto n_comment = generate_random_string_column(31, 114, num_rows, stream, mr); + + // Create the `nation` table + std::vector> columns; + columns.push_back(std::move(n_nationkey)); + columns.push_back(std::move(n_name)); + columns.push_back(std::move(n_regionkey)); + columns.push_back(std::move(n_comment)); + return std::make_unique(std::move(columns)); +} + +/** + * @brief Generate the `region` table + * + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_region(rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + // Define the number of rows + constexpr cudf::size_type num_rows = 5; + + // Generate the `r_regionkey` column + auto r_regionkey = + generate_primary_key_column(cudf::numeric_scalar(0), num_rows, stream, mr); + + // Generate the `r_name` column + auto r_name = + cudf::test::strings_column_wrapper({"AFRICA", "AMERICA", "ASIA", "EUROPE", "MIDDLE EAST"}) + .release(); + + // Generate the `r_comment` column + // NOTE: This column is not compliant with clause 4.2.2.10 of the TPC-H specification + auto r_comment = generate_random_string_column(31, 115, num_rows, stream, mr); + + // Create the `region` table + std::vector> columns; + columns.push_back(std::move(r_regionkey)); + columns.push_back(std::move(r_name)); + columns.push_back(std::move(r_comment)); + return std::make_unique(std::move(columns)); +} + +} // namespace cudf::datagen diff --git a/cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.hpp b/cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.hpp new file mode 100644 index 00000000000..a6286dd8dba --- /dev/null +++ b/cpp/benchmarks/common/tpch_data_generator/tpch_data_generator.hpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace CUDF_EXPORT cudf { +namespace datagen { + +/** + * @brief Generate the `orders`, `lineitem`, and `part` tables + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::tuple, std::unique_ptr, std::unique_ptr> +generate_orders_lineitem_part( + double scale_factor, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `partsupp` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_partsupp( + double scale_factor, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `supplier` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_supplier( + double scale_factor, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `customer` table + * + * @param scale_factor The scale factor to generate + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_customer( + double scale_factor, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `nation` table + * + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_nation( + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +/** + * @brief Generate the `region` table + * + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr generate_region( + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()); + +} // namespace datagen +} // namespace CUDF_EXPORT cudf From 5a81a80cef59649f059d55004f745001a59b3f6f Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:57:53 -0400 Subject: [PATCH 154/270] [BUG] Add gpu node type to cudf-pandas 3rd-party integration nightly CI job (#16704) Following up #16645, and adding a gpu node type to the nightly CI job Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16704 --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2c68f2861bb..8605fa46f68 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -132,6 +132,7 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + node_type: "gpu-v100-latest-1" container_image: "rapidsai/ci-conda:latest" run_script: | ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml From 2d6758f39592e6296a042eb8e771171c50899013 Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Fri, 30 Aug 2024 13:22:58 -0700 Subject: [PATCH 155/270] Enable batched multi-source reading of JSONL files with large records (#16687) Addresses #16664 Implements reallocate-and-retry logic when the initial buffer size estimate fails for byte range reading. Chunked reader test checks for correct reallocation for different chunk sizes. Authors: - Shruti Shivakumar (https://github.com/shrshi) Approvers: - Karthikeyan (https://github.com/karthikeyann) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/16687 --- cpp/src/io/json/read_json.cu | 48 ++++++++++++++++++++++++--------- cpp/tests/io/json/json_test.cpp | 47 ++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 13 deletions(-) diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index 2658cbbed2f..98e8e8d3c7e 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -138,14 +138,14 @@ datasource::owning_buffer get_record_range_raw_input( auto should_load_all_sources = !chunk_size || chunk_size >= total_source_size - chunk_offset; chunk_size = should_load_all_sources ? total_source_size - chunk_offset : chunk_size; - int const num_subchunks_prealloced = should_load_all_sources ? 0 : max_subchunks_prealloced; + int num_subchunks_prealloced = should_load_all_sources ? 0 : max_subchunks_prealloced; std::size_t const size_per_subchunk = estimate_size_per_subchunk(chunk_size); // The allocation for single source compressed input is estimated by assuming a ~4:1 // compression ratio. For uncompressed inputs, we can getter a better estimate using the idea // of subchunks. auto constexpr header_size = 4096; - std::size_t const buffer_size = + std::size_t buffer_size = reader_compression != compression_type::NONE ? total_source_size * estimated_compression_ratio + header_size : std::min(total_source_size, chunk_size + num_subchunks_prealloced * size_per_subchunk) + @@ -169,18 +169,40 @@ datasource::owning_buffer get_record_range_raw_input( // Find next delimiter std::int64_t next_delim_pos = -1; std::size_t next_subchunk_start = chunk_offset + chunk_size; - while (next_subchunk_start < total_source_size && next_delim_pos < buffer_offset) { - buffer_offset += readbufspan.size(); - readbufspan = ingest_raw_input(bufspan.last(buffer_size - buffer_offset), - sources, - reader_compression, - next_subchunk_start, - size_per_subchunk, - stream); - next_delim_pos = find_first_delimiter(readbufspan, '\n', stream) + buffer_offset; - if (next_delim_pos < buffer_offset) { next_subchunk_start += size_per_subchunk; } + while (next_delim_pos < buffer_offset) { + for (int subchunk = 0; + subchunk < num_subchunks_prealloced && next_delim_pos < buffer_offset && + next_subchunk_start < total_source_size; + subchunk++) { + buffer_offset += readbufspan.size(); + readbufspan = ingest_raw_input(bufspan.last(buffer_size - buffer_offset), + sources, + reader_compression, + next_subchunk_start, + size_per_subchunk, + stream); + next_delim_pos = find_first_delimiter(readbufspan, '\n', stream) + buffer_offset; + next_subchunk_start += size_per_subchunk; + } + if (next_delim_pos < buffer_offset) { + if (next_subchunk_start >= total_source_size) { + // If we have reached the end of source list but the source does not terminate with a + // newline character + next_delim_pos = buffer_offset + readbufspan.size(); + } else { + // Our buffer_size estimate is insufficient to read until the end of the line! We need to + // allocate more memory and try again! + num_subchunks_prealloced *= 2; + buffer_size = reader_compression != compression_type::NONE + ? 2 * buffer_size + : std::min(total_source_size, + buffer_size + num_subchunks_prealloced * size_per_subchunk) + + num_extra_delimiters; + buffer.resize(buffer_size, stream); + bufspan = device_span(reinterpret_cast(buffer.data()), buffer.size()); + } + } } - if (next_delim_pos < buffer_offset) next_delim_pos = buffer_offset + readbufspan.size(); return datasource::owning_buffer( std::move(buffer), diff --git a/cpp/tests/io/json/json_test.cpp b/cpp/tests/io/json/json_test.cpp index 576a698ba31..c26e5ca3edb 100644 --- a/cpp/tests/io/json/json_test.cpp +++ b/cpp/tests/io/json/json_test.cpp @@ -680,6 +680,53 @@ TEST_F(JsonReaderTest, JsonLinesByteRange) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->get_column(0), int64_wrapper{{3000, 4000, 5000}}); } +TEST_F(JsonReaderTest, JsonLinesByteRangeWithRealloc) +{ + std::string long_string = "haha"; + std::size_t log_repetitions = 12; + long_string.reserve(long_string.size() * (1UL << log_repetitions)); + for (std::size_t i = 0; i < log_repetitions; i++) { + long_string += long_string; + } + + auto json_string = [&long_string]() { + std::string json_string = R"( + { "a": { "y" : 6}, "b" : [1, 2, 3], "c": 11 } + { "a": { "y" : 6}, "b" : [4, 5 ], "c": 12 } + { "a": { "y" : 6}, "b" : [6 ], "c": 13 } + { "a": { "y" : 6}, "b" : [7 ], "c": 14 })"; + std::string replace_chars = "c"; + std::size_t pos = json_string.find(replace_chars); + while (pos != std::string::npos) { + // Replace the substring with the specified string + json_string.replace(pos, replace_chars.size(), long_string); + + // Find the next occurrence of the substring + pos = json_string.find(replace_chars, pos + long_string.size()); + } + return json_string; + }(); + + // Initialize parsing options (reading json lines). Set byte range offset and size so as to read + // the second row of input + cudf::io::json_reader_options json_lines_options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{cudf::host_span( + reinterpret_cast(json_string.data()), json_string.size())}) + .lines(true) + .compression(cudf::io::compression_type::NONE) + .recovery_mode(cudf::io::json_recovery_mode_t::FAIL) + .byte_range_offset(16430) + .byte_range_size(30); + + // Read full test data via existing, nested JSON lines reader + cudf::io::table_with_metadata result = cudf::io::read_json(json_lines_options); + + EXPECT_EQ(result.tbl->num_columns(), 3); + EXPECT_EQ(result.tbl->num_rows(), 1); + EXPECT_EQ(result.metadata.schema_info[2].name, long_string); +} + TEST_F(JsonReaderTest, JsonLinesMultipleFilesByteRange_AcrossFiles) { const std::string file1 = temp_env->get_temp_dir() + "JsonLinesMultipleFilesByteRangeTest1.json"; From c6c720f48815ec93a543cb42fbb128d3c0eb983e Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:47:26 -0400 Subject: [PATCH 156/270] Implement exposed null mask APIs in pylibcudf (#15908) Contributes to https://github.com/rapidsai/cudf/issues/15162 Authors: - Charles Blackmon-Luca (https://github.com/charlesbluca) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15908 --- docs/cudf/source/conf.py | 2 + .../user_guide/api_docs/pylibcudf/index.rst | 1 + .../api_docs/pylibcudf/null_mask.rst | 6 + python/cudf/cudf/_lib/null_mask.pyx | 103 +++---------- python/pylibcudf/pylibcudf/CMakeLists.txt | 1 + python/pylibcudf/pylibcudf/__init__.pxd | 2 + python/pylibcudf/pylibcudf/__init__.py | 2 + .../pylibcudf/pylibcudf/libcudf/null_mask.pxd | 2 - python/pylibcudf/pylibcudf/null_mask.pxd | 18 +++ python/pylibcudf/pylibcudf/null_mask.pyx | 142 ++++++++++++++++++ .../pylibcudf/tests/test_null_mask.py | 59 ++++++++ 11 files changed, 252 insertions(+), 86 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/null_mask.rst create mode 100644 python/pylibcudf/pylibcudf/null_mask.pxd create mode 100644 python/pylibcudf/pylibcudf/null_mask.pyx create mode 100644 python/pylibcudf/pylibcudf/tests/test_null_mask.py diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 43e2d6031bc..c58bc42327c 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -342,6 +342,7 @@ def clean_all_xml_files(path): "cudf.Series": ("cudf.core.series.Series", "cudf.Series"), "cudf.Index": ("cudf.core.index.Index", "cudf.Index"), "cupy.core.core.ndarray": ("cupy.ndarray", "cupy.ndarray"), + "DeviceBuffer": ("rmm._lib.device_buffer.DeviceBuffer", "rmm.DeviceBuffer"), } @@ -383,6 +384,7 @@ def _generate_namespaces(namespaces): # Cython types that don't alias cleanly because of # https://github.com/cython/cython/issues/5609 "size_type", + "size_t", "type_id", # Unknown base types "int32_t", diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 505765bba0f..6a2b66e8ea0 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -23,6 +23,7 @@ This page provides API documentation for pylibcudf. join lists merge + null_mask quantiles reduce replace diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/null_mask.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/null_mask.rst new file mode 100644 index 00000000000..4799c62eace --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/null_mask.rst @@ -0,0 +1,6 @@ +========= +null_mask +========= + +.. automodule:: pylibcudf.null_mask + :members: diff --git a/python/cudf/cudf/_lib/null_mask.pyx b/python/cudf/cudf/_lib/null_mask.pyx index 3a7b6a59bf3..d54e8e66281 100644 --- a/python/cudf/cudf/_lib/null_mask.pyx +++ b/python/cudf/cudf/_lib/null_mask.pyx @@ -1,39 +1,11 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. -from enum import Enum - -from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer +import pylibcudf +from pylibcudf.null_mask import MaskState from cudf.core.buffer import acquire_spill_lock, as_buffer -from libcpp.memory cimport make_unique, unique_ptr -from libcpp.pair cimport pair -from libcpp.utility cimport move - -from pylibcudf.libcudf.column.column_view cimport column_view -from pylibcudf.libcudf.null_mask cimport ( - bitmask_allocation_size_bytes as cpp_bitmask_allocation_size_bytes, - bitmask_and as cpp_bitmask_and, - bitmask_or as cpp_bitmask_or, - copy_bitmask as cpp_copy_bitmask, - create_null_mask as cpp_create_null_mask, - underlying_type_t_mask_state, -) -from pylibcudf.libcudf.table.table_view cimport table_view -from pylibcudf.libcudf.types cimport mask_state, size_type - from cudf._lib.column cimport Column -from cudf._lib.utils cimport table_view_from_columns - - -class MaskState(Enum): - """ - Enum for null mask creation state - """ - UNALLOCATED = mask_state.UNALLOCATED - UNINITIALIZED = mask_state.UNINITIALIZED - ALL_VALID = mask_state.ALL_VALID - ALL_NULL = mask_state.ALL_NULL @acquire_spill_lock() @@ -45,33 +17,20 @@ def copy_bitmask(Column col): if col.base_mask is None: return None - cdef column_view col_view = col.view() - cdef device_buffer db - cdef unique_ptr[device_buffer] up_db - - with nogil: - db = move(cpp_copy_bitmask(col_view)) - up_db = move(make_unique[device_buffer](move(db))) - - rmm_db = DeviceBuffer.c_from_unique_ptr(move(up_db)) + rmm_db = pylibcudf.null_mask.copy_bitmask(col.to_pylibcudf(mode="read")) buf = as_buffer(rmm_db) return buf -def bitmask_allocation_size_bytes(size_type num_bits): +def bitmask_allocation_size_bytes(num_bits): """ Given a size, calculates the number of bytes that should be allocated for a column validity mask """ - cdef size_t output_size - - with nogil: - output_size = cpp_bitmask_allocation_size_bytes(num_bits) + return pylibcudf.null_mask.bitmask_allocation_size_bytes(num_bits) - return output_size - -def create_null_mask(size_type size, state=MaskState.UNINITIALIZED): +def create_null_mask(size, state=MaskState.UNINITIALIZED): """ Given a size and a mask state, allocate a mask that can properly represent the given size with the given mask state @@ -83,48 +42,24 @@ def create_null_mask(size_type size, state=MaskState.UNINITIALIZED): state : ``MaskState``, default ``MaskState.UNINITIALIZED`` State the null mask should be created in """ - if not isinstance(state, MaskState): - raise TypeError( - "`state` is required to be of type `MaskState`, got " - + (type(state).__name__) - ) - - cdef device_buffer db - cdef unique_ptr[device_buffer] up_db - cdef mask_state c_mask_state = ( - (state.value) - ) - - with nogil: - db = move(cpp_create_null_mask(size, c_mask_state)) - up_db = move(make_unique[device_buffer](move(db))) - - rmm_db = DeviceBuffer.c_from_unique_ptr(move(up_db)) + rmm_db = pylibcudf.null_mask.create_null_mask(size, state) buf = as_buffer(rmm_db) return buf @acquire_spill_lock() -def bitmask_and(columns: list): - cdef table_view c_view = table_view_from_columns(columns) - cdef pair[device_buffer, size_type] c_result - cdef unique_ptr[device_buffer] up_db - with nogil: - c_result = move(cpp_bitmask_and(c_view)) - up_db = move(make_unique[device_buffer](move(c_result.first))) - dbuf = DeviceBuffer.c_from_unique_ptr(move(up_db)) - buf = as_buffer(dbuf) - return buf, c_result.second +def bitmask_and(list columns): + rmm_db, other = pylibcudf.null_mask.bitmask_and( + [col.to_pylibcudf(mode="read") for col in columns] + ) + buf = as_buffer(rmm_db) + return buf, other @acquire_spill_lock() -def bitmask_or(columns: list): - cdef table_view c_view = table_view_from_columns(columns) - cdef pair[device_buffer, size_type] c_result - cdef unique_ptr[device_buffer] up_db - with nogil: - c_result = move(cpp_bitmask_or(c_view)) - up_db = move(make_unique[device_buffer](move(c_result.first))) - dbuf = DeviceBuffer.c_from_unique_ptr(move(up_db)) - buf = as_buffer(dbuf) - return buf, c_result.second +def bitmask_or(list columns): + rmm_db, other = pylibcudf.null_mask.bitmask_or( + [col.to_pylibcudf(mode="read") for col in columns] + ) + buf = as_buffer(rmm_db) + return buf, other diff --git a/python/pylibcudf/pylibcudf/CMakeLists.txt b/python/pylibcudf/pylibcudf/CMakeLists.txt index f81a32e07f9..a4f17344cb0 100644 --- a/python/pylibcudf/pylibcudf/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/CMakeLists.txt @@ -29,6 +29,7 @@ set(cython_sources join.pyx lists.pyx merge.pyx + null_mask.pyx quantiles.pyx reduce.pyx replace.pyx diff --git a/python/pylibcudf/pylibcudf/__init__.pxd b/python/pylibcudf/pylibcudf/__init__.pxd index 71f523fc3cd..841efa59bda 100644 --- a/python/pylibcudf/pylibcudf/__init__.pxd +++ b/python/pylibcudf/pylibcudf/__init__.pxd @@ -15,6 +15,7 @@ from . cimport ( join, lists, merge, + null_mask, quantiles, reduce, replace, @@ -57,6 +58,7 @@ __all__ = [ "join", "lists", "merge", + "null_mask", "quantiles", "reduce", "replace", diff --git a/python/pylibcudf/pylibcudf/__init__.py b/python/pylibcudf/pylibcudf/__init__.py index e784c6c6dd5..d3878a89a6a 100644 --- a/python/pylibcudf/pylibcudf/__init__.py +++ b/python/pylibcudf/pylibcudf/__init__.py @@ -26,6 +26,7 @@ join, lists, merge, + null_mask, quantiles, reduce, replace, @@ -69,6 +70,7 @@ "join", "lists", "merge", + "null_mask", "quantiles", "reduce", "replace", diff --git a/python/pylibcudf/pylibcudf/libcudf/null_mask.pxd b/python/pylibcudf/pylibcudf/libcudf/null_mask.pxd index 3fc2c7e8f1e..5f582091b06 100644 --- a/python/pylibcudf/pylibcudf/libcudf/null_mask.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/null_mask.pxd @@ -8,8 +8,6 @@ from pylibcudf.libcudf.types cimport bitmask_type, mask_state, size_type from rmm._lib.device_buffer cimport device_buffer -ctypedef int32_t underlying_type_t_mask_state - cdef extern from "cudf/null_mask.hpp" namespace "cudf" nogil: cdef device_buffer copy_bitmask "cudf::copy_bitmask" ( diff --git a/python/pylibcudf/pylibcudf/null_mask.pxd b/python/pylibcudf/pylibcudf/null_mask.pxd new file mode 100644 index 00000000000..ab5c0080312 --- /dev/null +++ b/python/pylibcudf/pylibcudf/null_mask.pxd @@ -0,0 +1,18 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from pylibcudf.libcudf.types cimport mask_state, size_type + +from rmm._lib.device_buffer cimport DeviceBuffer + +from .column cimport Column + + +cpdef DeviceBuffer copy_bitmask(Column col) + +cpdef size_t bitmask_allocation_size_bytes(size_type number_of_bits) + +cpdef DeviceBuffer create_null_mask(size_type size, mask_state state = *) + +cpdef tuple bitmask_and(list columns) + +cpdef tuple bitmask_or(list columns) diff --git a/python/pylibcudf/pylibcudf/null_mask.pyx b/python/pylibcudf/pylibcudf/null_mask.pyx new file mode 100644 index 00000000000..5bdde06f21f --- /dev/null +++ b/python/pylibcudf/pylibcudf/null_mask.pyx @@ -0,0 +1,142 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport make_unique +from libcpp.pair cimport pair +from libcpp.utility cimport move +from pylibcudf.libcudf cimport null_mask as cpp_null_mask +from pylibcudf.libcudf.types cimport mask_state, size_type + +from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer + +from pylibcudf.libcudf.types import mask_state as MaskState # no-cython-lint + +from .column cimport Column +from .table cimport Table + + +cdef DeviceBuffer buffer_to_python(device_buffer buf): + return DeviceBuffer.c_from_unique_ptr(make_unique[device_buffer](move(buf))) + + +cpdef DeviceBuffer copy_bitmask(Column col): + """Copies ``col``'s bitmask into a ``DeviceBuffer``. + + For details, see :cpp:func:`copy_bitmask`. + + Parameters + ---------- + col : Column + Column whose bitmask needs to be copied + + Returns + ------- + rmm.DeviceBuffer + A ``DeviceBuffer`` containing ``col``'s bitmask, or an empty ``DeviceBuffer`` + if ``col`` is not nullable + """ + cdef device_buffer db + + with nogil: + db = move(cpp_null_mask.copy_bitmask(col.view())) + + return buffer_to_python(move(db)) + +cpdef size_t bitmask_allocation_size_bytes(size_type number_of_bits): + """ + Computes the required bytes necessary to represent the specified number of bits + with a 64B padding boundary. + + For details, see :cpp:func:`bitmask_allocation_size_bytes`. + + Parameters + ---------- + number_of_bits : size_type + The number of bits that need to be represented + + Returns + ------- + size_t + The necessary number of bytes + """ + with nogil: + return cpp_null_mask.bitmask_allocation_size_bytes(number_of_bits) + + +cpdef DeviceBuffer create_null_mask( + size_type size, + mask_state state = mask_state.UNINITIALIZED +): + """Creates a ``DeviceBuffer`` for use as a null value indicator bitmask of a + ``Column``. + + For details, see :cpp:func:`create_null_mask`. + + Parameters + ---------- + size : size_type + The number of elements to be represented by the mask + state : mask_state, optional + The desired state of the mask. Can be one of { MaskState.UNALLOCATED, + MaskState.UNINITIALIZED, MaskState.ALL_VALID, MaskState.ALL_NULL } + (default MaskState.UNINITIALIZED) + + Returns + ------- + rmm.DeviceBuffer + A ``DeviceBuffer`` for use as a null bitmask satisfying the desired size and + state + """ + cdef device_buffer db + + with nogil: + db = move(cpp_null_mask.create_null_mask(size, state)) + + return buffer_to_python(move(db)) + + +cpdef tuple bitmask_and(list columns): + """Performs bitwise AND of the bitmasks of a list of columns. + + For details, see :cpp:func:`bitmask_and`. + + Parameters + ---------- + columns : list + The list of columns + + Returns + ------- + tuple[DeviceBuffer, size_type] + A tuple of the resulting mask and count of unset bits + """ + cdef Table c_table = Table(columns) + cdef pair[device_buffer, size_type] c_result + + with nogil: + c_result = move(cpp_null_mask.bitmask_and(c_table.view())) + + return buffer_to_python(move(c_result.first)), c_result.second + + +cpdef tuple bitmask_or(list columns): + """Performs bitwise OR of the bitmasks of a list of columns. + + For details, see :cpp:func:`bitmask_or`. + + Parameters + ---------- + columns : list + The list of columns + + Returns + ------- + tuple[DeviceBuffer, size_type] + A tuple of the resulting mask and count of unset bits + """ + cdef Table c_table = Table(columns) + cdef pair[device_buffer, size_type] c_result + + with nogil: + c_result = move(cpp_null_mask.bitmask_or(c_table.view())) + + return buffer_to_python(move(c_result.first)), c_result.second diff --git a/python/pylibcudf/pylibcudf/tests/test_null_mask.py b/python/pylibcudf/pylibcudf/tests/test_null_mask.py new file mode 100644 index 00000000000..3edcae59edc --- /dev/null +++ b/python/pylibcudf/pylibcudf/tests/test_null_mask.py @@ -0,0 +1,59 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import pyarrow as pa +import pylibcudf as plc +import pytest +from pylibcudf.null_mask import MaskState + +import rmm + + +@pytest.fixture(params=[False, True]) +def nullable(request): + return request.param + + +@pytest.fixture(params=["float32", "float64"]) +def column(request, nullable): + values = [2.5, 2.49, 1.6, 8, -1.5, -1.7, -0.5, 0.5] + typ = {"float32": pa.float32(), "float64": pa.float64()}[request.param] + if nullable: + values[2] = None + return plc.interop.from_arrow(pa.array(values, type=typ)) + + +def test_copy_bitmask(column, nullable): + expected = column.null_mask().obj if nullable else rmm.DeviceBuffer() + got = plc.null_mask.copy_bitmask(column) + + assert expected.size == got.size + assert expected.tobytes() == got.tobytes() + + +def test_bitmask_allocation_size_bytes(): + assert plc.null_mask.bitmask_allocation_size_bytes(0) == 0 + assert plc.null_mask.bitmask_allocation_size_bytes(1) == 64 + assert plc.null_mask.bitmask_allocation_size_bytes(512) == 64 + assert plc.null_mask.bitmask_allocation_size_bytes(513) == 128 + assert plc.null_mask.bitmask_allocation_size_bytes(1024) == 128 + assert plc.null_mask.bitmask_allocation_size_bytes(1025) == 192 + + +@pytest.mark.parametrize("size", [0, 1, 512, 1024]) +@pytest.mark.parametrize( + "state", + [ + MaskState.UNALLOCATED, + MaskState.UNINITIALIZED, + MaskState.ALL_VALID, + MaskState.ALL_NULL, + ], +) +def test_create_null_mask(size, state): + mask = plc.null_mask.create_null_mask(size, state) + + assert mask.size == ( + 0 + if state == MaskState.UNALLOCATED + else plc.null_mask.bitmask_allocation_size_bytes(size) + ) From 5e420ff63ba2997a37bf5dfbfaa73c5f05225f9d Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 30 Aug 2024 17:44:03 -0400 Subject: [PATCH 157/270] Use merge base when calculating changed files (#16709) `get-pr-info.outputs.base.sha` does not actually give the merge base, but merely the tip of the target branch. Calculate the merge base and pass it to the `changed-files` step. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Ray Douglass (https://github.com/raydouglass) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16709 --- .github/workflows/pr.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 35c7e3d95b6..0d79568f589 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -56,14 +56,21 @@ jobs: - name: Checkout code repo uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} - fetch-depth: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).commits }} + fetch-depth: 0 persist-credentials: false + - name: Calculate merge base + id: calculate-merge-base + env: + PR_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} + BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + run: | + (echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") > "$GITHUB_OUTPUT" - name: Get changed files id: changed-files uses: tj-actions/changed-files@v45 with: - base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + base_sha: ${{ steps.calculate-merge-base.outputs.merge-base }} + sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} files_yaml: | cpp: - '**' From 4ad4b2347160212b10f394719f575c6e477f129e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 31 Aug 2024 11:39:01 -0500 Subject: [PATCH 158/270] remove some unnecessary libcudf nightly builds (#16714) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #16650 and #15483. `libcudf` wheels are identical (same content, same filename) across Python versions, but due to an oversight in the PRs linked above, we're currently building nightlies of them once per Python version supported by RAPIDS 😭 You can see this on recent runs of the `build` workflow: image ([build link](https://github.com/rapidsai/cudf/actions/runs/10627299703/job/29460218854)) This PR fixes that by applying the same matrix filter to `libcudf` nightly build jobs as is currently applied to PR jobs: https://github.com/rapidsai/cudf/blob/5e420ff63ba2997a37bf5dfbfaa73c5f05225f9d/.github/workflows/pr.yaml#L195-L200 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16714 --- .github/workflows/build.yaml | 2 ++ .github/workflows/pr.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 72daff7b66b..b5d17022a3a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -71,6 +71,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0d79568f589..8730804e8b6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -197,6 +197,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: + # build for every combination of arch and CUDA version, but only for the latest Python matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: "ci/build_wheel_libcudf.sh" From 76059580abb7a60128545d6ed977c942ea39b3be Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Sun, 1 Sep 2024 11:56:27 -0500 Subject: [PATCH 159/270] Remove java ColumnView.copyWithBooleanColumnAsValidity (#16660) This depends on https://github.com/NVIDIA/spark-rapids/pull/11399 Essentially ifElse is faster than this API and this API is not safe to use generically. https://github.com/NVIDIA/spark-rapids/issues/11397#issuecomment-2310570124 So I am removing it after replacing all calls to it with calls to `ifElse/cudf::copy_if_else` Authors: - Robert (Bobby) Evans (https://github.com/revans2) Approvers: - Alessandro Bellina (https://github.com/abellina) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/16660 --- .../main/java/ai/rapids/cudf/ColumnView.java | 38 ----------------- java/src/main/native/src/ColumnViewJni.cpp | 15 ------- java/src/main/native/src/ColumnViewJni.cu | 31 -------------- java/src/main/native/src/ColumnViewJni.hpp | 16 ------- .../java/ai/rapids/cudf/ColumnVectorTest.java | 42 +------------------ 5 files changed, 1 insertion(+), 141 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 8ff2f0f0a73..6bd4e06c47e 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -913,25 +913,6 @@ public final ColumnVector mergeAndSetValidity(BinaryOp mergeOp, ColumnView... co return new ColumnVector(bitwiseMergeAndSetValidity(getNativeView(), columnViews, mergeOp.nativeId)); } - /** - * Creates a deep copy of a column while replacing the validity mask. The validity mask is the - * device_vector equivalent of the boolean column given as argument. - * - * The boolColumn must have the same number of rows as the current column. - * The result column will have the same number of rows as the current column. - * For all indices `i` where the boolColumn is `true`, the result column will have a valid value at index i. - * For all other values (i.e. `false` or `null`), the result column will have nulls. - * - * If the current column has a null at a given index `i`, and the new validity mask is `true` at index `i`, - * then the row value is undefined. - * - * @param boolColumn bool column whose value is to be used as the validity mask. - * @return Deep copy of the column with replaced validity mask. - */ - public final ColumnVector copyWithBooleanColumnAsValidity(ColumnView boolColumn) { - return new ColumnVector(copyWithBooleanColumnAsValidity(getNativeView(), boolColumn.getNativeView())); - } - ///////////////////////////////////////////////////////////////////////////// // DATE/TIME ///////////////////////////////////////////////////////////////////////////// @@ -4767,25 +4748,6 @@ private static native long clamper(long nativeView, long loScalarHandle, long lo private static native long bitwiseMergeAndSetValidity(long baseHandle, long[] viewHandles, int nullConfig) throws CudfException; - /** - * Native method to deep copy a column while replacing the null mask. The null mask is the - * device_vector equivalent of the boolean column given as argument. - * - * The boolColumn must have the same number of rows as the exemplar column. - * The result column will have the same number of rows as the exemplar. - * For all indices `i` where the boolean column is `true`, the result column will have a valid value at index i. - * For all other values (i.e. `false` or `null`), the result column will have nulls. - * - * If the exemplar column has a null at a given index `i`, and the new validity mask is `true` at index `i`, - * then the resultant row value is undefined. - * - * @param exemplarViewHandle column view of the column that is deep copied. - * @param boolColumnViewHandle bool column whose value is to be used as the null mask. - * @return Deep copy of the column with replaced null mask. - */ - private static native long copyWithBooleanColumnAsValidity(long exemplarViewHandle, - long boolColumnViewHandle) throws CudfException; - //////// // Native cudf::column_view life cycle and metadata access methods. Life cycle methods // should typically only be called from the OffHeap inner class. diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index 4551325ebb1..72f0ad19912 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -2090,21 +2090,6 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_bitwiseMergeAndSetValidit CATCH_STD(env, 0); } -JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_copyWithBooleanColumnAsValidity( - JNIEnv* env, jobject j_object, jlong exemplar_handle, jlong validity_column_handle) -{ - JNI_NULL_CHECK(env, exemplar_handle, "ColumnView handle is null", 0); - JNI_NULL_CHECK(env, validity_column_handle, "Validity column handle is null", 0); - try { - cudf::jni::auto_set_device(env); - auto const exemplar = *reinterpret_cast(exemplar_handle); - auto const validity = *reinterpret_cast(validity_column_handle); - return release_as_jlong( - cudf::jni::new_column_with_boolean_column_as_validity(exemplar, validity)); - } - CATCH_STD(env, 0); -} - //////// // Native cudf::column_view life cycle and metadata access methods. Life cycle methods // should typically only be called from the CudfColumn inner class. diff --git a/java/src/main/native/src/ColumnViewJni.cu b/java/src/main/native/src/ColumnViewJni.cu index 2dbff923544..46261b087ae 100644 --- a/java/src/main/native/src/ColumnViewJni.cu +++ b/java/src/main/native/src/ColumnViewJni.cu @@ -43,37 +43,6 @@ namespace cudf::jni { -std::unique_ptr new_column_with_boolean_column_as_validity( - cudf::column_view const& exemplar, cudf::column_view const& validity_column) -{ - CUDF_EXPECTS(validity_column.type().id() == type_id::BOOL8, - "Validity column must be of type bool"); - CUDF_EXPECTS(validity_column.size() == exemplar.size(), - "Exemplar and validity columns must have the same size"); - - auto validity_device_view = cudf::column_device_view::create(validity_column); - auto validity_begin = cudf::detail::make_optional_iterator( - *validity_device_view, cudf::nullate::DYNAMIC{validity_column.has_nulls()}); - auto validity_end = validity_begin + validity_device_view->size(); - auto [null_mask, null_count] = cudf::detail::valid_if( - validity_begin, - validity_end, - [] __device__(auto optional_bool) { return optional_bool.value_or(false); }, - cudf::get_default_stream(), - rmm::mr::get_current_device_resource()); - auto const exemplar_without_null_mask = - cudf::column_view{exemplar.type(), - exemplar.size(), - exemplar.head(), - nullptr, - 0, - exemplar.offset(), - std::vector{exemplar.child_begin(), exemplar.child_end()}}; - auto deep_copy = std::make_unique(exemplar_without_null_mask); - deep_copy->set_null_mask(std::move(null_mask), null_count); - return deep_copy; -} - std::unique_ptr generate_list_offsets(cudf::column_view const& list_length, rmm::cuda_stream_view stream) { diff --git a/java/src/main/native/src/ColumnViewJni.hpp b/java/src/main/native/src/ColumnViewJni.hpp index c9eef0139ea..c8c441e8fae 100644 --- a/java/src/main/native/src/ColumnViewJni.hpp +++ b/java/src/main/native/src/ColumnViewJni.hpp @@ -22,22 +22,6 @@ namespace cudf::jni { -/** - * @brief Creates a deep copy of the exemplar column, with its validity set to the equivalent - * of the boolean `validity` column's value. - * - * The bool_column must have the same number of rows as the exemplar column. - * The result column will have the same number of rows as the exemplar. - * For all indices `i` where the boolean column is `true`, the result column will have a valid value - * at index i. For all other values (i.e. `false` or `null`), the result column will have nulls. - * - * @param exemplar The column to be deep copied. - * @param bool_column bool column whose value is to be used as the validity. - * @return Deep copy of the exemplar, with the replaced validity. - */ -std::unique_ptr new_column_with_boolean_column_as_validity( - cudf::column_view const& exemplar, cudf::column_view const& bool_column); - /** * @brief Generates list offsets with lengths of each list. * diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index 7136b162c13..708744569df 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -6395,46 +6395,6 @@ void testReplaceSameIndexColumnInStruct() { assertTrue(e.getMessage().contains("Duplicate mapping found for replacing child index")); } - @Test - void testCopyWithBooleanColumnAsValidity() { - final Boolean T = true; - final Boolean F = false; - final Integer X = null; - - // Straight-line: Invalidate every other row. - try (ColumnVector exemplar = ColumnVector.fromBoxedInts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); - ColumnVector validity = ColumnVector.fromBoxedBooleans(F, T, F, T, F, T, F, T, F, T); - ColumnVector expected = ColumnVector.fromBoxedInts(X, 2, X, 4, X, 6, X, 8, X, 10); - ColumnVector result = exemplar.copyWithBooleanColumnAsValidity(validity)) { - assertColumnsAreEqual(expected, result); - } - - // Straight-line: Invalidate all Rows. - try (ColumnVector exemplar = ColumnVector.fromBoxedInts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); - ColumnVector validity = ColumnVector.fromBoxedBooleans(F, F, F, F, F, F, F, F, F, F); - ColumnVector expected = ColumnVector.fromBoxedInts(X, X, X, X, X, X, X, X, X, X); - ColumnVector result = exemplar.copyWithBooleanColumnAsValidity(validity)) { - assertColumnsAreEqual(expected, result); - } - - // Nulls in the validity column are treated as invalid. - try (ColumnVector exemplar = ColumnVector.fromBoxedInts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); - ColumnVector validity = ColumnVector.fromBoxedBooleans(F, T, F, T, F, T, F, null, F, null); - ColumnVector expected = ColumnVector.fromBoxedInts(X, 2, X, 4, X, 6, X, X, X, X); - ColumnVector result = exemplar.copyWithBooleanColumnAsValidity(validity)) { - assertColumnsAreEqual(expected, result); - } - - // Negative case: Mismatch in row count. - Exception x = assertThrows(CudfException.class, () -> { - try (ColumnVector exemplar = ColumnVector.fromBoxedInts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); - ColumnVector validity = ColumnVector.fromBoxedBooleans(F, T, F, T); - ColumnVector result = exemplar.copyWithBooleanColumnAsValidity(validity)) { - } - }); - assertTrue(x.getMessage().contains("Exemplar and validity columns must have the same size")); - } - @Test void testSegmentedGather() { HostColumnVector.DataType dt = new ListType(true, new BasicType(true, DType.STRING)); From 557aabf8d0be528881aadb9795e6d92790a085a8 Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Tue, 3 Sep 2024 11:43:05 -0500 Subject: [PATCH 160/270] Ensure we pass the has_nulls tparam to mixed_join kernels (#16708) Fixes https://github.com/rapidsai/cudf/issues/16706 I'll build/test our stack with this change, but it looks like a typo. If there's a quick unit test we can add I'd be happy to hear recommendations or for someone else to follow on with such a test. Authors: - Alessandro Bellina (https://github.com/abellina) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Nghia Truong (https://github.com/ttnghia) - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cudf/pull/16708 --- cpp/src/join/mixed_join_kernel.cuh | 2 +- cpp/src/join/mixed_join_size_kernel.cuh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/join/mixed_join_kernel.cuh b/cpp/src/join/mixed_join_kernel.cuh index 9d011d43de6..368b1fba870 100644 --- a/cpp/src/join/mixed_join_kernel.cuh +++ b/cpp/src/join/mixed_join_kernel.cuh @@ -130,7 +130,7 @@ void launch_mixed_join(table_device_view left_table, int64_t shmem_size_per_block, rmm::cuda_stream_view stream) { - mixed_join + mixed_join <<>>( left_table, right_table, diff --git a/cpp/src/join/mixed_join_size_kernel.cuh b/cpp/src/join/mixed_join_size_kernel.cuh index a1066e32331..84e9be45030 100644 --- a/cpp/src/join/mixed_join_size_kernel.cuh +++ b/cpp/src/join/mixed_join_size_kernel.cuh @@ -124,7 +124,7 @@ std::size_t launch_compute_mixed_join_output_size( // Allocate storage for the counter used to get the size of the join output rmm::device_scalar size(0, stream, mr); - compute_mixed_join_output_size + compute_mixed_join_output_size <<>>( left_table, right_table, From 25779d95d413e0ddf9379dee22e36eea7bf5f08e Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 3 Sep 2024 12:24:36 -0500 Subject: [PATCH 161/270] Add boost-devel to Java CI Docker image (#16707) Fixes #16678. Adds the boost-devel package to the Java CI Docker environment now that the Boost headers are not being picked up implicitly after libcudf dropped the Arrow dependency in #16640. libcudfjni still requires Arrow for now, and thus requires Boost headers. Authors: - Jason Lowe (https://github.com/jlowe) Approvers: - Alessandro Bellina (https://github.com/abellina) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16707 --- java/ci/Dockerfile.rocky | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/ci/Dockerfile.rocky b/java/ci/Dockerfile.rocky index 6b87f3ed34e..152af22f7e4 100644 --- a/java/ci/Dockerfile.rocky +++ b/java/ci/Dockerfile.rocky @@ -28,7 +28,7 @@ ARG TARGETPLATFORM=linux/amd64 FROM --platform=$TARGETPLATFORM nvidia/cuda:$CUDA_VERSION-devel-rockylinux$OS_RELEASE ARG TOOLSET_VERSION=11 ### Install basic requirements -RUN dnf --enablerepo=powertools install -y scl-utils gcc-toolset-${TOOLSET_VERSION} git zlib-devel maven tar wget patch ninja-build +RUN dnf --enablerepo=powertools install -y scl-utils gcc-toolset-${TOOLSET_VERSION} git zlib-devel maven tar wget patch ninja-build boost-devel ## pre-create the CMAKE_INSTALL_PREFIX folder, set writable by any user for Jenkins RUN mkdir /usr/local/rapids /rapids && chmod 777 /usr/local/rapids /rapids From 0097b454254ac30739c59dee8f29a91e6643360b Mon Sep 17 00:00:00 2001 From: Hirota Akio <33370421+a-hirota@users.noreply.github.com> Date: Wed, 4 Sep 2024 02:28:16 +0900 Subject: [PATCH 162/270] Fix typo in column_factories.hpp comment from 'depth 1' to 'depth 2' (#16700) This PR fixes a typo in the `cpp/include/cudf/column/column_factories.hpp` file. The comment incorrectly mentioned "data (depth 1)" instead of "data (depth 2)". This correction improves code clarity and documentation accuracy. Authors: - Hirota Akio (https://github.com/a-hirota) Approvers: - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16700 --- cpp/include/cudf/column/column_factories.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/cudf/column/column_factories.hpp b/cpp/include/cudf/column/column_factories.hpp index c1f295b7ea8..b2dcb25acb5 100644 --- a/cpp/include/cudf/column/column_factories.hpp +++ b/cpp/include/cudf/column/column_factories.hpp @@ -469,7 +469,7 @@ std::unique_ptr make_strings_column(size_type num_strings, * offsets (depth 1) {0, 2, 5, 7} * data (depth 1) * offsets (depth 2) - * data (depth 1) {1, 2, 3, 4, 5, 6, 7} + * data (depth 2) {1, 2, 3, 4, 5, 6, 7} * @endcode * * @param[in] num_rows The number of lists the column represents. From e18b537315c07b73d1eb26354208249605e3e8be Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 3 Sep 2024 08:30:15 -1000 Subject: [PATCH 163/270] Use Series._from_column more consistently to avoid validation (#16716) This modifies cases where `_from_column` provided the same logic or where 1 column was produced so `._from_column` was valid to use Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16716 --- python/cudf/cudf/_lib/text.pyx | 2 +- python/cudf/cudf/core/dataframe.py | 15 ++++----------- python/cudf/cudf/core/series.py | 14 ++++++-------- python/cudf/cudf/io/text.py | 2 +- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/python/cudf/cudf/_lib/text.pyx b/python/cudf/cudf/_lib/text.pyx index ece69b424bb..b2c7232f549 100644 --- a/python/cudf/cudf/_lib/text.pyx +++ b/python/cudf/cudf/_lib/text.pyx @@ -86,4 +86,4 @@ def read_text(object filepaths_or_buffers, delim, c_options)) - return {None: Column.from_unique_ptr(move(c_col))} + return Column.from_unique_ptr(move(c_col)) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 0d632f4775f..7a171fe9e05 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -473,15 +473,8 @@ def __getitem__(self, arg): ca = self._frame._data index = self._frame.index if col_is_scalar: - s = Series._from_data( - data=ColumnAccessor( - {key: ca._data[key] for key in column_names}, - multiindex=ca.multiindex, - level_names=ca.level_names, - verify=False, - ), - index=index, - ) + name = column_names[0] + s = Series._from_column(ca._data[name], name=name, index=index) return s._getitem_preprocessed(row_spec) if column_names != list(self._frame._column_names): frame = self._frame._from_data( @@ -7770,8 +7763,8 @@ def interleave_columns(self): "interleave_columns does not support 'category' dtype." ) - return self._constructor_sliced._from_data( - {None: libcudf.reshape.interleave_columns([*self._columns])} + return self._constructor_sliced._from_column( + libcudf.reshape.interleave_columns([*self._columns]) ) @_performance_tracking diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index aadbd80f4b4..48445f018d3 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -611,9 +611,7 @@ def from_masked_array(cls, data, mask, null_count=None): 4 14 dtype: int64 """ - col = as_column(data).set_mask(mask) - ca = ColumnAccessor({None: col}, verify=False) - return cls._from_data(ca) + return cls._from_column(as_column(data).set_mask(mask)) @_performance_tracking def __init__( @@ -1150,7 +1148,7 @@ def reset_index( if name is no_default: name = 0 if self.name is None else self.name data[name] = data.pop(self.name) - return cudf.core.dataframe.DataFrame._from_data(data, index) + return self._constructor_expanddim._from_data(data, index) # For ``name`` behavior, see: # https://github.com/pandas-dev/pandas/issues/44575 # ``name`` has to be ignored when `drop=True` @@ -1661,9 +1659,7 @@ def _concat(cls, objs, axis=0, index: bool = True): if len(objs): col = col._with_type_metadata(objs[0].dtype) - return cls._from_data( - ColumnAccessor({name: col}, verify=False), index=result_index - ) + return cls._from_column(col, name=name, index=result_index) @property # type: ignore @_performance_tracking @@ -1977,7 +1973,9 @@ def between(self, left, right, inclusive="both") -> Series: "Inclusive has to be either string of 'both', " "'left', 'right', or 'neither'." ) - return self._from_data({self.name: lmask & rmask}, self.index) + return self._from_column( + lmask & rmask, name=self.name, index=self.index + ) @_performance_tracking def all(self, axis=0, bool_only=None, skipna=True, **kwargs): diff --git a/python/cudf/cudf/io/text.py b/python/cudf/cudf/io/text.py index 0043efce1e4..5ce738cae0e 100644 --- a/python/cudf/cudf/io/text.py +++ b/python/cudf/cudf/io/text.py @@ -33,7 +33,7 @@ def read_text( filepath_or_buffer, "read_text" ) - return cudf.Series._from_data( + return cudf.Series._from_column( libtext.read_text( filepath_or_buffer, delimiter=delimiter, From a83ac6f27254b2ebf99397d81b776c74f93469bf Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:07:49 -1000 Subject: [PATCH 164/270] Add return type annotations to MultiIndex (#16696) Mostly just return type annotations. No logic changes. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Matthew Murray (https://github.com/Matt711) URL: https://github.com/rapidsai/cudf/pull/16696 --- docs/cudf/source/conf.py | 2 + python/cudf/cudf/core/multiindex.py | 109 ++++++++++++++++------------ 2 files changed, 63 insertions(+), 48 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index c58bc42327c..95813907bf4 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -566,6 +566,8 @@ def on_missing_reference(app, env, node, contnode): ("py:obj", "cudf.Index.to_flat_index"), ("py:obj", "cudf.MultiIndex.to_flat_index"), ("py:meth", "pyarrow.Table.to_pandas"), + ("py:class", "pd.DataFrame"), + ("py:class", "pandas.core.indexes.frozen.FrozenList"), ("py:class", "pa.Array"), ("py:class", "ScalarLike"), ("py:class", "ParentType"), diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index a66e2936e3b..e00890ac5c3 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -247,7 +247,7 @@ def to_series(self, index=None, name=None): ) @_performance_tracking - def astype(self, dtype, copy: bool = True): + def astype(self, dtype, copy: bool = True) -> Self: if not is_object_dtype(dtype): raise TypeError( "Setting a MultiIndex dtype to anything other than object is " @@ -256,7 +256,7 @@ def astype(self, dtype, copy: bool = True): return self @_performance_tracking - def rename(self, names, inplace=False): + def rename(self, names, inplace: bool = False) -> Self | None: """ Alter MultiIndex level names @@ -303,7 +303,9 @@ def rename(self, names, inplace=False): return self.set_names(names, level=None, inplace=inplace) @_performance_tracking - def set_names(self, names, level=None, inplace=False): + def set_names( + self, names, level=None, inplace: bool = False + ) -> Self | None: names_is_list_like = is_list_like(names) level_is_list_like = is_list_like(level) @@ -345,7 +347,7 @@ def _from_data( cls, data: MutableMapping, name: Any = None, - ) -> MultiIndex: + ) -> Self: """ Use when you have a ColumnAccessor-like mapping but no codes and levels. """ @@ -394,7 +396,7 @@ def copy( names=None, deep=False, name=None, - ): + ) -> Self: """Returns copy of MultiIndex object. Returns a copy of `MultiIndex`. The `levels` and `codes` value can be @@ -457,7 +459,7 @@ def copy( ) @_performance_tracking - def __repr__(self): + def __repr__(self) -> str: max_seq_items = pd.get_option("display.max_seq_items") or len(self) if len(self) > max_seq_items: @@ -503,7 +505,7 @@ def __repr__(self): @property # type: ignore @_external_only_api("Use ._codes instead") @_performance_tracking - def codes(self): + def codes(self) -> pd.core.indexes.frozen.FrozenList: """ Returns the codes of the underlying MultiIndex. @@ -531,7 +533,7 @@ def get_slice_bound(self, label, side): @property # type: ignore @_performance_tracking - def nlevels(self): + def nlevels(self) -> int: """Integer number of levels in this MultiIndex.""" return self._num_columns @@ -590,7 +592,7 @@ def _get_level_label(self, level): return self.names[level] @_performance_tracking - def isin(self, values, level=None): + def isin(self, values, level=None) -> cp.ndarray: """Return a boolean array where the index values are in values. Compute boolean array of whether each index value is found in @@ -864,7 +866,7 @@ def _validate_indexer( | slice | tuple[Any, ...] | list[tuple[Any, ...]], - ): + ) -> None: if isinstance(indexer, numbers.Number): return if isinstance(indexer, tuple): @@ -900,12 +902,12 @@ def __eq__(self, other): @property # type: ignore @_performance_tracking - def size(self): + def size(self) -> int: # The size of a MultiIndex is only dependent on the number of rows. return self._num_rows @_performance_tracking - def take(self, indices): + def take(self, indices) -> Self: if isinstance(indices, cudf.Series) and indices.has_nulls: raise ValueError("Column must have no nulls.") obj = super().take(indices) @@ -957,7 +959,12 @@ def __getitem__(self, index): return result @_performance_tracking - def to_frame(self, index=True, name=no_default, allow_duplicates=False): + def to_frame( + self, + index: bool = True, + name=no_default, + allow_duplicates: bool = False, + ) -> cudf.DataFrame: """ Create a DataFrame with the levels of the MultiIndex as columns. @@ -1034,7 +1041,7 @@ def to_frame(self, index=True, name=no_default, allow_duplicates=False): ) @_performance_tracking - def get_level_values(self, level): + def get_level_values(self, level) -> cudf.Index: """ Return the values at the requested level @@ -1067,30 +1074,30 @@ def get_level_values(self, level): ) return level_values - def _is_numeric(self): + def _is_numeric(self) -> bool: return False - def _is_boolean(self): + def _is_boolean(self) -> bool: return False - def _is_integer(self): + def _is_integer(self) -> bool: return False - def _is_floating(self): + def _is_floating(self) -> bool: return False - def _is_object(self): + def _is_object(self) -> bool: return False - def _is_categorical(self): + def _is_categorical(self) -> bool: return False - def _is_interval(self): + def _is_interval(self) -> bool: return False @classmethod @_performance_tracking - def _concat(cls, objs): + def _concat(cls, objs) -> Self: source_data = [o.to_frame(index=False) for o in objs] # TODO: Verify if this is really necessary or if we can rely on @@ -1100,17 +1107,19 @@ def _concat(cls, objs): for obj in source_data[1:]: obj.columns = colnames - source_data = cudf.DataFrame._concat(source_data) + source_df = cudf.DataFrame._concat(source_data) try: # Only set names if all objs have the same names (names,) = {o.names for o in objs} - {None} except ValueError: - names = [None] * source_data._num_columns - return cudf.MultiIndex.from_frame(source_data, names=names) + names = [None] * source_df._num_columns + return cudf.MultiIndex.from_frame(source_df, names=names) @classmethod @_performance_tracking - def from_tuples(cls, tuples, sortorder: int | None = None, names=None): + def from_tuples( + cls, tuples, sortorder: int | None = None, names=None + ) -> Self: """ Convert list of tuples to MultiIndex. @@ -1153,7 +1162,7 @@ def from_tuples(cls, tuples, sortorder: int | None = None, names=None): return cls.from_pandas(pdi) @_performance_tracking - def to_numpy(self): + def to_numpy(self) -> np.ndarray: return self.values_host def to_flat_index(self): @@ -1167,7 +1176,7 @@ def to_flat_index(self): @property # type: ignore @_performance_tracking - def values_host(self): + def values_host(self) -> np.ndarray: """ Return a numpy representation of the MultiIndex. @@ -1195,7 +1204,7 @@ def values_host(self): @property # type: ignore @_performance_tracking - def values(self): + def values(self) -> cp.ndarray: """ Return a CuPy representation of the MultiIndex. @@ -1236,7 +1245,7 @@ def from_frame( df: pd.DataFrame | cudf.DataFrame, sortorder: int | None = None, names=None, - ): + ) -> Self: """ Make a MultiIndex from a DataFrame. @@ -1303,7 +1312,9 @@ def from_frame( @classmethod @_performance_tracking - def from_product(cls, iterables, sortorder: int | None = None, names=None): + def from_product( + cls, iterables, sortorder: int | None = None, names=None + ) -> Self: """ Make a MultiIndex from the cartesian product of multiple iterables. @@ -1355,7 +1366,7 @@ def from_arrays( arrays, sortorder=None, names=None, - ) -> MultiIndex: + ) -> Self: """ Convert arrays to MultiIndex. @@ -1410,7 +1421,7 @@ def from_arrays( ) @_performance_tracking - def _poplevels(self, level): + def _poplevels(self, level) -> None | MultiIndex | cudf.Index: """ Remove and return the specified levels from self. @@ -1461,7 +1472,7 @@ def _poplevels(self, level): return popped @_performance_tracking - def swaplevel(self, i=-2, j=-1): + def swaplevel(self, i=-2, j=-1) -> Self: """ Swap level i with level j. Calling this method does not change the ordering of the values. @@ -1512,7 +1523,7 @@ def swaplevel(self, i=-2, j=-1): return midx @_performance_tracking - def droplevel(self, level=-1): + def droplevel(self, level=-1) -> MultiIndex | cudf.Index: """ Removes the specified levels from the MultiIndex. @@ -1598,7 +1609,9 @@ def to_pandas( @classmethod @_performance_tracking - def from_pandas(cls, multiindex: pd.MultiIndex, nan_as_null=no_default): + def from_pandas( + cls, multiindex: pd.MultiIndex, nan_as_null=no_default + ) -> Self: """ Convert from a Pandas MultiIndex @@ -1633,11 +1646,11 @@ def from_pandas(cls, multiindex: pd.MultiIndex, nan_as_null=no_default): @cached_property # type: ignore @_performance_tracking - def is_unique(self): + def is_unique(self) -> bool: return len(self) == len(self.unique()) @property - def dtype(self): + def dtype(self) -> np.dtype: return np.dtype("O") @_performance_tracking @@ -1706,7 +1719,7 @@ def is_monotonic_decreasing(self) -> bool: ) @_performance_tracking - def fillna(self, value): + def fillna(self, value) -> Self: """ Fill null values with the specified value. @@ -1758,7 +1771,7 @@ def nunique(self, dropna: bool = True) -> int: mi = self.dropna(how="all") if dropna else self return len(mi.unique()) - def _clean_nulls_from_index(self): + def _clean_nulls_from_index(self) -> Self: """ Convert all na values(if any) in MultiIndex object to `` as a preprocessing step to `__repr__` methods. @@ -1769,20 +1782,20 @@ def _clean_nulls_from_index(self): ) @_performance_tracking - def memory_usage(self, deep=False): + def memory_usage(self, deep: bool = False) -> int: usage = sum(col.memory_usage for col in self._columns) usage += sum(level.memory_usage(deep=deep) for level in self._levels) usage += sum(code.memory_usage for code in self._codes) return usage @_performance_tracking - def difference(self, other, sort=None): + def difference(self, other, sort=None) -> Self: if hasattr(other, "to_pandas"): other = other.to_pandas() return cudf.from_pandas(self.to_pandas().difference(other, sort)) @_performance_tracking - def append(self, other): + def append(self, other) -> Self: """ Append a collection of MultiIndex objects together @@ -2000,7 +2013,7 @@ def get_loc(self, key): mask[true_inds] = True return mask - def _get_reconciled_name_object(self, other) -> MultiIndex: + def _get_reconciled_name_object(self, other) -> Self: """ If the result of a set operation will be self, return self, unless the names change, in which @@ -2026,7 +2039,7 @@ def _maybe_match_names(self, other): ] @_performance_tracking - def union(self, other, sort=None): + def union(self, other, sort=None) -> Self: if not isinstance(other, MultiIndex): msg = "other must be a MultiIndex or a list of tuples" try: @@ -2050,7 +2063,7 @@ def union(self, other, sort=None): return self._union(other, sort=sort) @_performance_tracking - def _union(self, other, sort=None): + def _union(self, other, sort=None) -> Self: # TODO: When to_frame is refactored to return a # deep copy in future, we should push most of the common # logic between MultiIndex._union & BaseIndex._union into @@ -2076,7 +2089,7 @@ def _union(self, other, sort=None): return midx @_performance_tracking - def _intersection(self, other, sort=None): + def _intersection(self, other, sort=None) -> Self: if self.names != other.names: deep = True col_names = list(range(0, self.nlevels)) @@ -2167,7 +2180,7 @@ def _columns_for_reset_index( else: yield from self._split_columns_by_levels(levels, in_levels=True) - def repeat(self, repeats, axis=None): + def repeat(self, repeats, axis=None) -> Self: return self._from_data( self._data._from_columns_like_self( super()._repeat([*self._columns], repeats, axis) From fa1486e1d1d09116d2b5f57dfef7d9307ebc76c6 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 3 Sep 2024 16:31:30 -0400 Subject: [PATCH 165/270] Remove ERROR_TEST gtest from libcudf (#16722) Removes the `ERROR_TEST` gtest from libcudf. This test was only verifying some macros on mostly CUDA behavior and not libcudf specific functions. The tests have become troublesome to support in CI especially in conjunction with other tools like `compute-sanitizer`. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) URL: https://github.com/rapidsai/cudf/pull/16722 --- cpp/tests/CMakeLists.txt | 4 - cpp/tests/error/error_handling_test.cu | 136 ------------------------- 2 files changed, 140 deletions(-) delete mode 100644 cpp/tests/error/error_handling_test.cu diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index f86acbcc51b..1bedb344a01 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -110,10 +110,6 @@ ConfigureTest(SCALAR_TEST scalar/scalar_test.cpp scalar/scalar_device_view_test. # * timestamps tests ------------------------------------------------------------------------------ ConfigureTest(TIMESTAMPS_TEST wrappers/timestamps_test.cu) -# ################################################################################################## -# * cudf tests ------------------------------------------------------------------------------------ -ConfigureTest(ERROR_TEST error/error_handling_test.cu) - # ################################################################################################## # * groupby tests --------------------------------------------------------------------------------- ConfigureTest( diff --git a/cpp/tests/error/error_handling_test.cu b/cpp/tests/error/error_handling_test.cu deleted file mode 100644 index 9c7459fa69d..00000000000 --- a/cpp/tests/error/error_handling_test.cu +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2018-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include - -TEST(ExpectsTest, FalseCondition) -{ - EXPECT_THROW(CUDF_EXPECTS(false, "condition is false"), cudf::logic_error); -} - -TEST(ExpectsTest, TrueCondition) { EXPECT_NO_THROW(CUDF_EXPECTS(true, "condition is true")); } - -TEST(CudaTryTest, Error) { EXPECT_THROW(CUDF_CUDA_TRY(cudaErrorLaunchFailure), cudf::cuda_error); } - -TEST(CudaTryTest, Success) { EXPECT_NO_THROW(CUDF_CUDA_TRY(cudaSuccess)); } - -TEST(StreamCheck, success) { EXPECT_NO_THROW(CUDF_CHECK_CUDA(0)); } - -namespace { -// Some silly kernel that will cause an error -CUDF_KERNEL void test_kernel(int* data) { data[threadIdx.x] = threadIdx.x; } -} // namespace - -// In a release build and without explicit synchronization, CUDF_CHECK_CUDA may -// or may not fail on erroneous asynchronous CUDA calls. Invoke -// cudaStreamSynchronize to guarantee failure on error. In a non-release build, -// CUDF_CHECK_CUDA deterministically fails on erroneous asynchronous CUDA -// calls. -TEST(StreamCheck, FailedKernel) -{ - rmm::cuda_stream stream; - int a; - test_kernel<<<0, 0, 0, stream.value()>>>(&a); -#ifdef NDEBUG - stream.synchronize(); -#endif - EXPECT_THROW(CUDF_CHECK_CUDA(stream.value()), cudf::cuda_error); -} - -TEST(StreamCheck, CatchFailedKernel) -{ - rmm::cuda_stream stream; - int a; - test_kernel<<<0, 0, 0, stream.value()>>>(&a); -#ifndef NDEBUG - stream.synchronize(); -#endif - EXPECT_THROW(CUDF_CHECK_CUDA(stream.value()), cudf::cuda_error); -} - -CUDF_KERNEL void kernel() { asm("trap;"); } - -TEST(DeathTest, CudaFatalError) -{ - testing::FLAGS_gtest_death_test_style = "threadsafe"; - auto call_kernel = []() { - kernel<<<1, 1, 0, cudf::get_default_stream().value()>>>(); - try { - CUDF_CUDA_TRY(cudaDeviceSynchronize()); - } catch (const cudf::fatal_cuda_error& fe) { - std::abort(); - } - }; - ASSERT_DEATH(call_kernel(), ""); -} - -#ifndef NDEBUG - -CUDF_KERNEL void assert_false_kernel() { cudf_assert(false && "this kernel should die"); } - -CUDF_KERNEL void assert_true_kernel() { cudf_assert(true && "this kernel should live"); } - -TEST(DebugAssertDeathTest, cudf_assert_false) -{ - testing::FLAGS_gtest_death_test_style = "threadsafe"; - - auto call_kernel = []() { - auto const stream = cudf::get_default_stream().value(); - assert_false_kernel<<<1, 1, 0, stream>>>(); - - // Kernel should fail with `cudaErrorAssert` - // This error invalidates the current device context, so we need to kill - // the current process. Running with EXPECT_DEATH spawns a new process for - // each attempted kernel launch - if (cudaErrorAssert == cudaDeviceSynchronize()) { std::abort(); } - - // If we reach this point, the cudf_assert didn't work so we exit normally, which will cause - // EXPECT_DEATH to fail. - }; - - EXPECT_DEATH(call_kernel(), "this kernel should die"); -} - -TEST(DebugAssert, cudf_assert_true) -{ - auto const stream = cudf::get_default_stream().value(); - assert_true_kernel<<<1, 1, 0, stream>>>(); - ASSERT_EQ(cudaSuccess, cudaDeviceSynchronize()); -} - -#endif - -// These tests don't use CUDF_TEST_PROGRAM_MAIN because : -// 1.) They don't need the RMM Pool -// 2.) The RMM Pool interferes with the death test -int main(int argc, char** argv) -{ - if (getenv("LIBCUDF_MEMCHECK_ENABLED")) { return 0; } - - ::testing::InitGoogleTest(&argc, argv); - auto const cmd_opts = parse_cudf_test_opts(argc, argv); - auto adaptor = make_stream_mode_adaptor(cmd_opts); - return RUN_ALL_TESTS(); -} From 26091a44b3dbf0f56fc0dfc5f081077f2d00681f Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:10:24 -0400 Subject: [PATCH 166/270] Refactor cudf pandas integration tests CI (#16728) Following up #16645 with a couple improvements Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16728 --- ..._library_tests.sh => run-library-tests.sh} | 24 +++++++------------ .../third-party-integration/test.sh | 2 +- 2 files changed, 10 insertions(+), 16 deletions(-) rename ci/cudf_pandas_scripts/third-party-integration/{ci_run_library_tests.sh => run-library-tests.sh} (69%) diff --git a/ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh b/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh similarity index 69% rename from ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh rename to ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh index 54a56508cdc..d44d25d658c 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh @@ -9,23 +9,17 @@ cleanup() { trap cleanup EXIT -runtest_gold() { +runtest() { local lib=$1 + local mode=$2 - pytest \ - -v \ - --continue-on-collection-errors \ - --cache-clear \ - --numprocesses=${NUM_PROCESSES} \ - --dist=worksteal \ - ${TEST_DIR}/test_${lib}*.py -} - -runtest_cudf_pandas() { - local lib=$1 + local plugin="" + if [ "$mode" = "cudf" ]; then + plugin="-p cudf.pandas" + fi pytest \ - -p cudf.pandas \ + $plugin \ -v \ --continue-on-collection-errors \ --cache-clear \ @@ -38,8 +32,8 @@ main() { local lib=$1 # generation phase - runtest_gold ${lib} - runtest_cudf_pandas ${lib} + runtest ${lib} "gold" + runtest ${lib} "cudf" # assertion phase pytest \ diff --git a/ci/cudf_pandas_scripts/third-party-integration/test.sh b/ci/cudf_pandas_scripts/third-party-integration/test.sh index 89b28c30e39..f8ddbaba0f3 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/test.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/test.sh @@ -72,7 +72,7 @@ main() { fi done - TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/ci_run_library_tests.sh ${lib} + TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh ${lib} rapids-logger "Test script exiting with value: ${EXITCODE}" done From 1b6f02d536d253465d2c601f222fb0acede8a942 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Wed, 4 Sep 2024 12:02:40 -0500 Subject: [PATCH 167/270] Multi-file and Parquet-aware prefetching from remote storage (#16657) Follow up to https://github.com/rapidsai/cudf/pull/16613 Supersedes https://github.com/rapidsai/cudf/pull/16166 Improves remote-IO read performance when multiple files are read at once. Also enables partial IO for remote Parquet files (previously removed in `24.10` by https://github.com/rapidsai/cudf/pull/16589). Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16657 --- python/cudf/cudf/io/parquet.py | 40 +++++++++ python/cudf/cudf/tests/test_s3.py | 47 ++++++++++ python/cudf/cudf/utils/ioutils.py | 141 ++++++++++++++++++++++++++---- 3 files changed, 212 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 526f12aa94e..62be7378e9e 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -577,11 +577,51 @@ def read_parquet( ) filepath_or_buffer = paths if paths else filepath_or_buffer + # Prepare remote-IO options + prefetch_options = kwargs.pop("prefetch_options", {}) + if not ioutils._is_local_filesystem(fs): + # The default prefetch method depends on the + # `row_groups` argument. In most cases we will use + # method="all" by default, because it is fastest + # when we need to read most of the file(s). + # If a (simple) `row_groups` selection is made, we + # use method="parquet" to avoid transferring the + # entire file over the network + method = prefetch_options.get("method") + _row_groups = None + if method in (None, "parquet"): + if row_groups is None: + # If the user didn't specify a method, don't use + # 'parquet' prefetcher for column projection alone. + method = method or "all" + elif all(r == row_groups[0] for r in row_groups): + # Row group selection means we are probably + # reading half the file or less. We should + # avoid a full file transfer by default. + method = "parquet" + _row_groups = row_groups[0] + elif (method := method or "all") == "parquet": + raise ValueError( + "The 'parquet' prefetcher requires a uniform " + "row-group selection for all paths within the " + "same `read_parquet` call. " + "Got: {row_groups}" + ) + if method == "parquet": + prefetch_options = prefetch_options.update( + { + "method": method, + "columns": columns, + "row_groups": _row_groups, + } + ) + filepaths_or_buffers = ioutils.get_reader_filepath_or_buffer( path_or_data=filepath_or_buffer, fs=fs, storage_options=storage_options, bytes_per_thread=bytes_per_thread, + prefetch_options=prefetch_options, ) # Warn user if they are not using cudf for IO diff --git a/python/cudf/cudf/tests/test_s3.py b/python/cudf/cudf/tests/test_s3.py index 3b23a53091e..0958b68084d 100644 --- a/python/cudf/cudf/tests/test_s3.py +++ b/python/cudf/cudf/tests/test_s3.py @@ -229,6 +229,53 @@ def test_read_parquet( assert_eq(expect, got2) +@pytest.mark.parametrize("method", ["all", "parquet"]) +@pytest.mark.parametrize("blocksize", [1024 * 1024, 1024]) +def test_read_parquet_prefetch_options( + s3_base, + s3so, + pdf, + method, + blocksize, +): + bucket = "parquet" + fname_1 = "test_parquet_reader_prefetch_options_1.parquet" + buffer_1 = BytesIO() + pdf.to_parquet(path=buffer_1) + buffer_1.seek(0) + + fname_2 = "test_parquet_reader_prefetch_options_2.parquet" + buffer_2 = BytesIO() + pdf_2 = pdf.copy() + pdf_2["Integer"] += 1 + pdf_2.to_parquet(path=buffer_2) + buffer_2.seek(0) + + with s3_context( + s3_base=s3_base, + bucket=bucket, + files={ + fname_1: buffer_1, + fname_2: buffer_2, + }, + ): + got = cudf.read_parquet( + [ + f"s3://{bucket}/{fname_1}", + f"s3://{bucket}/{fname_2}", + ], + storage_options=s3so, + prefetch_options={ + "method": method, + "blocksize": blocksize, + }, + columns=["String", "Integer"], + ) + + expect = pd.concat([pdf, pdf_2], ignore_index=True)[["String", "Integer"]] + assert_eq(expect, got) + + @pytest.mark.parametrize("bytes_per_thread", [32, 1024]) @pytest.mark.parametrize("columns", [None, ["List", "Struct"]]) @pytest.mark.parametrize("index", [None, "Integer"]) diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 6b146be0fa3..1627107b57d 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -1,6 +1,8 @@ # Copyright (c) 2019-2024, NVIDIA CORPORATION. import datetime +import functools +import operator import os import urllib import warnings @@ -18,6 +20,12 @@ from cudf.core._compat import PANDAS_LT_300 from cudf.utils.docutils import docfmt_partial +try: + import fsspec.parquet as fsspec_parquet + +except ImportError: + fsspec_parquet = None + _BYTES_PER_THREAD_DEFAULT = 256 * 1024 * 1024 _ROW_GROUP_SIZE_BYTES_DEFAULT = 128 * 1024 * 1024 @@ -187,6 +195,11 @@ allow_mismatched_pq_schemas : boolean, default False If True, enables reading (matching) columns specified in `columns` and `filters` options from the input files with otherwise mismatched schemas. +prefetch_options : dict, default None + WARNING: This is an experimental feature and may be removed at any + time without warning or deprecation period. + Dictionary of options to use to prefetch bytes from remote storage. + These options are passed through to `get_reader_filepath_or_buffer`. Returns ------- @@ -1439,6 +1452,14 @@ Glob pattern to use when expanding directories into file paths (e.g. "*.json"). If this parameter is not specified, directories will not be expanded. +prefetch_options : dict, default None + WARNING: This is an experimental feature and may be removed at any + time without warning or deprecation period. + Dictionary of options to use to prefetch bytes from remote storage. + These options are only used when `path_or_data` is a list of remote + paths. If 'method' is set to 'all' (the default), the only supported + option is 'blocksize' (default 256 MB). If method is set to 'parquet', + 'columns' and 'row_groups' are also supported (default None). Returns ------- @@ -1620,6 +1641,7 @@ def get_reader_filepath_or_buffer( warn_on_raw_text_input=None, warn_meta=None, expand_dir_pattern=None, + prefetch_options=None, ): """{docstring}""" @@ -1690,26 +1712,15 @@ def get_reader_filepath_or_buffer( raw_text_input = True elif fs is not None: - # TODO: We can use cat_ranges and/or parquet-aware logic - # to copy all remote data into host memory at once here. - # The current solution iterates over files, and copies - # ALL data from each file (even when we are performing - # partial IO, and don't need the entire file) if len(paths) == 0: raise FileNotFoundError( f"{input_sources} could not be resolved to any files" ) - filepaths_or_buffers = [ - BytesIO( - _fsspec_data_transfer( - fpath, - fs=fs, - mode=mode, - bytes_per_thread=bytes_per_thread, - ) - ) - for fpath in paths - ] + filepaths_or_buffers = _prefetch_remote_buffers( + paths, + fs, + **(prefetch_options or {}), + ) else: raw_text_input = True @@ -2099,3 +2110,101 @@ def _read_byte_ranges( for worker in workers: worker.join() + + +def _get_remote_bytes_all( + remote_paths, fs, *, blocksize=_BYTES_PER_THREAD_DEFAULT +): + # TODO: Experiment with a heuristic to avoid the fs.sizes + # call when we are reading many files at once (the latency + # of collecting the file sizes is unnecessary in this case) + if max(sizes := fs.sizes(remote_paths)) <= blocksize: + # Don't bother breaking up individual files + return fs.cat_ranges(remote_paths, None, None) + else: + # Construct list of paths, starts, and ends + paths, starts, ends = map( + list, + zip( + *( + (r, j, min(j + blocksize, s)) + for r, s in zip(remote_paths, sizes) + for j in range(0, s, blocksize) + ) + ), + ) + + # Collect the byte ranges + chunks = fs.cat_ranges(paths, starts, ends) + + # Construct local byte buffers + # (Need to make sure path offsets are ordered correctly) + unique_count = dict(zip(*np.unique(paths, return_counts=True))) + offset = np.cumsum([0] + [unique_count[p] for p in remote_paths]) + buffers = [ + functools.reduce(operator.add, chunks[offset[i] : offset[i + 1]]) + for i in range(len(remote_paths)) + ] + return buffers + + +def _get_remote_bytes_parquet( + remote_paths, + fs, + *, + columns=None, + row_groups=None, + blocksize=_BYTES_PER_THREAD_DEFAULT, +): + if fsspec_parquet is None or (columns is None and row_groups is None): + return _get_remote_bytes_all(remote_paths, fs, blocksize=blocksize) + + sizes = fs.sizes(remote_paths) + data = fsspec_parquet._get_parquet_byte_ranges( + remote_paths, + fs, + columns=columns, + row_groups=row_groups, + max_block=blocksize, + ) + + buffers = [] + for size, path in zip(sizes, remote_paths): + path_data = data[path] + buf = np.empty(size, dtype="b") + for range_offset in path_data.keys(): + chunk = path_data[range_offset] + buf[range_offset[0] : range_offset[1]] = np.frombuffer( + chunk, dtype="b" + ) + buffers.append(buf.tobytes()) + return buffers + + +def _prefetch_remote_buffers( + paths, + fs, + *, + method="all", + **prefetch_options, +): + # Gather bytes ahead of time for remote filesystems + if fs and paths and not _is_local_filesystem(fs): + try: + prefetcher = { + "parquet": _get_remote_bytes_parquet, + "all": _get_remote_bytes_all, + }[method] + except KeyError: + raise ValueError( + f"{method} is not a supported remote-data prefetcher." + " Expected 'parquet' or 'all'." + ) + return prefetcher( + paths, + fs, + **prefetch_options, + ) + + else: + return paths From ad1369d2d6eabf4b0ae480a10463a74f3034aece Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 5 Sep 2024 01:11:07 +0200 Subject: [PATCH 168/270] CI: Test against old versions of key dependencies (#16570) This adds explicit tests with old versions of key dependencies. Specifically: - `numba==0.57` - `numpy==1.23` - `pandas==2.0` - ~`fsspec==0.6.0`~ excluded it. `transformers==4.39.3` requires `huggingface_hub` which requires `fsspec>=2023.5.0`. In principle one could include it e.g. only for conda which doesn't pull in `transformers`, but that seemed not worth the trouble? - `cupy==12.0.0` - `pyarrow==16.1.0` See also https://github.com/rapidsai/build-planning/issues/81 (Marking as draft until I see that things work.) Authors: - Sebastian Berg (https://github.com/seberg) - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Charles Blackmon-Luca (https://github.com/charlesbluca) URL: https://github.com/rapidsai/cudf/pull/16570 --- ci/cudf_pandas_scripts/run_tests.sh | 13 +- ci/test_python_common.sh | 3 +- ci/test_wheel_cudf.sh | 14 ++ ci/test_wheel_cudf_polars.sh | 11 ++ ci/test_wheel_dask_cudf.sh | 13 ++ dependencies.yaml | 22 +++ .../cudf/cudf/tests/indexes/test_interval.py | 4 + .../test_avro_reader_fastavro_integration.py | 5 + python/cudf/cudf/tests/test_binops.py | 41 +++++- python/cudf/cudf/tests/test_categorical.py | 5 + python/cudf/cudf/tests/test_concat.py | 99 ++++++++----- python/cudf/cudf/tests/test_csv.py | 12 +- python/cudf/cudf/tests/test_dataframe.py | 19 ++- python/cudf/cudf/tests/test_datetime.py | 35 ++++- python/cudf/cudf/tests/test_doctests.py | 5 + python/cudf/cudf/tests/test_groupby.py | 112 +++++++++++++++ python/cudf/cudf/tests/test_index.py | 37 ++++- python/cudf/cudf/tests/test_indexing.py | 8 ++ python/cudf/cudf/tests/test_interpolate.py | 4 + python/cudf/cudf/tests/test_interval.py | 5 + python/cudf/cudf/tests/test_join_order.py | 130 +++++++++++++++++- python/cudf/cudf/tests/test_mvc.py | 8 +- python/cudf/cudf/tests/test_numerical.py | 3 +- python/cudf/cudf/tests/test_orc.py | 8 +- python/cudf/cudf/tests/test_parquet.py | 5 + python/cudf/cudf/tests/test_reductions.py | 5 + python/cudf/cudf/tests/test_replace.py | 20 ++- python/cudf/cudf/tests/test_resampling.py | 9 ++ python/cudf/cudf/tests/test_reshape.py | 17 ++- python/cudf/cudf/tests/test_stats.py | 8 ++ .../cudf_pandas_tests/test_cudf_pandas.py | 12 +- .../dask_cudf/tests/test_applymap.py | 6 + .../dask_cudf/tests/test_distributed.py | 5 + .../dask_cudf/dask_cudf/tests/test_groupby.py | 5 + 34 files changed, 638 insertions(+), 70 deletions(-) diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 8b85695c861..1c2724a9a5d 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -54,8 +54,19 @@ else RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist - # echo to expand wildcard before adding `[extra]` requires for pip + echo "" > ./constraints.txt + if [[ $RAPIDS_DEPENDENCIES == "oldest" ]]; then + # `test_python` constraints are for `[test]` not `[cudf-pandas-tests]` + rapids-dependency-file-generator \ + --output requirements \ + --file-key test_python \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \ + | tee ./constraints.txt + fi + python -m pip install \ + -v \ + --constraint ./constraints.txt \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" \ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index e8849588aa5..d0675b0431a 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -14,7 +14,8 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file-key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \ + | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 6861d699695..28ded2f8e0f 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -10,8 +10,22 @@ RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +rapids-logger "Install cudf, pylibcudf, and test requirements" + +# Constrain to minimum dependency versions if job is set up as "oldest" +echo "" > ./constraints.txt +if [[ $RAPIDS_DEPENDENCIES == "oldest" ]]; then + rapids-dependency-file-generator \ + --output requirements \ + --file-key py_test_cudf \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \ + | tee ./constraints.txt +fi + # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ + -v \ + --constraint ./constraints.txt \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 0baf6c9e277..9844090258a 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -25,9 +25,20 @@ RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-f RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist rapids-logger "Installing cudf_polars and its dependencies" +# Constraint to minimum dependency versions if job is set up as "oldest" +echo "" > ./constraints.txt +if [[ $RAPIDS_DEPENDENCIES == "oldest" ]]; then + rapids-dependency-file-generator \ + --output requirements \ + --file-key py_test_cudf_polars \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \ + | tee ./constraints.txt +fi # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ + -v \ + --constraint ./constraints.txt \ "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index fa74b2398f7..0d39807d56c 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -11,8 +11,21 @@ RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +rapids-logger "Install dask_cudf, cudf, pylibcudf, and test requirements" +# Constraint to minimum dependency versions if job is set up as "oldest" +echo "" > ./constraints.txt +if [[ $RAPIDS_DEPENDENCIES == "oldest" ]]; then + rapids-dependency-file-generator \ + --output requirements \ + --file-key py_test_dask_cudf \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \ + | tee ./constraints.txt +fi + # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ + -v \ + --constraint ./constraints.txt \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ diff --git a/dependencies.yaml b/dependencies.yaml index c6851d9cb90..f8b231efd6d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -696,6 +696,28 @@ dependencies: - pytest<8 - pytest-cov - pytest-xdist + specific: + # Define additional constraints for testing with oldest dependencies. + - output_types: [conda, requirements] + matrices: + - matrix: {dependencies: "oldest"} + packages: + - numba==0.57.* + - numpy==1.23.* + - pandas==2.0.* + - pyarrow==14.0.0 + - cupy==12.0.0 # ignored as pip constraint + - matrix: + packages: + - output_types: requirements + # Using --constraints for pip install, so we list cupy multiple times + matrices: + - matrix: {dependencies: "oldest"} + packages: + - cupy-cuda11x==12.0.0 + - cupy-cuda12x==12.0.0 + - matrix: + packages: test_python_pylibcudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf/cudf/tests/indexes/test_interval.py b/python/cudf/cudf/tests/indexes/test_interval.py index 6653a94c9be..25edf788daf 100644 --- a/python/cudf/cudf/tests/indexes/test_interval.py +++ b/python/cudf/cudf/tests/indexes/test_interval.py @@ -149,6 +149,10 @@ def test_interval_range_periods_basic_dtype(start_t, end_t, periods_t): assert_eq(pindex, gindex) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Does not warn on older versions of pandas", +) def test_interval_range_periods_warnings(): start_val, end_val, periods_val = 0, 4, 1.0 diff --git a/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py b/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py index 2ec1d1d2f28..9d69e626c3d 100644 --- a/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py +++ b/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py @@ -23,6 +23,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION from cudf.testing import assert_eq from cudf.testing.dataset_generator import rand_dataframe @@ -302,6 +303,10 @@ def get_days_from_epoch(date: datetime.date | None) -> int | None: @pytest.mark.parametrize("namespace", [None, "root_ns"]) @pytest.mark.parametrize("nullable", [True, False]) @pytest.mark.parametrize("prepend_null", [True, False]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas (datetime(9999, ...) too large)", +) def test_can_parse_avro_date_logical_type(namespace, nullable, prepend_null): avro_type = {"logicalType": "date", "type": "int"} if nullable: diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 4256ec872e6..2e8519509e2 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -13,7 +13,11 @@ import cudf from cudf import Index, Series -from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.core.buffer.spill_manager import get_global_manager from cudf.testing import _utils as utils, assert_eq from cudf.utils.dtypes import ( @@ -1781,6 +1785,20 @@ def test_datetime_dateoffset_binaryop( reason="https://github.com/pandas-dev/pandas/issues/57448", ) ) + if ( + not PANDAS_GE_220 + and dtype in {"datetime64[ms]", "datetime64[s]"} + and frequency in ("microseconds", "nanoseconds") + and n_periods != 0 + ): + pytest.skip(reason="https://github.com/pandas-dev/pandas/pull/55595") + if ( + not PANDAS_GE_220 + and dtype == "datetime64[us]" + and frequency == "nanoseconds" + and n_periods != 0 + ): + pytest.skip(reason="https://github.com/pandas-dev/pandas/pull/55595") date_col = [ f"2000-01-01 00:00:{components}", @@ -1834,7 +1852,11 @@ def test_datetime_dateoffset_binaryop( "ignore:Discarding nonzero nanoseconds:UserWarning" ) @pytest.mark.parametrize("op", [operator.add, operator.sub]) -def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) +def test_datetime_dateoffset_binaryop_multiple(request, date_col, kwargs, op): gsr = cudf.Series(date_col, dtype="datetime64[ns]") psr = gsr.to_pandas() @@ -1873,6 +1895,21 @@ def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): def test_datetime_dateoffset_binaryop_reflected( n_periods, frequency, dtype, components ): + if ( + not PANDAS_GE_220 + and dtype in {"datetime64[ms]", "datetime64[s]"} + and frequency in ("microseconds", "nanoseconds") + and n_periods != 0 + ): + pytest.skip(reason="https://github.com/pandas-dev/pandas/pull/55595") + if ( + not PANDAS_GE_220 + and dtype == "datetime64[us]" + and frequency == "nanoseconds" + and n_periods != 0 + ): + pytest.skip(reason="https://github.com/pandas-dev/pandas/pull/55595") + date_col = [ f"2000-01-01 00:00:{components}", f"2000-01-31 00:00:{components}", diff --git a/python/cudf/cudf/tests/test_categorical.py b/python/cudf/cudf/tests/test_categorical.py index ae58af8ebce..cd1ad21ae59 100644 --- a/python/cudf/cudf/tests/test_categorical.py +++ b/python/cudf/cudf/tests/test_categorical.py @@ -11,6 +11,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION from cudf.testing import assert_eq from cudf.testing._utils import NUMERIC_TYPES, assert_exceptions_equal @@ -858,6 +859,10 @@ def test_cat_from_scalar(scalar): assert_eq(ps, gs) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Does not warn on older versions of pandas", +) def test_cat_groupby_fillna(): ps = pd.Series(["a", "b", "c"], dtype="category") gs = cudf.from_pandas(ps) diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index c1c03de48d4..8da589ba45b 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -9,6 +9,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_220 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing import assert_eq from cudf.testing._utils import assert_exceptions_equal, expect_warning_if @@ -451,45 +452,75 @@ def test_concat_mixed_input(): [pd.Series([1, 2, 3]), pd.DataFrame({"a": []})], [pd.Series([], dtype="float64"), pd.DataFrame({"a": []})], [pd.Series([], dtype="float64"), pd.DataFrame({"a": [1, 2]})], - [ - pd.Series([1, 2, 3.0, 1.2], name="abc"), - pd.DataFrame({"a": [1, 2]}), - ], - [ - pd.Series( - [1, 2, 3.0, 1.2], name="abc", index=[100, 110, 120, 130] - ), - pd.DataFrame({"a": [1, 2]}), - ], - [ - pd.Series( - [1, 2, 3.0, 1.2], name="abc", index=["a", "b", "c", "d"] + pytest.param( + [ + pd.Series([1, 2, 3.0, 1.2], name="abc"), + pd.DataFrame({"a": [1, 2]}), + ], + marks=pytest.mark.skipif( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - pd.DataFrame({"a": [1, 2]}, index=["a", "b"]), - ], - [ - pd.Series( - [1, 2, 3.0, 1.2, 8, 100], - name="New name", - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2], name="abc", index=[100, 110, 120, 130] + ), + pd.DataFrame({"a": [1, 2]}), + ], + marks=pytest.mark.skipif( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - pd.DataFrame( - {"a": [1, 2, 4, 10, 11, 12]}, - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2], name="abc", index=["a", "b", "c", "d"] + ), + pd.DataFrame({"a": [1, 2]}, index=["a", "b"]), + ], + marks=pytest.mark.skipif( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - ], - [ - pd.Series( - [1, 2, 3.0, 1.2, 8, 100], - name="New name", - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2, 8, 100], + name="New name", + index=["a", "b", "c", "d", "e", "f"], + ), + pd.DataFrame( + {"a": [1, 2, 4, 10, 11, 12]}, + index=["a", "b", "c", "d", "e", "f"], + ), + ], + marks=pytest.mark.skipif( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - pd.DataFrame( - {"a": [1, 2, 4, 10, 11, 12]}, - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2, 8, 100], + name="New name", + index=["a", "b", "c", "d", "e", "f"], + ), + pd.DataFrame( + {"a": [1, 2, 4, 10, 11, 12]}, + index=["a", "b", "c", "d", "e", "f"], + ), + ] + * 7, + marks=pytest.mark.skipif( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - ] - * 7, + ), ], ) def test_concat_series_dataframe_input(objs): diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 40ba415e681..cee3d23eadc 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -16,9 +16,13 @@ import cudf from cudf import read_csv -from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.testing import assert_eq -from cudf.testing._utils import assert_exceptions_equal +from cudf.testing._utils import assert_exceptions_equal, expect_warning_if def make_numeric_dataframe(nrows, dtype): @@ -1270,14 +1274,14 @@ def test_csv_reader_delim_whitespace(): # with header row with pytest.warns(FutureWarning): cu_df = read_csv(StringIO(buffer), delim_whitespace=True) - with pytest.warns(FutureWarning): + with expect_warning_if(PANDAS_GE_220): pd_df = pd.read_csv(StringIO(buffer), delim_whitespace=True) assert_eq(pd_df, cu_df) # without header row with pytest.warns(FutureWarning): cu_df = read_csv(StringIO(buffer), delim_whitespace=True, header=None) - with pytest.warns(FutureWarning): + with expect_warning_if(PANDAS_GE_220): pd_df = pd.read_csv( StringIO(buffer), delim_whitespace=True, header=None ) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 9122a1074ac..f4d1578bda7 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -26,7 +26,11 @@ import cudf from cudf.api.extensions import no_default -from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.column import column from cudf.errors import MixedTypeError @@ -3561,8 +3565,11 @@ def test_dataframe_empty_sort_index(): @pytest.mark.parametrize("inplace", [True, False]) @pytest.mark.parametrize("na_position", ["first", "last"]) def test_dataframe_sort_index( - index, axis, ascending, inplace, ignore_index, na_position + request, index, axis, ascending, inplace, ignore_index, na_position ): + if not PANDAS_GE_220 and axis in (1, "columns") and ignore_index: + pytest.skip(reason="Bug fixed in pandas-2.2") + pdf = pd.DataFrame( {"b": [1, 3, 2], "a": [1, 4, 3], "c": [4, 1, 5]}, index=index, @@ -3612,6 +3619,10 @@ def test_dataframe_sort_index( @pytest.mark.parametrize("ignore_index", [True, False]) @pytest.mark.parametrize("inplace", [True, False]) @pytest.mark.parametrize("na_position", ["first", "last"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_dataframe_mulitindex_sort_index( request, axis, level, ascending, inplace, ignore_index, na_position ): @@ -6747,6 +6758,10 @@ def test_dataframe_init_from_arrays_cols(data, cols, index): None, ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_dataframe_assign_scalar(request, col_data, assign_val): request.applymarker( pytest.mark.xfail( diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 7be4faa42c3..4a2345fc009 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -14,7 +14,11 @@ import cudf import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series -from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.core.index import DatetimeIndex from cudf.testing import assert_eq from cudf.testing._utils import ( @@ -801,6 +805,10 @@ def test_to_datetime_different_formats_notimplemented(): cudf.to_datetime(["2015-02-01", "2015-02-01 10:10:10"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas.", +) def test_datetime_can_cast_safely(): sr = cudf.Series( ["1679-01-01", "2000-01-31", "2261-01-01"], dtype="datetime64[ms]" @@ -847,6 +855,10 @@ def test_datetime_array_timeunit_cast(dtype): @pytest.mark.parametrize("timeunit", ["D", "W", "M", "Y"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_datetime_scalar_timeunit_cast(timeunit): testscalar = np.datetime64("2016-11-20", timeunit) @@ -1535,6 +1547,10 @@ def test_date_range_start_end_periods(start, end, periods): ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_date_range_start_end_freq(start, end, freq): if isinstance(freq, str): _gfreq = _pfreq = freq @@ -1551,6 +1567,10 @@ def test_date_range_start_end_freq(start, end, freq): ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_date_range_start_freq_periods(start, freq, periods): if isinstance(freq, str): _gfreq = _pfreq = freq @@ -1643,6 +1663,9 @@ def test_date_range_raise_overflow(): ], ) def test_date_range_raise_unsupported(freqstr_unsupported): + if not PANDAS_GE_220 and freqstr_unsupported.endswith("E"): + pytest.skip(reason="YE, etc. support was added in pandas 2.2") + s, e = "2001-01-01", "2008-01-31" pd.date_range(start=s, end=e, freq=freqstr_unsupported) with pytest.raises(ValueError, match="does not yet support"): @@ -1654,7 +1677,7 @@ def test_date_range_raise_unsupported(freqstr_unsupported): if freqstr_unsupported != "3MS": freqstr_unsupported = freqstr_unsupported.lower() with pytest.raises(ValueError, match="does not yet support"): - with pytest.warns(FutureWarning): + with expect_warning_if(PANDAS_GE_220): cudf.date_range(start=s, end=e, freq=freqstr_unsupported) @@ -1995,6 +2018,10 @@ def test_first(idx, offset): ) ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="warning not present in older pandas versions", +) def test_first_start_at_end_of_month(idx, offset): p = pd.Series(range(len(idx)), index=idx) g = cudf.from_pandas(p) @@ -2319,6 +2346,10 @@ def test_datetime_to_str(data, dtype): assert_eq(actual.to_pandas(nullable=True), expected) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_datetime_string_to_datetime_resolution_loss_raises(): data = ["2020-01-01 00:00:00.00001"] dtype = "datetime64[s]" diff --git a/python/cudf/cudf/tests/test_doctests.py b/python/cudf/cudf/tests/test_doctests.py index 794660cffcb..5d3d18cbe95 100644 --- a/python/cudf/cudf/tests/test_doctests.py +++ b/python/cudf/cudf/tests/test_doctests.py @@ -11,6 +11,7 @@ from packaging import version import cudf +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION pytestmark = pytest.mark.filterwarnings("ignore::FutureWarning") @@ -96,6 +97,10 @@ def prinoptions(cls): itertools.chain(*[_find_doctests_in_obj(mod) for mod in tests]), ids=lambda docstring: docstring.name, ) + @pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Doctests not expected to pass on older versions of pandas", + ) def test_docstring(self, docstring): # We ignore differences in whitespace in the doctest output, and enable # the use of an ellipsis "..." to match any string in the doctest diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 74f04c0584f..0aaa71e50d7 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -188,6 +188,10 @@ def test_groupby_as_index_single_agg(pdf, gdf, as_index): @pytest.mark.parametrize("engine", ["cudf", "jit"]) @pytest.mark.parametrize("as_index", [True, False]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Include groups missing on old versions of pandas", +) def test_groupby_as_index_apply(pdf, gdf, as_index, engine): gdf = gdf.groupby("y", as_index=as_index).apply( lambda df: df["x"].mean(), engine=engine @@ -298,6 +302,10 @@ def assert_values_equal(arr): assert_values_equal(pddf[k].values) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply(): np.random.seed(0) df = DataFrame() @@ -338,6 +346,10 @@ def f3(df, k, L, m): @pytest.mark.parametrize("func,args", create_test_groupby_apply_args_params()) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_args(func, args): np.random.seed(0) df = DataFrame() @@ -500,6 +512,10 @@ def func(df): "func", ["min", "max", "sum", "mean", "var", "std", "idxmin", "idxmax"] ) @pytest.mark.parametrize("dataset", ["small", "large", "nans"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Include groups missing on old versions of pandas", +) def test_groupby_apply_jit_unary_reductions( func, dtype, dataset, groupby_jit_datasets ): @@ -530,6 +546,10 @@ def func(df): # test unary index reductions for special values +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def groupby_apply_jit_idx_reductions_special_vals_inner( func, data, dtype, special_val ): @@ -555,6 +575,10 @@ def func(df): @pytest.mark.parametrize("func", ["min", "max", "sum", "mean", "var", "std"]) @pytest.mark.parametrize("special_val", [np.nan, np.inf, -np.inf]) @pytest.mark.parametrize("dataset", ["small", "large", "nans"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Include groups missing on old versions of pandas", +) def test_groupby_apply_jit_reductions_special_vals( func, dtype, dataset, groupby_jit_datasets, special_val ): @@ -583,6 +607,10 @@ def test_groupby_apply_jit_reductions_special_vals( ], ) @pytest.mark.parametrize("dataset", ["small", "large", "nans"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="include_groups keyword new in pandas 2.2", +) def test_groupby_apply_jit_idx_reductions_special_vals( func, dtype, dataset, groupby_jit_datasets, special_val ): @@ -593,6 +621,10 @@ def test_groupby_apply_jit_idx_reductions_special_vals( @pytest.mark.parametrize("dtype", ["int32"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_jit_sum_integer_overflow(dtype): max = np.iinfo(dtype).max @@ -627,6 +659,10 @@ def func(group): "large", ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_jit_correlation(dataset, groupby_jit_datasets, dtype): dataset = groupby_jit_datasets[dataset] @@ -653,6 +689,10 @@ def func(group): @pytest.mark.parametrize("dtype", ["int32", "int64"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_jit_correlation_zero_variance(dtype): # pearson correlation is undefined when the variance of either # variable is zero. This test ensures that the jit implementation @@ -711,6 +751,10 @@ def func(group): @pytest.mark.parametrize("dtype", ["uint8", "str"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_unsupported_dtype(dtype): df = cudf.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) df["b"] = df["b"].astype(dtype) @@ -739,6 +783,10 @@ def func(group): lambda df: df["val1"].mean() + df["val2"].std(), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_jit_basic(func, groupby_jit_data_small): run_groupby_apply_jit_test(groupby_jit_data_small, func, ["key1", "key2"]) @@ -759,12 +807,20 @@ def f3(df, k, L, m): @pytest.mark.parametrize( "func,args", create_test_groupby_apply_jit_args_params() ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_jit_args(func, args, groupby_jit_data_small): run_groupby_apply_jit_test( groupby_jit_data_small, func, ["key1", "key2"], *args ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_jit_block_divergence(): # https://github.com/rapidsai/cudf/issues/12686 df = cudf.DataFrame( @@ -782,6 +838,10 @@ def diverging_block(grp_df): run_groupby_apply_jit_test(df, diverging_block, ["a"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_caching(): # Make sure similar functions that differ # by simple things like constants actually @@ -818,6 +878,10 @@ def f(group): assert precompiled.currsize == 3 +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_no_bytecode_fallback(): # tests that a function which contains no bytecode # attribute, but would still be executable using @@ -836,6 +900,10 @@ def f(group): assert_groupby_results_equal(expect, got) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_return_col_from_df(): # tests a UDF that consists of purely colwise # ops, such as `lambda group: group.x + group.y` @@ -862,6 +930,10 @@ def func(df): @pytest.mark.parametrize("func", [lambda group: group.sum()]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_return_df(func): # tests a UDF that reduces over a dataframe # and produces a series with the original column names @@ -1940,6 +2012,10 @@ def test_groupby_agg_combinations(agg): ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Include groups missing on old versions of pandas", +) def test_groupby_apply_noempty_group(): pdf = pd.DataFrame( {"a": [1, 1, 2, 2], "b": [1, 2, 1, 2], "c": [1, 2, 3, 4]} @@ -2208,6 +2284,10 @@ def f3(x, k, L, m): @pytest.mark.parametrize( "func,args", create_test_groupby_apply_return_scalars_params() ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_apply_return_scalars(func, args): pdf = pd.DataFrame( { @@ -2266,6 +2346,10 @@ def f5(x, k, L, m): @pytest.mark.parametrize( "func,args", create_test_groupby_apply_return_series_dataframe_params() ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Include groups missing on old versions of pandas", +) def test_groupby_apply_return_series_dataframe(func, args): pdf = pd.DataFrame( {"key": [0, 0, 1, 1, 2, 2, 2], "val": [0, 1, 2, 3, 4, 5, 6]} @@ -2744,6 +2828,10 @@ def test_groupby_diff_row_zero_shift(nelem): # TODO: test for category columns when cudf.Scalar supports category type @pytest.mark.parametrize("nelem", [10, 100, 1000]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="warning not present in older pandas versions", +) def test_groupby_fillna_multi_value(nelem): t = rand_dataframe( dtypes_meta=[ @@ -2790,6 +2878,10 @@ def test_groupby_fillna_multi_value(nelem): # TODO: test for category columns when cudf.Scalar supports category type # TODO: cudf.fillna does not support decimal column to column fill yet @pytest.mark.parametrize("nelem", [10, 100, 1000]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="warning not present in older pandas versions", +) def test_groupby_fillna_multi_value_df(nelem): t = rand_dataframe( dtypes_meta=[ @@ -2843,6 +2935,10 @@ def test_groupby_fillna_multi_value_df(nelem): "data", [[1, None, 2, None, 3, None], [1, 2, 3, 4, 5, 6]] ) @pytest.mark.parametrize("args", [{"value": 42}, {"method": "ffill"}]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="warning not present in older pandas versions", +) def test_groupby_various_by_fillna(by, data, args): ps = pd.Series(data) gs = cudf.from_pandas(ps) @@ -3146,6 +3242,10 @@ def test_groupby_freq_s(label, closed): ), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Warnings only given on newer versions.", +) def test_groupby_get_group(pdf, group, name, obj): gdf = cudf.from_pandas(pdf) @@ -3644,6 +3744,10 @@ def test_group_by_pandas_sort_order(groups, sort): "last", ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_group_by_empty_reduction(dtype, reduce_op): gdf = cudf.DataFrame({"a": [], "b": [], "c": []}, dtype=dtype) pdf = gdf.to_pandas() @@ -3664,6 +3768,10 @@ def test_group_by_empty_reduction(dtype, reduce_op): "apply_op", ["sum", "min", "max", "idxmax"], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_group_by_empty_apply(request, dtype, apply_op): request.applymarker( pytest.mark.xfail( @@ -3719,6 +3827,10 @@ def test_groupby_consecutive_operations(): assert_groupby_results_equal(actual, expected, check_dtype=False) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Warning only given on newer versions.", +) def test_categorical_grouping_pandas_compatibility(): gdf = cudf.DataFrame( { diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 722a64cb553..3f483219423 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -16,6 +16,11 @@ import cudf from cudf.api.extensions import no_default +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.core.index import CategoricalIndex, DatetimeIndex, Index, RangeIndex from cudf.testing import assert_eq from cudf.testing._utils import ( @@ -791,9 +796,27 @@ def test_index_to_series(data): "name_data,name_other", [("abc", "c"), (None, "abc"), ("abc", pd.NA), ("abc", "abc")], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_index_difference(data, other, sort, name_data, name_other): pd_data = pd.Index(data, name=name_data) pd_other = pd.Index(other, name=name_other) + if ( + not PANDAS_GE_220 + and isinstance(pd_data.dtype, pd.CategoricalDtype) + and not isinstance(pd_other.dtype, pd.CategoricalDtype) + and pd_other.isnull().any() + ): + pytest.skip(reason="https://github.com/pandas-dev/pandas/issues/57318") + + if ( + not PANDAS_GE_220 + and len(pd_other) == 0 + and len(pd_data) != len(pd_data.unique()) + ): + pytest.skip(reason="Bug fixed in pandas-2.2+") gd_data = cudf.from_pandas(pd_data) gd_other = cudf.from_pandas(pd_other) @@ -1017,6 +1040,10 @@ def test_index_equal_misc(data, other): ["abcd", "defgh", "werty", "poiu"], ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Does not warn on older versions of pandas", +) def test_index_append(data, other): pd_data = pd.Index(data) pd_other = pd.Index(other) @@ -1220,6 +1247,10 @@ def test_index_append_error(data, other): ), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Does not warn on older versions of pandas", +) def test_index_append_list(data, other): pd_data = data pd_other = other @@ -2084,6 +2115,10 @@ def test_get_indexer_multi_numeric_deviate(key, method): @pytest.mark.parametrize("method", ["ffill", "bfill"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_get_indexer_multi_error(method): pi = pd.MultiIndex.from_tuples( [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 10), (1, 1, 1), (2, 2, 1)] @@ -2527,7 +2562,7 @@ def test_isin_index(index, values): ) with expect_warning_if(is_dt_str): got = gidx.isin(values) - with expect_warning_if(is_dt_str): + with expect_warning_if(PANDAS_GE_220 and is_dt_str): expected = pidx.isin(values) assert_eq(got, expected) diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index 9df2852dde8..00ae99466bb 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -1016,6 +1016,10 @@ def test_series_setitem_iloc(key, value, nulls): (slice(0, 2), [0.5, 0.25]), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_series_setitem_dtype(key, value): psr = pd.Series([1, 2, 3], dtype="int32") gsr = cudf.from_pandas(psr) @@ -1634,6 +1638,10 @@ def test_dataframe_loc_iloc_inplace_update_with_RHS_dataframe( assert_eq(expected, actual) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="No warning in older versions of pandas", +) def test_dataframe_loc_inplace_update_with_invalid_RHS_df_columns(): gdf = cudf.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}) pdf = gdf.to_pandas() diff --git a/python/cudf/cudf/tests/test_interpolate.py b/python/cudf/cudf/tests/test_interpolate.py index a4f0b9fc97e..c76a49103e2 100644 --- a/python/cudf/cudf/tests/test_interpolate.py +++ b/python/cudf/cudf/tests/test_interpolate.py @@ -125,6 +125,10 @@ def test_interpolate_series_values_or_index(data, index, method): ), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Does not fail on older versions of pandas", +) def test_interpolate_dataframe_error_cases(data, kwargs): gsr = cudf.DataFrame(data) psr = gsr.to_pandas() diff --git a/python/cudf/cudf/tests/test_interval.py b/python/cudf/cudf/tests/test_interval.py index 2d194107658..5e1dd33fbf1 100644 --- a/python/cudf/cudf/tests/test_interval.py +++ b/python/cudf/cudf/tests/test_interval.py @@ -6,6 +6,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_220 from cudf.testing import assert_eq @@ -168,6 +169,10 @@ def test_interval_index_unique(): @pytest.mark.parametrize("box", [pd.Series, pd.IntervalIndex]) @pytest.mark.parametrize("tz", ["US/Eastern", None]) +@pytest.mark.skipif( + condition=not PANDAS_GE_220, + reason="ME frequency new in pandas 2.2", +) def test_interval_with_datetime(tz, box): dti = pd.date_range( start=pd.Timestamp("20180101", tz=tz), diff --git a/python/cudf/cudf/tests/test_join_order.py b/python/cudf/cudf/tests/test_join_order.py index 9ea4ba007d2..9a95f0e01ab 100644 --- a/python/cudf/cudf/tests/test_join_order.py +++ b/python/cudf/cudf/tests/test_join_order.py @@ -1,13 +1,19 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. import itertools +import operator import string +from collections import defaultdict import numpy as np import pytest import cudf -from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.testing import assert_eq @@ -35,10 +41,124 @@ def right(): # Behaviour in sort=False case didn't match documentation in many # cases prior to https://github.com/pandas-dev/pandas/pull/54611 # (released as part of pandas 2.2) -def expected(left, right, sort, *, how): - left = left.to_pandas() - right = right.to_pandas() - return left.merge(right, on="key", how=how, sort=sort) +if PANDAS_GE_220: + # Behaviour in sort=False case didn't match documentation in many + # cases prior to https://github.com/pandas-dev/pandas/pull/54611 + # (released as part of pandas 2.2) + def expected(left, right, sort, *, how): + left = left.to_pandas() + right = right.to_pandas() + return left.merge(right, on="key", how=how, sort=sort) + +else: + + def expect_inner(left, right, sort): + left_key = left.key.values_host.tolist() + left_val = left.val.values_host.tolist() + right_key = right.key.values_host.tolist() + right_val = right.val.values_host.tolist() + + right_have = defaultdict(list) + for i, k in enumerate(right_key): + right_have[k].append(i) + keys = [] + val_x = [] + val_y = [] + for k, v in zip(left_key, left_val): + if k not in right_have: + continue + for i in right_have[k]: + keys.append(k) + val_x.append(v) + val_y.append(right_val[i]) + + if sort: + # Python sort is stable, so this will preserve input order for + # equal items. + keys, val_x, val_y = zip( + *sorted(zip(keys, val_x, val_y), key=operator.itemgetter(0)) + ) + return cudf.DataFrame({"key": keys, "val_x": val_x, "val_y": val_y}) + + def expect_left(left, right, sort): + left_key = left.key.values_host.tolist() + left_val = left.val.values_host.tolist() + right_key = right.key.values_host.tolist() + right_val = right.val.values_host.tolist() + + right_have = defaultdict(list) + for i, k in enumerate(right_key): + right_have[k].append(i) + keys = [] + val_x = [] + val_y = [] + for k, v in zip(left_key, left_val): + if k not in right_have: + right_vals = [None] + else: + right_vals = [right_val[i] for i in right_have[k]] + + for rv in right_vals: + keys.append(k) + val_x.append(v) + val_y.append(rv) + + if sort: + # Python sort is stable, so this will preserve input order for + # equal items. + keys, val_x, val_y = zip( + *sorted(zip(keys, val_x, val_y), key=operator.itemgetter(0)) + ) + return cudf.DataFrame({"key": keys, "val_x": val_x, "val_y": val_y}) + + def expect_outer(left, right, sort): + left_key = left.key.values_host.tolist() + left_val = left.val.values_host.tolist() + right_key = right.key.values_host.tolist() + right_val = right.val.values_host.tolist() + right_have = defaultdict(list) + for i, k in enumerate(right_key): + right_have[k].append(i) + keys = [] + val_x = [] + val_y = [] + for k, v in zip(left_key, left_val): + if k not in right_have: + right_vals = [None] + else: + right_vals = [right_val[i] for i in right_have[k]] + for rv in right_vals: + keys.append(k) + val_x.append(v) + val_y.append(rv) + left_have = set(left_key) + for k, v in zip(right_key, right_val): + if k not in left_have: + keys.append(k) + val_x.append(None) + val_y.append(v) + + # Python sort is stable, so this will preserve input order for + # equal items. + # outer joins are always sorted, but we test both sort values + keys, val_x, val_y = zip( + *sorted(zip(keys, val_x, val_y), key=operator.itemgetter(0)) + ) + return cudf.DataFrame({"key": keys, "val_x": val_x, "val_y": val_y}) + + def expected(left, right, sort, *, how): + if how == "inner": + return expect_inner(left, right, sort) + elif how == "outer": + return expect_outer(left, right, sort) + elif how == "left": + return expect_left(left, right, sort) + elif how == "right": + return expect_left(right, left, sort).rename( + {"val_x": "val_y", "val_y": "val_x"}, axis=1 + ) + else: + raise NotImplementedError() @pytest.mark.parametrize("how", ["inner", "left", "right", "outer"]) diff --git a/python/cudf/cudf/tests/test_mvc.py b/python/cudf/cudf/tests/test_mvc.py index 7dd25ebc500..055bc5757b3 100644 --- a/python/cudf/cudf/tests/test_mvc.py +++ b/python/cudf/cudf/tests/test_mvc.py @@ -1,8 +1,9 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. import subprocess import sys import pytest +from packaging import version IS_CUDA_11 = False IS_CUDA_12 = False @@ -14,9 +15,12 @@ # do not test cuda 12 if pynvjitlink isn't present HAVE_PYNVJITLINK = False try: + import numba import pynvjitlink # noqa: F401 - HAVE_PYNVJITLINK = True + HAVE_PYNVJITLINK = version.parse(numba.__version__) >= version.parse( + "0.58" + ) except ModuleNotFoundError: pass diff --git a/python/cudf/cudf/tests/test_numerical.py b/python/cudf/cudf/tests/test_numerical.py index 1b0589254f5..b1a2f081cd2 100644 --- a/python/cudf/cudf/tests/test_numerical.py +++ b/python/cudf/cudf/tests/test_numerical.py @@ -5,6 +5,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_220 from cudf.testing import assert_eq from cudf.testing._utils import NUMERIC_TYPES, expect_warning_if from cudf.utils.dtypes import np_dtypes_to_pandas_dtypes @@ -373,7 +374,7 @@ def test_to_numeric_error(data, errors): ): cudf.to_numeric(data, errors=errors) else: - with expect_warning_if(errors == "ignore"): + with expect_warning_if(PANDAS_GE_220 and errors == "ignore"): expect = pd.to_numeric(data, errors=errors) with expect_warning_if(errors == "ignore"): got = cudf.to_numeric(data, errors=errors) diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index e0884a5819a..c2a30b76bea 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -1679,7 +1679,13 @@ def run_orc_columns_and_index_param(index_obj, index, columns): "columns", [ None, - [], + pytest.param( + [], + marks=pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Bug in older version of pandas", + ), + ), ], ) def test_orc_columns_and_index_param(index_obj, index, columns): diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 6623c537ddf..8b59a7eef08 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -23,6 +23,7 @@ import cudf from cudf._lib.parquet import read_parquet_chunked +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION from cudf.io.parquet import ( ParquetDatasetWriter, ParquetWriter, @@ -3034,6 +3035,10 @@ def test_parquet_reader_rle_boolean(datadir): # a list column in a schema, the cudf reader was confusing # nesting information between a list column and a subsequent # string column, ultimately causing a crash. +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Older versions of pandas do not have DataFrame.map()", +) def test_parquet_reader_one_level_list2(datadir): # we are reading in a file containing binary types, but cudf returns # those as strings. so we have to massage the pandas data to get diff --git a/python/cudf/cudf/tests/test_reductions.py b/python/cudf/cudf/tests/test_reductions.py index a70a2ea15dd..f276f394cd0 100644 --- a/python/cudf/cudf/tests/test_reductions.py +++ b/python/cudf/cudf/tests/test_reductions.py @@ -10,6 +10,7 @@ import cudf from cudf import Series +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing import _utils as utils, assert_eq from cudf.testing._utils import NUMERIC_TYPES, expect_warning_if, gen_rand @@ -342,6 +343,10 @@ def test_any_all_axis_none(data, op): "median", ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Warning not given on older versions of pandas", +) def test_reductions_axis_none_warning(op): df = cudf.DataFrame({"a": [1, 2, 3], "b": [10, 2, 3]}) pdf = df.to_pandas() diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index e5ee0127a74..3a8928297c0 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -10,7 +10,11 @@ import pytest import cudf -from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing import assert_eq from cudf.testing._utils import ( @@ -66,7 +70,7 @@ def test_series_replace_all(gsr, to_replace, value): ) with expect_warning_if(expect_warn): actual = gsr.replace(to_replace=gd_to_replace, value=gd_value) - with expect_warning_if(expect_warn): + with expect_warning_if(expect_warn and PANDAS_GE_220): if pd_value is None: # TODO: Remove this workaround once cudf # introduces `no_default` values @@ -91,7 +95,7 @@ def test_series_replace(): # Categorical psr3 = pd.Series(["one", "two", "three"], dtype="category") - with pytest.warns(FutureWarning): + with expect_warning_if(PANDAS_GE_220, FutureWarning): psr4 = psr3.replace("one", "two") sr3 = cudf.from_pandas(psr3) with pytest.warns(FutureWarning): @@ -100,7 +104,7 @@ def test_series_replace(): psr4.sort_values().reset_index(drop=True), sr4.sort_values().reset_index(drop=True), ) - with pytest.warns(FutureWarning): + with expect_warning_if(PANDAS_GE_220, FutureWarning): psr5 = psr3.replace("one", "five") with pytest.warns(FutureWarning): sr5 = sr3.replace("one", "five") @@ -517,7 +521,7 @@ def test_fillna_categorical(psr_data, fill_value, inplace): pd.date_range( "2010-01-01", "2020-01-10", - freq="1YE", + freq="1YE" if PANDAS_GE_220 else "1y", ) ), pd.Series(["2010-01-01", None, "2011-10-10"], dtype="datetime64[ns]"), @@ -564,7 +568,7 @@ def test_fillna_categorical(psr_data, fill_value, inplace): pd.date_range( "2010-01-01", "2020-01-10", - freq="1YE", + freq="1YE" if PANDAS_GE_220 else "1y", ) ) + pd.Timedelta("1d"), @@ -1069,6 +1073,10 @@ def test_numeric_series_replace_dtype(series_dtype, replacement): ), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Warning not given on older versions of pandas", +) def test_replace_inplace(pframe, replace_args): gpu_frame = cudf.from_pandas(pframe) pandas_frame = pframe.copy() diff --git a/python/cudf/cudf/tests/test_resampling.py b/python/cudf/cudf/tests/test_resampling.py index 95fa8e9a50a..a61477981f8 100644 --- a/python/cudf/cudf/tests/test_resampling.py +++ b/python/cudf/cudf/tests/test_resampling.py @@ -5,6 +5,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION from cudf.testing import assert_eq @@ -147,6 +148,10 @@ def test_dataframe_resample_level(): ("10D", "1D", "s"), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_resampling_frequency_conversion(in_freq, sampling_freq, out_freq): # test that we cast to the appropriate frequency # when resampling: @@ -164,6 +169,10 @@ def test_resampling_frequency_conversion(in_freq, sampling_freq, out_freq): assert got.index.dtype == np.dtype(f"datetime64[{out_freq}]") +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_resampling_downsampling_ms(): pdf = pd.DataFrame( { diff --git a/python/cudf/cudf/tests/test_reshape.py b/python/cudf/cudf/tests/test_reshape.py index 50db4302b75..4235affd4d1 100644 --- a/python/cudf/cudf/tests/test_reshape.py +++ b/python/cudf/cudf/tests/test_reshape.py @@ -8,10 +8,19 @@ import pytest import cudf -from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION +from cudf.core._compat import ( + PANDAS_CURRENT_SUPPORTED_VERSION, + PANDAS_GE_220, + PANDAS_VERSION, +) from cudf.core.buffer.spill_manager import get_global_manager from cudf.testing import assert_eq -from cudf.testing._utils import ALL_TYPES, DATETIME_TYPES, NUMERIC_TYPES +from cudf.testing._utils import ( + ALL_TYPES, + DATETIME_TYPES, + NUMERIC_TYPES, + expect_warning_if, +) pytest_xfail = pytest.mark.xfail pytestmark = pytest.mark.spilling @@ -220,7 +229,7 @@ def test_df_stack_multiindex_column_axis(columns, index, level, dropna): with pytest.warns(FutureWarning): got = gdf.stack(level=level, dropna=dropna, future_stack=False) - with pytest.warns(FutureWarning): + with expect_warning_if(PANDAS_GE_220, FutureWarning): expect = pdf.stack(level=level, dropna=dropna, future_stack=False) assert_eq(expect, got, check_dtype=False) @@ -265,7 +274,7 @@ def test_df_stack_multiindex_column_axis_pd_example(level): df = pd.DataFrame(np.random.randn(4, 4), columns=columns) - with pytest.warns(FutureWarning): + with expect_warning_if(PANDAS_GE_220, FutureWarning): expect = df.stack(level=level, future_stack=False) gdf = cudf.from_pandas(df) with pytest.warns(FutureWarning): diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index d5f63fdab77..f952cea07f8 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -447,6 +447,10 @@ def test_cov1d(data1, data2): ], ) @pytest.mark.parametrize("method", ["spearman", "pearson"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Warnings missing on older pandas (scipy version seems unrelated?)", +) def test_corr1d(data1, data2, method): if method == "spearman": # Pandas uses scipy.stats.spearmanr code-path @@ -585,6 +589,10 @@ def test_min_count_ops(data, ops, skipna, min_count): ], ) @pytest.mark.parametrize("dtype", ["datetime64[ns]", "timedelta64[ns]"]) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_cov_corr_datetime_timedelta(data1, data2, dtype): gsr1 = cudf.Series(data1, dtype=dtype) gsr2 = cudf.Series(data2, dtype=dtype) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 505d5d0b9cc..d10c531d757 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -23,6 +23,7 @@ from numba import NumbaDeprecationWarning from pytz import utc +from cudf.core._compat import PANDAS_GE_220 from cudf.pandas import LOADED, Profiler from cudf.pandas.fast_slow_proxy import _Unusable, is_proxy_object @@ -536,12 +537,15 @@ def test_array_ufunc(series): @pytest.mark.xfail(strict=False, reason="Fails in CI, passes locally.") def test_groupby_apply_func_returns_series(dataframe): pdf, df = dataframe + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expect = pdf.groupby("a").apply( - lambda group: pd.Series({"x": 1}), include_groups=False - ) - got = df.groupby("a").apply( - lambda group: xpd.Series({"x": 1}), include_groups=False + lambda group: pd.Series({"x": 1}), **kwargs ) + got = df.groupby("a").apply(lambda group: xpd.Series({"x": 1}), **kwargs) tm.assert_equal(expect, got) diff --git a/python/dask_cudf/dask_cudf/tests/test_applymap.py b/python/dask_cudf/dask_cudf/tests/test_applymap.py index d84235481c3..e4e79b7b8cf 100644 --- a/python/dask_cudf/dask_cudf/tests/test_applymap.py +++ b/python/dask_cudf/dask_cudf/tests/test_applymap.py @@ -5,6 +5,8 @@ from dask import dataframe as dd +from cudf.core._compat import PANDAS_GE_210 + from dask_cudf.tests.utils import _make_random_frame @@ -18,6 +20,10 @@ ], ) @pytest.mark.parametrize("has_na", [True, False]) +@pytest.mark.skipif( + not PANDAS_GE_210, + reason="DataFrame.map requires pandas>=2.1.0", +) def test_applymap_basic(func, has_na): size = 2000 pdf, dgdf = _make_random_frame(size, include_na=False) diff --git a/python/dask_cudf/dask_cudf/tests/test_distributed.py b/python/dask_cudf/dask_cudf/tests/test_distributed.py index be10b0d4843..d03180852eb 100644 --- a/python/dask_cudf/dask_cudf/tests/test_distributed.py +++ b/python/dask_cudf/dask_cudf/tests/test_distributed.py @@ -80,6 +80,11 @@ def test_str_series_roundtrip(): def test_p2p_shuffle(): + pytest.importorskip( + "pyarrow", + minversion="14.0.1", + reason="P2P shuffling requires pyarrow>=14.0.1", + ) # Check that we can use `shuffle_method="p2p"` with dask_cuda.LocalCUDACluster(n_workers=1) as cluster: with Client(cluster): diff --git a/python/dask_cudf/dask_cudf/tests/test_groupby.py b/python/dask_cudf/dask_cudf/tests/test_groupby.py index cf916b713b2..7b9f0ca328a 100644 --- a/python/dask_cudf/dask_cudf/tests/test_groupby.py +++ b/python/dask_cudf/dask_cudf/tests/test_groupby.py @@ -9,6 +9,7 @@ from dask.utils_test import hlg_layer import cudf +from cudf.core._compat import PANDAS_CURRENT_SUPPORTED_VERSION, PANDAS_VERSION from cudf.testing._utils import expect_warning_if import dask_cudf @@ -316,6 +317,10 @@ def test_groupby_dropna_cudf(dropna, by): (None, ["a", "d"]), ], ) +@pytest.mark.skipif( + PANDAS_VERSION < PANDAS_CURRENT_SUPPORTED_VERSION, + reason="Fails in older versions of pandas", +) def test_groupby_dropna_dask(dropna, by): # NOTE: This test is borrowed from upstream dask # (dask/dask/dataframe/tests/test_groupby.py) From e1ab1e799d7a29289419014e19ec5c6f2e99ae91 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Thu, 5 Sep 2024 09:48:03 -0400 Subject: [PATCH 169/270] Make isinstance check pass for proxy ndarrays (#16601) Closes #14537. Authors: - Matthew Murray (https://github.com/Matt711) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16601 --- python/cudf/cudf/pandas/_wrappers/numpy.py | 23 +++++++++ python/cudf/cudf/pandas/fast_slow_proxy.py | 26 +++++++++- python/cudf/cudf/pandas/proxy_base.py | 22 ++++++++ .../cudf_pandas_tests/test_cudf_pandas.py | 50 ++++++++++++++++++- 4 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 python/cudf/cudf/pandas/proxy_base.py diff --git a/python/cudf/cudf/pandas/_wrappers/numpy.py b/python/cudf/cudf/pandas/_wrappers/numpy.py index 90ac5198270..d5e669cb58f 100644 --- a/python/cudf/cudf/pandas/_wrappers/numpy.py +++ b/python/cudf/cudf/pandas/_wrappers/numpy.py @@ -10,10 +10,13 @@ from packaging import version from ..fast_slow_proxy import ( + _fast_slow_function_call, _FastSlowAttribute, + is_proxy_object, make_final_proxy_type, make_intermediate_proxy_type, ) +from ..proxy_base import ProxyNDarrayBase from .common import ( array_interface, array_method, @@ -105,18 +108,38 @@ def wrap_ndarray(cls, arr: cupy.ndarray | numpy.ndarray, constructor): return super(cls, cls)._fsproxy_wrap(arr, constructor) +def ndarray__array_ufunc__(self, ufunc, method, *inputs, **kwargs): + result, _ = _fast_slow_function_call( + getattr(ufunc, method), + *inputs, + **kwargs, + ) + if isinstance(result, tuple): + if is_proxy_object(result[0]) and isinstance( + result[0]._fsproxy_wrapped, numpy.ndarray + ): + return tuple(numpy.asarray(x) for x in result) + elif is_proxy_object(result) and isinstance( + result._fsproxy_wrapped, numpy.ndarray + ): + return numpy.asarray(result) + return result + + ndarray = make_final_proxy_type( "ndarray", cupy.ndarray, numpy.ndarray, fast_to_slow=cupy.ndarray.get, slow_to_fast=cupy.asarray, + bases=(ProxyNDarrayBase,), additional_attributes={ "__array__": array_method, # So that pa.array(wrapped-numpy-array) works "__arrow_array__": arrow_array_method, "__cuda_array_interface__": cuda_array_interface, "__array_interface__": array_interface, + "__array_ufunc__": ndarray__array_ufunc__, # ndarrays are unhashable "__hash__": None, # iter(cupy-array) produces an iterable of zero-dim device diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index 4b0fd9a5b36..afa1ce5f86c 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -19,6 +19,7 @@ from ..options import _env_get_bool from ..testing import assert_eq from .annotation import nvtx +from .proxy_base import ProxyNDarrayBase def call_operator(fn, args, kwargs): @@ -564,7 +565,17 @@ def _fsproxy_wrap(cls, value, func): _FinalProxy subclasses can override this classmethod if they need particular behaviour when wrapped up. """ - proxy = object.__new__(cls) + # TODO: Replace the if-elif-else using singledispatch helper function + base_class = _get_proxy_base_class(cls) + if base_class is object: + proxy = base_class.__new__(cls) + elif base_class is ProxyNDarrayBase: + proxy = base_class.__new__(cls, value) + else: + raise TypeError( + f"Cannot create an proxy instance of {cls.__name__} using base class {base_class.__name__}. " + f"Expected either 'object' or another type in 'PROXY_BASE_CLASSES'" + ) proxy._fsproxy_wrapped = value return proxy @@ -1193,6 +1204,19 @@ def is_proxy_object(obj: Any) -> bool: return False +def _get_proxy_base_class(cls): + """Returns the proxy base class if one exists""" + for proxy_class in PROXY_BASE_CLASSES: + if proxy_class in cls.__mro__: + return proxy_class + return object + + +PROXY_BASE_CLASSES: set[type] = { + ProxyNDarrayBase, +} + + NUMPY_TYPES: set[str] = set(np.sctypeDict.values()) diff --git a/python/cudf/cudf/pandas/proxy_base.py b/python/cudf/cudf/pandas/proxy_base.py new file mode 100644 index 00000000000..6f732834e94 --- /dev/null +++ b/python/cudf/cudf/pandas/proxy_base.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import cupy as cp +import numpy as np + + +class ProxyNDarrayBase(np.ndarray): + def __new__(cls, arr): + if isinstance(arr, cp.ndarray): + arr = arr.get() + if not isinstance(arr, np.ndarray): + raise TypeError( + "Unsupported array type. Must be numpy.ndarray or cupy.ndarray" + ) + return np.asarray(arr, dtype=arr.dtype).view(cls) + + def __array_finalize__(self, obj): + if obj is None: + return + self._fsproxy_wrapped = getattr(obj, "_fsproxy_wrapped", obj) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index d10c531d757..c4ab4b0a853 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -14,18 +14,20 @@ import types from io import BytesIO, StringIO +import cupy as cp import jupyter_client import nbformat import numpy as np import pyarrow as pa import pytest from nbconvert.preprocessors import ExecutePreprocessor -from numba import NumbaDeprecationWarning +from numba import NumbaDeprecationWarning, vectorize from pytz import utc from cudf.core._compat import PANDAS_GE_220 from cudf.pandas import LOADED, Profiler from cudf.pandas.fast_slow_proxy import _Unusable, is_proxy_object +from cudf.testing import assert_eq if not LOADED: raise ImportError("These tests must be run with cudf.pandas loaded") @@ -1690,3 +1692,49 @@ def test_notebook_slow_repr(): assert ( string in html_result ), f"Expected string {string} not found in the output" + + +def test_numpy_ndarray_isinstancecheck(array): + arr1, arr2 = array + assert isinstance(arr1, np.ndarray) + assert isinstance(arr2, np.ndarray) + + +def test_numpy_ndarray_np_ufunc(array): + arr1, arr2 = array + + @np.vectorize + def add_one_ufunc(arr): + return arr + 1 + + assert_eq(add_one_ufunc(arr1), add_one_ufunc(arr2)) + + +def test_numpy_ndarray_cp_ufunc(array): + arr1, arr2 = array + + @cp.vectorize + def add_one_ufunc(arr): + return arr + 1 + + assert_eq(add_one_ufunc(cp.asarray(arr1)), add_one_ufunc(arr2)) + + +def test_numpy_ndarray_numba_ufunc(array): + arr1, arr2 = array + + @vectorize + def add_one_ufunc(arr): + return arr + 1 + + assert_eq(add_one_ufunc(arr1), add_one_ufunc(arr2)) + + +def test_numpy_ndarray_numba_cuda_ufunc(array): + arr1, arr2 = array + + @vectorize(["int64(int64)"], target="cuda") + def add_one_ufunc(a): + return a + 1 + + assert_eq(cp.asarray(add_one_ufunc(arr1)), cp.asarray(add_one_ufunc(arr2))) From 949f1719226f0b27a4df8fedbf4624f46fb0589d Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 5 Sep 2024 09:52:01 -0400 Subject: [PATCH 170/270] Performance improvement for strings::slice for wide strings (#16574) Improves performance of wide strings (avg > 64 bytes) when using `cudf::strings::slice_strings`. Addresses some concerns from issue #15924 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/16574 --- cpp/src/strings/slice.cu | 182 ++++++++++++++++++++++++++++++--------- 1 file changed, 141 insertions(+), 41 deletions(-) diff --git a/cpp/src/strings/slice.cu b/cpp/src/strings/slice.cu index cf82a837c51..d8324a9b08e 100644 --- a/cpp/src/strings/slice.cu +++ b/cpp/src/strings/slice.cu @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,8 @@ #include #include +#include +#include #include #include #include @@ -40,6 +43,9 @@ namespace cudf { namespace strings { namespace detail { namespace { + +constexpr size_type AVG_CHAR_BYTES_THRESHOLD = 128; + /** * @brief Function logic for compute_substrings_from_fn API * @@ -51,17 +57,19 @@ struct substring_from_fn { IndexIterator const starts; IndexIterator const stops; - __device__ string_view operator()(size_type idx) const + __device__ string_index_pair operator()(size_type idx) const { - if (d_column.is_null(idx)) { return string_view{nullptr, 0}; } + if (d_column.is_null(idx)) { return string_index_pair{nullptr, 0}; } auto const d_str = d_column.template element(idx); auto const length = d_str.length(); auto const start = std::max(starts[idx], 0); - if (start >= length) { return string_view{}; } + if (start >= length) { return string_index_pair{"", 0}; } - auto const stop = stops[idx]; - auto const end = (((stop < 0) || (stop > length)) ? length : stop); - return start < end ? d_str.substr(start, end - start) : string_view{}; + auto const stop = stops[idx]; + auto const end = (((stop < 0) || (stop > length)) ? length : stop); + auto const sub_str = start < end ? d_str.substr(start, end - start) : string_view{}; + return sub_str.empty() ? string_index_pair{"", 0} + : string_index_pair{sub_str.data(), sub_str.size_bytes()}; } substring_from_fn(column_device_view const& d_column, IndexIterator starts, IndexIterator stops) @@ -70,6 +78,82 @@ struct substring_from_fn { } }; +template +CUDF_KERNEL void substring_from_kernel(column_device_view const d_strings, + IndexIterator starts, + IndexIterator stops, + string_index_pair* d_output) +{ + auto const idx = cudf::detail::grid_1d::global_thread_id(); + auto const str_idx = idx / cudf::detail::warp_size; + if (str_idx >= d_strings.size()) { return; } + + namespace cg = cooperative_groups; + auto const warp = cg::tiled_partition(cg::this_thread_block()); + + if (d_strings.is_null(str_idx)) { + if (warp.thread_rank() == 0) { d_output[str_idx] = string_index_pair{nullptr, 0}; } + return; + } + auto const d_str = d_strings.element(str_idx); + if (d_str.empty()) { + if (warp.thread_rank() == 0) { d_output[str_idx] = string_index_pair{"", 0}; } + return; + } + + auto const start = max(starts[str_idx], 0); + auto stop = [stop = stops[str_idx]] { + return (stop < 0) ? std::numeric_limits::max() : stop; + }(); + auto const end = d_str.data() + d_str.size_bytes(); + + auto start_counts = thrust::make_pair(0, 0); + auto stop_counts = thrust::make_pair(0, 0); + + auto itr = d_str.data() + warp.thread_rank(); + + size_type char_count = 0; + size_type byte_count = 0; + while (byte_count < d_str.size_bytes()) { + if (char_count <= start) { start_counts = {char_count, byte_count}; } + if (char_count <= stop) { + stop_counts = {char_count, byte_count}; + } else { + break; + } + size_type const cc = (itr < end) && is_begin_utf8_char(*itr); + size_type const bc = (itr < end); + char_count += cg::reduce(warp, cc, cg::plus()); + byte_count += cg::reduce(warp, bc, cg::plus()); + itr += cudf::detail::warp_size; + } + + if (warp.thread_rank() == 0) { + if (start >= char_count) { + d_output[str_idx] = string_index_pair{"", 0}; + return; + } + + // we are just below start/stop and must now increment up to it from here + auto first_byte = start_counts.second; + if (start_counts.first < start) { + auto const sub_str = string_view(d_str.data() + first_byte, d_str.size_bytes() - first_byte); + first_byte += std::get<0>(bytes_to_character_position(sub_str, start - start_counts.first)); + } + + stop = max(stop, char_count); + auto last_byte = stop_counts.second; + if (stop_counts.first < stop) { + auto const sub_str = string_view(d_str.data() + last_byte, d_str.size_bytes() - last_byte); + last_byte += std::get<0>(bytes_to_character_position(sub_str, stop - stop_counts.first)); + } + + d_output[str_idx] = (first_byte < last_byte) + ? string_index_pair{d_str.data() + first_byte, last_byte - first_byte} + : string_index_pair{"", 0}; + } +} + /** * @brief Function logic for the substring API. * @@ -149,54 +233,67 @@ struct substring_fn { * * @tparam IndexIterator Iterator type for character position values * - * @param d_column Input strings column to substring + * @param input Input strings column to substring * @param starts Start positions index iterator * @param stops Stop positions index iterator * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory */ template -std::unique_ptr compute_substrings_from_fn(column_device_view const& d_column, +std::unique_ptr compute_substrings_from_fn(strings_column_view const& input, IndexIterator starts, IndexIterator stops, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - auto results = rmm::device_uvector(d_column.size(), stream); - thrust::transform(rmm::exec_policy(stream), - thrust::counting_iterator(0), - thrust::counting_iterator(d_column.size()), - results.begin(), - substring_from_fn{d_column, starts, stops}); - return make_strings_column(results, string_view{nullptr, 0}, stream, mr); + auto results = rmm::device_uvector(input.size(), stream); + + auto const d_column = column_device_view::create(input.parent(), stream); + + if ((input.chars_size(stream) / (input.size() - input.null_count())) < AVG_CHAR_BYTES_THRESHOLD) { + thrust::transform(rmm::exec_policy(stream), + thrust::counting_iterator(0), + thrust::counting_iterator(input.size()), + results.begin(), + substring_from_fn{*d_column, starts, stops}); + } else { + constexpr thread_index_type block_size = 512; + auto const threads = + static_cast(input.size()) * cudf::detail::warp_size; + auto const num_blocks = util::div_rounding_up_safe(threads, block_size); + substring_from_kernel + <<>>(*d_column, starts, stops, results.data()); + } + return make_strings_column(results.begin(), results.end(), stream, mr); } } // namespace // -std::unique_ptr slice_strings(strings_column_view const& strings, +std::unique_ptr slice_strings(strings_column_view const& input, numeric_scalar const& start, numeric_scalar const& stop, numeric_scalar const& step, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - if (strings.is_empty()) return make_empty_column(type_id::STRING); + if (input.size() == input.null_count()) { + return std::make_unique(input.parent(), stream, mr); + } auto const step_valid = step.is_valid(stream); - auto const step_value = step_valid ? step.value(stream) : 0; + auto const step_value = step_valid ? step.value(stream) : 1; if (step_valid) { CUDF_EXPECTS(step_value != 0, "Step parameter must not be 0"); } - auto const d_column = column_device_view::create(strings.parent(), stream); - // optimization for (step==1 and start < stop) -- expect this to be most common - if (step_value == 1 and start.is_valid(stream) and stop.is_valid(stream)) { - auto const start_value = start.value(stream); - auto const stop_value = stop.value(stream); + if (step_value == 1) { + auto const start_value = start.is_valid(stream) ? start.value(stream) : 0; + auto const stop_value = + stop.is_valid(stream) ? stop.value(stream) : std::numeric_limits::max(); // note that any negative values here must use the alternate function below if ((start_value >= 0) && (start_value < stop_value)) { // this is about 2x faster on long strings for this common case - return compute_substrings_from_fn(*d_column, + return compute_substrings_from_fn(input, thrust::constant_iterator(start_value), thrust::constant_iterator(stop_value), stream, @@ -204,31 +301,35 @@ std::unique_ptr slice_strings(strings_column_view const& strings, } } + auto const d_column = column_device_view::create(input.parent(), stream); + auto const d_start = get_scalar_device_view(const_cast&>(start)); auto const d_stop = get_scalar_device_view(const_cast&>(stop)); auto const d_step = get_scalar_device_view(const_cast&>(step)); auto [offsets, chars] = make_strings_children( - substring_fn{*d_column, d_start, d_stop, d_step}, strings.size(), stream, mr); + substring_fn{*d_column, d_start, d_stop, d_step}, input.size(), stream, mr); - return make_strings_column(strings.size(), + return make_strings_column(input.size(), std::move(offsets), chars.release(), - strings.null_count(), - cudf::detail::copy_bitmask(strings.parent(), stream, mr)); + input.null_count(), + cudf::detail::copy_bitmask(input.parent(), stream, mr)); } -std::unique_ptr slice_strings(strings_column_view const& strings, +std::unique_ptr slice_strings(strings_column_view const& input, column_view const& starts_column, column_view const& stops_column, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - size_type strings_count = strings.size(); - if (strings_count == 0) return make_empty_column(type_id::STRING); - CUDF_EXPECTS(starts_column.size() == strings_count, + if (input.size() == input.null_count()) { + return std::make_unique(input.parent(), stream, mr); + } + + CUDF_EXPECTS(starts_column.size() == input.size(), "Parameter starts must have the same number of rows as strings."); - CUDF_EXPECTS(stops_column.size() == strings_count, + CUDF_EXPECTS(stops_column.size() == input.size(), "Parameter stops must have the same number of rows as strings."); CUDF_EXPECTS(cudf::have_same_types(starts_column, stops_column), "Parameters starts and stops must be of the same type.", @@ -242,17 +343,16 @@ std::unique_ptr slice_strings(strings_column_view const& strings, "Positions values must be fixed width type.", cudf::data_type_error); - auto strings_column = column_device_view::create(strings.parent(), stream); - auto starts_iter = cudf::detail::indexalator_factory::make_input_iterator(starts_column); - auto stops_iter = cudf::detail::indexalator_factory::make_input_iterator(stops_column); - return compute_substrings_from_fn(*strings_column, starts_iter, stops_iter, stream, mr); + auto starts_iter = cudf::detail::indexalator_factory::make_input_iterator(starts_column); + auto stops_iter = cudf::detail::indexalator_factory::make_input_iterator(stops_column); + return compute_substrings_from_fn(input, starts_iter, stops_iter, stream, mr); } } // namespace detail // external API -std::unique_ptr slice_strings(strings_column_view const& strings, +std::unique_ptr slice_strings(strings_column_view const& input, numeric_scalar const& start, numeric_scalar const& stop, numeric_scalar const& step, @@ -260,17 +360,17 @@ std::unique_ptr slice_strings(strings_column_view const& strings, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::slice_strings(strings, start, stop, step, stream, mr); + return detail::slice_strings(input, start, stop, step, stream, mr); } -std::unique_ptr slice_strings(strings_column_view const& strings, +std::unique_ptr slice_strings(strings_column_view const& input, column_view const& starts_column, column_view const& stops_column, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::slice_strings(strings, starts_column, stops_column, stream, mr); + return detail::slice_strings(input, starts_column, stops_column, stream, mr); } } // namespace strings From 0cc059fb2b81adbdc9593052292838995dc78b10 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Thu, 5 Sep 2024 15:07:29 -0700 Subject: [PATCH 171/270] Upgrade to nvcomp 4.0.1 (#16076) This PR bumps nvcomp to 4.0.1. Depends on: - https://github.com/conda-forge/nvcomp-feedstock/pull/15 - https://github.com/rapidsai/rapids-cmake/pull/633 - https://github.com/rapidsai/kvikio/pull/449 Authors: - Vukasin Milovanovic (https://github.com/vuule) - Robert Maynard (https://github.com/robertmaynard) - Peixin (https://github.com/pxLi) - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) - Robert Maynard (https://github.com/robertmaynard) URL: https://github.com/rapidsai/cudf/pull/16076 --- ci/build_wheel_cudf.sh | 2 -- ci/build_wheel_pylibcudf.sh | 2 -- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- dependencies.yaml | 2 +- java/pom.xml | 4 +--- java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java | 3 --- java/src/main/native/CMakeLists.txt | 5 ++--- python/libcudf/CMakeLists.txt | 3 +-- 10 files changed, 8 insertions(+), 19 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index e5565c4b53c..fb93b06dbe2 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -23,8 +23,6 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ --exclude libnvcomp.so \ - --exclude libnvcomp_bitcomp.so \ - --exclude libnvcomp_gdeflate.so \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 0e4745bda28..5e9f7f8a0c4 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -21,8 +21,6 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ --exclude libnvcomp.so \ - --exclude libnvcomp_bitcomp.so \ - --exclude libnvcomp_gdeflate.so \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 7f6967d7287..fa4c77d67b4 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -58,7 +58,7 @@ dependencies: - numpy>=1.23,<3.0a0 - numpydoc - nvcc_linux-64=11.8 -- nvcomp==3.0.6 +- nvcomp==4.0.1 - nvtx>=0.2.1 - openpyxl - packaging diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index c1315e73f16..9b487347a5e 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -56,7 +56,7 @@ dependencies: - numba>=0.57 - numpy>=1.23,<3.0a0 - numpydoc -- nvcomp==3.0.6 +- nvcomp==4.0.1 - nvtx>=0.2.1 - openpyxl - packaging diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 4b1c4cca828..dae04c08aca 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -35,7 +35,7 @@ spdlog_version: - ">=1.12.0,<1.13" nvcomp_version: - - "=3.0.6" + - "=4.0.1" zlib_version: - ">=1.2.13" diff --git a/dependencies.yaml b/dependencies.yaml index f8b231efd6d..a3f0ffeec82 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -354,7 +354,7 @@ dependencies: - flatbuffers==24.3.25 - librdkafka>=1.9.0,<1.10.0a0 # Align nvcomp version with rapids-cmake - - nvcomp==3.0.6 + - nvcomp==4.0.1 - spdlog>=1.12.0,<1.13 rapids_build_skbuild: common: diff --git a/java/pom.xml b/java/pom.xml index 9694e741f16..e4f1cdf64e7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1,6 +1,6 @@