remove streamz git dependency, standardize build dependency names, co…

…nsolidate some dependency lists (#16611)

Proposes some additional cleanup in `dependencies.yaml`, for things I noticed while working through #15483.

* standardizes the naming of keys in the `files:` section for build dependencies
  - *`py_build_{project}` = dependencies for the `[build-system]` table*
  - *`py_rapids_build_{project}` = dependencies for the `[tool.rapids-build-backend]` table*
  - *this is how it was done over most of the other repos in rapidsai/build-planning#31, it was just missed because `cudf` was one of the first repos to add `rapids-build-backend`*
* removes the dependency on building `streamz` from latest source on GitHub
  - *`custreamz` conda packages and wheels depend on packages for those, not this git dependency*
    - https://github.com/rapidsai/cudf/blob/2f7d35435db2b5ed9ead96cf43e2a710db5e5e6d/dependencies.yaml#L752-L754
    - https://github.com/rapidsai/cudf/blob/2f7d35435db2b5ed9ead96cf43e2a710db5e5e6d/conda/recipes/custreamz/meta.yaml#L45-L47
  - *if this is really needed, I don't think it belongs in the `build_python_cudf` set*
  - *the last commit to `streamz` was 2 years ago (https://github.com/python-streamz/streamz), this doesn't seem like a `rapids-dask-dependency`, try-to-always-test-against-latest, situation to me*
  - *I'm guessing this is left over from a time before `streamz` was regularly publishing wheels... it's been in `dependencies.yaml` since that file was first introduced here in November 2022 (#11674)*
  - *the last release, v0.6.4, was made on July 27, 2022. There have been around 20 commits to `master` since then ([history link](https://github.com/python-streamz/streamz/commits/master/)) ... but if `custreamz` really needed those, I'd expect `custreamz` to depend on the version built from GitHub sources. I strongly suspect that that isn't the case.*
* removes `build_python_cudf` and `build_python_libcudf` lists in `dependencies.yaml`, in favor of re-using the `depends_on_rmm` and `depends_on_pylibcudf` lists

Authors:
  - James Lamb (https://github.com/jameslamb)

Approvers:
  - Bradley Dice (https://github.com/bdice)

URL: #16611

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -66,7 +66,6 @@ dependencies:
 - pandas
 - pandas>=2.0,<2.2.3dev0
 - pandoc
-- pip
 - pre-commit
 - ptxcompiler
 - pyarrow==16.1.0.*
@@ -99,6 +98,4 @@ dependencies:
 - transformers==4.39.3
 - typing_extensions>=4.0.0
 - zlib>=1.2.13
-- pip:
-  - git+https://github.com/python-streamz/streamz.git@master
 name: all_cuda-118_arch-x86_64

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -64,7 +64,6 @@ dependencies:
 - pandas
 - pandas>=2.0,<2.2.3dev0
 - pandoc
-- pip
 - pre-commit
 - pyarrow==16.1.0.*
 - pydata-sphinx-theme!=0.14.2
@@ -97,6 +96,4 @@ dependencies:
 - transformers==4.39.3
 - typing_extensions>=4.0.0
 - zlib>=1.2.13
-- pip:
-  - git+https://github.com/python-streamz/streamz.git@master
 name: all_cuda-125_arch-x86_64

dependencies.yaml

@@ -10,10 +10,10 @@ files:
       - build_all
       - build_cpp
       - build_python_common
-      - build_python_pylibcudf
-      - build_python_cudf
       - cuda
       - cuda_version
+      - depends_on_cupy
+      - depends_on_rmm
       - develop
       - docs
       - libarrow_build
@@ -31,7 +31,6 @@ files:
       - test_python_cudf
       - test_python_dask_cudf
       - test_python_pylibcudf
-      - depends_on_cupy
   test_static_build:
     output: none
     includes:
@@ -95,7 +94,8 @@ files:
     includes:
       - build_base
       - build_python_common
-      - build_python_cudf
+      - depends_on_pylibcudf
+      - depends_on_rmm
   py_run_cudf:
     output: pyproject
     pyproject_dir: python/cudf
@@ -107,6 +107,7 @@ files:
       - pyarrow_run
       - depends_on_cupy
       - depends_on_pylibcudf
+      - depends_on_rmm
   py_test_cudf:
     output: pyproject
     pyproject_dir: python/cudf
@@ -116,14 +117,14 @@ files:
     includes:
       - test_python_common
       - test_python_cudf
-  py_rapids_build_pylibcudf:
+  py_build_pylibcudf:
     output: pyproject
     pyproject_dir: python/pylibcudf
     extras:
       table: build-system
     includes:
       - rapids_build_skbuild
-  py_build_pylibcudf:
+  py_rapids_build_pylibcudf:
     output: pyproject
     pyproject_dir: python/pylibcudf
     extras:
@@ -132,15 +133,16 @@ files:
     includes:
       - build_base
       - build_python_common
-      - build_python_pylibcudf
+      - depends_on_rmm
   py_run_pylibcudf:
     output: pyproject
     pyproject_dir: python/pylibcudf
     extras:
       table: project
     includes:
-      - run_pylibcudf
+      - depends_on_rmm
       - pyarrow_run
+      - run_pylibcudf
   py_test_pylibcudf:
     output: pyproject
     pyproject_dir: python/pylibcudf
@@ -215,14 +217,14 @@ files:
     includes:
       - test_python_common
       - test_python_dask_cudf
-  py_rapids_build_cudf_kafka:
+  py_build_cudf_kafka:
     output: pyproject
     pyproject_dir: python/cudf_kafka
     extras:
       table: build-system
     includes:
       - rapids_build_skbuild
-  py_build_cudf_kafka:
+  py_rapids_build_cudf_kafka:
     output: pyproject
     pyproject_dir: python/cudf_kafka
     extras:
@@ -364,65 +366,6 @@ dependencies:
           # Sync with conda build constraint & wheel run constraint.
           # TODO: Change to `2.0.*` for NumPy 2
           - numpy==1.23.*
-  build_python_pylibcudf:
-    common:
-      - output_types: conda
-        packages:
-          - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0
-      - output_types: requirements
-        packages:
-          # pip recognizes the index as a global option for the requirements.txt file
-          # This index is needed for rmm-cu{11,12}.
-          - --extra-index-url=https://pypi.nvidia.com
-          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
-    specific:
-      - output_types: [requirements, pyproject]
-        matrices:
-          - matrix:
-              cuda: "12.*"
-              cuda_suffixed: "true"
-            packages:
-              - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0
-          - matrix:
-              cuda: "11.*"
-              cuda_suffixed: "true"
-            packages:
-              - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0
-          - {matrix: null, packages: [*rmm_unsuffixed]}
-  build_python_cudf:
-    common:
-      - output_types: conda
-        packages:
-          - *rmm_unsuffixed
-          - pip
-          - pip:
-              - git+https://github.com/python-streamz/streamz.git@master
-      - output_types: requirements
-        packages:
-          # pip recognizes the index as a global option for the requirements.txt file
-          # This index is needed for rmm-cu{11,12}.
-          - --extra-index-url=https://pypi.nvidia.com
-          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
-          - git+https://github.com/python-streamz/streamz.git@master
-    specific:
-      - output_types: [requirements, pyproject]
-        matrices:
-          - matrix:
-              cuda: "12.*"
-              cuda_suffixed: "true"
-            packages:
-              - &pylibcudf_cu12 pylibcudf-cu12==24.10.*,>=0.0.0a0
-              - *rmm_cu12
-          - matrix:
-              cuda: "11.*"
-              cuda_suffixed: "true"
-            packages:
-              - &pylibcudf_cu11 pylibcudf-cu11==24.10.*,>=0.0.0a0
-              - *rmm_cu11
-          - matrix:
-            packages:
-              - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0
-              - *rmm_unsuffixed
   libarrow_build:
     common:
       - output_types: conda
@@ -635,9 +578,6 @@ dependencies:
           - nvtx>=0.2.1
           - packaging
           - typing_extensions>=4.0.0
-      - output_types: conda
-        packages:
-          - *rmm_unsuffixed
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -654,19 +594,6 @@ dependencies:
             packages: &run_pylibcudf_packages_all_cu11
               - cuda-python>=11.7.1,<12.0a0
           - {matrix: null, packages: *run_pylibcudf_packages_all_cu11}
-      - output_types: [requirements, pyproject]
-        matrices:
-          - matrix:
-              cuda: "12.*"
-              cuda_suffixed: "true"
-            packages:
-              - *rmm_cu12
-          - matrix:
-              cuda: "11.*"
-              cuda_suffixed: "true"
-            packages:
-              - *rmm_cu11
-          - {matrix: null, packages: [*rmm_unsuffixed]}
   run_cudf:
     common:
       - output_types: [conda, requirements, pyproject]
@@ -677,9 +604,6 @@ dependencies:
           - packaging
           - rich
           - typing_extensions>=4.0.0
-      - output_types: conda
-        packages:
-          - *rmm_unsuffixed
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -711,19 +635,16 @@ dependencies:
               cuda: "12.*"
               cuda_suffixed: "true"
             packages:
-              - *rmm_cu12
               - pynvjitlink-cu12>=0.0.0a0
           - matrix:
               cuda: "12.*"
               cuda_suffixed: "false"
             packages:
-              - *rmm_unsuffixed
               - *pynvjitlink_unsuffixed
           - matrix:
               cuda: "11.*"
               cuda_suffixed: "true"
             packages:
-              - *rmm_cu11
               - cubinlinker-cu11
               - ptxcompiler-cu11
           - matrix:
@@ -732,7 +653,6 @@ dependencies:
             packages: &run_cudf_cu11_unsuffixed
               - *cubinlinker_unsuffixed
               - *ptxcompiler_unsuffixed
-              - *rmm_unsuffixed
           - {matrix: null, packages: *run_cudf_cu11_unsuffixed}
   run_cudf_polars:
     common:
@@ -843,7 +763,7 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - *pylibcudf_unsuffixed
+          - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
@@ -857,12 +777,12 @@ dependencies:
               cuda: "12.*"
               cuda_suffixed: "true"
             packages:
-              - *pylibcudf_cu12
+              - pylibcudf-cu12==24.10.*,>=0.0.0a0
           - matrix:
               cuda: "11.*"
               cuda_suffixed: "true"
             packages:
-              - *pylibcudf_cu11
+              - pylibcudf-cu11==24.10.*,>=0.0.0a0
           - {matrix: null, packages: [*pylibcudf_unsuffixed]}
   depends_on_cudf:
     common:
@@ -929,6 +849,33 @@ dependencies:
             packages: &cupy_packages_cu11
               - cupy-cuda11x>=12.0.0
           - {matrix: null, packages: *cupy_packages_cu11}
+  depends_on_rmm:
+    common:
+      - output_types: conda
+        packages:
+          - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          # This index is needed for rmm-cu{11,12}.
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
+            packages:
+              - rmm-cu12==24.10.*,>=0.0.0a0
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
+            packages:
+              - rmm-cu11==24.10.*,>=0.0.0a0
+          - matrix:
+            packages:
+              - *rmm_unsuffixed
   test_python_pandas_cudf:
     common:
       - output_types: [requirements, pyproject]