From 01934d3ad5ecbab0741c95868e916c56d6c88d9a Mon Sep 17 00:00:00 2001
From: Anuradha Karuppiah <anuradhak@nvidia.com>
Date: Fri, 18 Oct 2024 22:01:21 +0000
Subject: [PATCH] Generate the requirements files from dependencies.yaml

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>
---
 .../morpheus-libs/morpheus_core_test.sh       | 13 ++++----
 .../morpheus-libs/morpheus_dfp_test.sh        | 13 ++++----
 .../morpheus-libs/morpheus_llm_test.sh        | 16 ++--------
 .../all_cuda-125_arch-x86_64.yaml             |  2 +-
 .../examples_cuda-125_arch-x86_64.yaml        |  2 +-
 dependencies.yaml                             | 31 ++++++++++---------
 python/morpheus/MANIFEST.in                   |  1 +
 .../morpheus/requirements_morpheus_core.txt   |  4 +++
 python/morpheus_dfp/MANIFEST.in               |  1 +
 .../requirements_morpheus_dfp.txt             |  4 +++
 python/morpheus_llm/MANIFEST.in               |  2 +-
 .../morpheus_llm/requirements.txt             |  6 ----
 .../requirements_morpheus_llm.txt             | 10 ++++++
 13 files changed, 57 insertions(+), 48 deletions(-)
 create mode 100644 python/morpheus/morpheus/requirements_morpheus_core.txt
 create mode 100644 python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt
 delete mode 100644 python/morpheus_llm/morpheus_llm/requirements.txt
 create mode 100644 python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt

diff --git a/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh
index d7379cda4c..cafcb7d5d7 100644
--- a/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh
+++ b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh
@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_core --matrix "" | tee "core_requirements.txt"
-
-pip install -r core_requirements.txt
+python3 <<EOF
+import pkgutil
+import subprocess
+data = pkgutil.get_data("morpheus", "requirements_morpheus_core.txt")
+requirements = data.decode("utf-8")
+subprocess.call(f"pip install {requirements}".split())
+EOF
 
 pytest tests/morpheus
diff --git a/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh
index 2c38db4cb3..8964ffdd24 100644
--- a/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh
+++ b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh
@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt"
-
-pip install -r dfp_requirements.txt
+python3 <<EOF
+import pkgutil
+import subprocess
+data = pkgutil.get_data("morpheus_dfp", "requirements_morpheus_dfp.txt")
+requirements = data.decode("utf-8")
+subprocess.call(f"pip install {requirements}".split())
+EOF
 
 pytest tests/morpheus_dfp
diff --git a/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh
index 20016e00cc..285b539ebb 100644
--- a/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh
+++ b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh
@@ -13,23 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# llm library and tests are dependent on a number of pypi packages - fixme
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt"
-
-pip install -r llm_requirements.txt
-
 # Install requirements if they are included in the package
 python3 <<EOF
 import pkgutil
 import subprocess
-try:
-    data = pkgutil.get_data("morpheus_llm", "requirements.txt")
-    requirements = data.decode("utf-8")
-    subprocess.call(f"pip install {requirements}".split())
-except FileNotFoundError:
-    pass
+data = pkgutil.get_data("morpheus_llm", "requirements_morpheus_llm.txt")
+requirements = data.decode("utf-8")
+subprocess.call(f"pip install {requirements}".split())
 EOF
 
 pytest tests/morpheus_llm
diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml
index 0281cf6b68..bcecbd4b3d 100644
--- a/conda/environments/all_cuda-125_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-125_arch-x86_64.yaml
@@ -43,6 +43,7 @@ dependencies:
 - doxygen=1.9.2
 - elasticsearch==8.9.0
 - exhale=0.3.6
+- faiss-cpu
 - feedparser=6.0
 - flake8
 - git-lfs
@@ -132,7 +133,6 @@ dependencies:
   - databricks-connect
   - dgl==2.0.0
   - dglgo
-  - faiss-cpu
   - google-search-results==2.4
   - langchain-nvidia-ai-endpoints==0.0.11
   - langchain==0.1.16
diff --git a/conda/environments/examples_cuda-125_arch-x86_64.yaml b/conda/environments/examples_cuda-125_arch-x86_64.yaml
index e387e2c9bf..c46c8e86fa 100644
--- a/conda/environments/examples_cuda-125_arch-x86_64.yaml
+++ b/conda/environments/examples_cuda-125_arch-x86_64.yaml
@@ -22,6 +22,7 @@ dependencies:
 - dill=0.3.7
 - docker-py=5.0
 - elasticsearch==8.9.0
+- faiss-cpu
 - feedparser=6.0
 - grpcio
 - grpcio-status
@@ -71,7 +72,6 @@ dependencies:
   - databricks-connect
   - dgl==2.0.0
   - dglgo
-  - faiss-cpu
   - google-search-results==2.4
   - langchain-nvidia-ai-endpoints==0.0.11
   - langchain==0.1.16
diff --git a/dependencies.yaml b/dependencies.yaml
index c43709e265..d90cc71cc5 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -176,23 +176,27 @@ files:
       - checks
       - python
 
-  # pip dependencies that are used during the conda test stage for morpheus-llm output
-  morpheus_llm:
-    output: none
-    includes:
-      - test_morpheus_core_pip
-
-  # pip dependencies that are used during the conda test stage for morpheus-core output
+  # morpheus-core pip dependencies are published as a morpheus package data file
   morpheus_core:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus/morpheus
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
 
-  # pip dependencies that are used during the conda test stage for morpheus-dfp output
+  # morpheus-dfp pip dependencies are published as a morpheus_dfp package data file
   morpheus_dfp:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus_dfp/morpheus_dfp
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
+
+  # morpheus-llm pip dependencies are published as a morpheus_llm package data file
+  morpheus_llm:
+    output: requirements
+    requirements_dir: python/morpheus_llm/morpheus_llm
+    includes:
+      - morpheus_core_pip
+      - morpheus_llm_pip
 
 channels:
   - conda-forge
@@ -401,14 +405,13 @@ dependencies:
           - pip:
               - pytest-kafka==0.6.0
 
-  test_morpheus_core_pip:
+  morpheus_core_pip:
     common:
       - output_types: [requirements]
         packages:
            - --extra-index-url https://download.pytorch.org/whl/cu124
            - torch==2.4.0+cu124
 
-
   morpheus_llm_pip:
     common:
       - output_types: [requirements]
diff --git a/python/morpheus/MANIFEST.in b/python/morpheus/MANIFEST.in
index 4503f2bcc4..5db675c35a 100644
--- a/python/morpheus/MANIFEST.in
+++ b/python/morpheus/MANIFEST.in
@@ -1,3 +1,4 @@
 include morpheus/_version.py
+include morpheus/requirements_morpheus_core.txt
 recursive-include morpheus/data *
 recursive-include morpheus *.cpython*.so py.typed *.pyi
diff --git a/python/morpheus/morpheus/requirements_morpheus_core.txt b/python/morpheus/morpheus/requirements_morpheus_core.txt
new file mode 100644
index 0000000000..32cdfbe1ac
--- /dev/null
+++ b/python/morpheus/morpheus/requirements_morpheus_core.txt
@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124
diff --git a/python/morpheus_dfp/MANIFEST.in b/python/morpheus_dfp/MANIFEST.in
index 807dc3f1b0..3911d61b04 100644
--- a/python/morpheus_dfp/MANIFEST.in
+++ b/python/morpheus_dfp/MANIFEST.in
@@ -1,2 +1,3 @@
 include morpheus_dfp/_version.py
+include morpheus_dfp/requirements_morpheus_dfp.txt
 recursive-include morpheus_dfp py.typed
diff --git a/python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt b/python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt
new file mode 100644
index 0000000000..32cdfbe1ac
--- /dev/null
+++ b/python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt
@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124
diff --git a/python/morpheus_llm/MANIFEST.in b/python/morpheus_llm/MANIFEST.in
index b57a645f08..0eef4f98a2 100644
--- a/python/morpheus_llm/MANIFEST.in
+++ b/python/morpheus_llm/MANIFEST.in
@@ -1,3 +1,3 @@
 include morpheus_llm/_version.py
-include morpheus_llm/requirements.txt
+include morpheus_llm/requirements_morpheus_llm.txt
 recursive-include morpheus_llm *.cpython*.so py.typed *.pyi
diff --git a/python/morpheus_llm/morpheus_llm/requirements.txt b/python/morpheus_llm/morpheus_llm/requirements.txt
deleted file mode 100644
index 7a47661239..0000000000
--- a/python/morpheus_llm/morpheus_llm/requirements.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-milvus==2.3.5
-pymilvus==2.3.6
-langchain==0.1.16
-langchain-nvidia-ai-endpoints==0.0.11
-google-search-results==2.4
-nemollm==0.3.5
diff --git a/python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt b/python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt
new file mode 100644
index 0000000000..880c725608
--- /dev/null
+++ b/python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt
@@ -0,0 +1,10 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+google-search-results==2.4
+langchain-nvidia-ai-endpoints==0.0.11
+langchain==0.1.16
+milvus==2.3.5
+nemollm==0.3.5
+pymilvus==2.3.6
+torch==2.4.0+cu124