From e648a5b6f2043506ecd2e9ab0bbb48bb354ea63f Mon Sep 17 00:00:00 2001
From: Anuradha Karuppiah <anuradhak@nvidia.com>
Date: Sat, 19 Oct 2024 16:24:54 -0700
Subject: [PATCH] Include requirements files in the morpheus packages (#1957)

pip dependencies are included as a part of the package -
morpheus/requirements_morpheus_core.txt
morpheus_dfp/requirements_morpheus_dfp.txt
morpheus_llm/requirements_morpheus_llm.txt

And can be installed as needed. For example -
```
import importlib.resources
import subprocess
requirements_file = importlib.resources.path("morpheus_llm", "requirements_morpheus_llm.txt")
subprocess.call(f"pip install -r {requirements_file}".split())
```

Authors:
  - Anuradha Karuppiah (https://github.com/AnuradhaKaruppiah)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: https://github.com/nv-morpheus/Morpheus/pull/1957
---
 .../morpheus-libs/morpheus_core_test.sh       | 12 +++---
 .../morpheus-libs/morpheus_dfp_test.sh        | 12 +++---
 .../morpheus-libs/morpheus_llm_test.sh        | 13 +++---
 dependencies.yaml                             | 43 ++++++++++---------
 python/morpheus/CMakeLists.txt                |  3 ++
 python/morpheus/MANIFEST.in                   |  1 +
 .../morpheus/requirements_morpheus_core.txt   |  4 ++
 python/morpheus_dfp/CMakeLists.txt            |  3 ++
 python/morpheus_dfp/MANIFEST.in               |  1 +
 .../requirements_morpheus_dfp.txt             |  4 ++
 python/morpheus_llm/CMakeLists.txt            |  3 ++
 python/morpheus_llm/MANIFEST.in               |  1 +
 .../requirements_morpheus_llm.txt             | 11 +++++
 13 files changed, 73 insertions(+), 38 deletions(-)
 create mode 100644 python/morpheus/morpheus/requirements_morpheus_core.txt
 create mode 100644 python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt
 create mode 100644 python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt

diff --git a/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh
index d7379cda4c..24a29f161f 100644
--- a/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh
+++ b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh
@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_core --matrix "" | tee "core_requirements.txt"
-
-pip install -r core_requirements.txt
+python3 <<EOF
+import importlib.resources
+import subprocess
+requirements_file = importlib.resources.path("morpheus", "requirements_morpheus_core.txt")
+subprocess.call(f"pip install -r {requirements_file}".split())
+EOF
 
 pytest tests/morpheus
diff --git a/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh
index 2c38db4cb3..74d60970b9 100644
--- a/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh
+++ b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh
@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt"
-
-pip install -r dfp_requirements.txt
+python3 <<EOF
+import importlib.resources
+import subprocess
+requirements_file = importlib.resources.path("morpheus_dfp", "requirements_morpheus_dfp.txt")
+subprocess.call(f"pip install -r {requirements_file}".split())
+EOF
 
 pytest tests/morpheus_dfp
diff --git a/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh
index 67e7b2f52c..37722d5a9b 100644
--- a/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh
+++ b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh
@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# llm library and tests are dependent on a number of pypi packages - fixme
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt"
-
-pip install -r llm_requirements.txt
+# Install requirements if they are included in the package
+python3 <<EOF
+import importlib.resources
+import subprocess
+requirements_file = importlib.resources.path("morpheus_llm", "requirements_morpheus_llm.txt")
+subprocess.call(f"pip install -r {requirements_file}".split())
+EOF
 
 pytest tests/morpheus_llm
diff --git a/dependencies.yaml b/dependencies.yaml
index 05393f209c..53ed12a9ab 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -176,24 +176,27 @@ files:
       - checks
       - python
 
-  # pip dependencies that are used during the conda test stage for morpheus-llm output
-  morpheus_llm:
-    output: none
-    includes:
-      - test_morpheus_llm_pip
-      - test_morpheus_core_pip
-
-  # pip dependencies that are used during the conda test stage for morpheus-core output
+  # morpheus-core pip dependencies are published as a morpheus package data file
   morpheus_core:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus/morpheus
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
 
-  # pip dependencies that are used during the conda test stage for morpheus-dfp output
+  # morpheus-dfp pip dependencies are published as a morpheus_dfp package data file
   morpheus_dfp:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus_dfp/morpheus_dfp
+    includes:
+      - morpheus_core_pip
+
+  # morpheus-llm pip dependencies are published as a morpheus_llm package data file
+  morpheus_llm:
+    output: requirements
+    requirements_dir: python/morpheus_llm/morpheus_llm
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
+      - morpheus_llm_pip
 
 channels:
   - conda-forge
@@ -402,24 +405,24 @@ dependencies:
           - pip:
               - pytest-kafka==0.6.0
 
-  test_morpheus_core_pip:
+  morpheus_core_pip:
     common:
       - output_types: [requirements]
         packages:
            - --extra-index-url https://download.pytorch.org/whl/cu124
            - torch==2.4.0+cu124
 
-  test_morpheus_llm_pip:
+  morpheus_llm_pip:
     common:
       - output_types: [requirements]
         packages:
-           - milvus==2.3.5 # update to match pymilvus when available
-           - pymilvus==2.3.6
-           - langchain==0.1.16
-           - langchain-nvidia-ai-endpoints==0.0.11
            - faiss-cpu
            - google-search-results==2.4
-            - nemollm==0.3.5
+           - langchain==0.1.16
+           - langchain-nvidia-ai-endpoints==0.0.11
+           - milvus==2.3.5 # update to match pymilvus when available
+           - pymilvus==2.3.6
+           - nemollm==0.3.5
 
   example-dfp-prod:
     common:
diff --git a/python/morpheus/CMakeLists.txt b/python/morpheus/CMakeLists.txt
index d410203477..a6211724c8 100644
--- a/python/morpheus/CMakeLists.txt
+++ b/python/morpheus/CMakeLists.txt
@@ -50,6 +50,9 @@ endif()
 file(GLOB morpheus_data_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus/data/*")
 morpheus_utils_add_python_sources(${morpheus_data_files})
 
+file(GLOB morpheus_requirement_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus/requirements*")
+morpheus_utils_add_python_sources(${morpheus_requirement_files})
+
 morpheus_utils_build_python_package(morpheus ${extra_args})
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)
diff --git a/python/morpheus/MANIFEST.in b/python/morpheus/MANIFEST.in
index 4503f2bcc4..5db675c35a 100644
--- a/python/morpheus/MANIFEST.in
+++ b/python/morpheus/MANIFEST.in
@@ -1,3 +1,4 @@
 include morpheus/_version.py
+include morpheus/requirements_morpheus_core.txt
 recursive-include morpheus/data *
 recursive-include morpheus *.cpython*.so py.typed *.pyi
diff --git a/python/morpheus/morpheus/requirements_morpheus_core.txt b/python/morpheus/morpheus/requirements_morpheus_core.txt
new file mode 100644
index 0000000000..32cdfbe1ac
--- /dev/null
+++ b/python/morpheus/morpheus/requirements_morpheus_core.txt
@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124
diff --git a/python/morpheus_dfp/CMakeLists.txt b/python/morpheus_dfp/CMakeLists.txt
index fbb1afe6ad..27821ea2d9 100644
--- a/python/morpheus_dfp/CMakeLists.txt
+++ b/python/morpheus_dfp/CMakeLists.txt
@@ -41,6 +41,9 @@ if(MORPHEUS_PYTHON_PERFORM_INSTALL)
   list(APPEND extra_args "INSTALL_WHEEL")
 endif()
 
+file(GLOB morpheus_dfp_requirement_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus_dfp/requirements*")
+morpheus_utils_add_python_sources(${morpheus_dfp_requirement_files})
+
 morpheus_utils_build_python_package(morpheus_dfp ${extra_args})
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)
diff --git a/python/morpheus_dfp/MANIFEST.in b/python/morpheus_dfp/MANIFEST.in
index 807dc3f1b0..3911d61b04 100644
--- a/python/morpheus_dfp/MANIFEST.in
+++ b/python/morpheus_dfp/MANIFEST.in
@@ -1,2 +1,3 @@
 include morpheus_dfp/_version.py
+include morpheus_dfp/requirements_morpheus_dfp.txt
 recursive-include morpheus_dfp py.typed
diff --git a/python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt b/python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt
new file mode 100644
index 0000000000..32cdfbe1ac
--- /dev/null
+++ b/python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt
@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124
diff --git a/python/morpheus_llm/CMakeLists.txt b/python/morpheus_llm/CMakeLists.txt
index d2093e759c..cae37d8fa3 100644
--- a/python/morpheus_llm/CMakeLists.txt
+++ b/python/morpheus_llm/CMakeLists.txt
@@ -45,6 +45,9 @@ if(MORPHEUS_PYTHON_PERFORM_INSTALL)
   list(APPEND extra_args "INSTALL_WHEEL")
 endif()
 
+file(GLOB morpheus_llm_requirement_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus_llm/requirements*")
+morpheus_utils_add_python_sources(${morpheus_llm_requirement_files})
+
 morpheus_utils_build_python_package(morpheus_llm ${extra_args})
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)
diff --git a/python/morpheus_llm/MANIFEST.in b/python/morpheus_llm/MANIFEST.in
index 457e409317..0eef4f98a2 100644
--- a/python/morpheus_llm/MANIFEST.in
+++ b/python/morpheus_llm/MANIFEST.in
@@ -1,2 +1,3 @@
 include morpheus_llm/_version.py
+include morpheus_llm/requirements_morpheus_llm.txt
 recursive-include morpheus_llm *.cpython*.so py.typed *.pyi
diff --git a/python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt b/python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt
new file mode 100644
index 0000000000..8f9a9620b9
--- /dev/null
+++ b/python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt
@@ -0,0 +1,11 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+faiss-cpu
+google-search-results==2.4
+langchain-nvidia-ai-endpoints==0.0.11
+langchain==0.1.16
+milvus==2.3.5
+nemollm==0.3.5
+pymilvus==2.3.6
+torch==2.4.0+cu124