Include requirements file in the morpheus_llm package for pip depende…

…ncies

Signed-off-by: Anuradha Karuppiah <[email protected]>

ci/conda/recipes/morpheus-libs/meta.yaml

@@ -88,6 +88,7 @@ outputs:
         - dill =0.3.7
         - docker-py =5.0.*
         - elasticsearch ==8.9.0
+        - faiss-cpu
         - feedparser =6.0.*
         - grpcio =1.62.*
         - lxml

ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh

@@ -20,4 +20,16 @@ rapids-dependency-file-generator \
 
 pip install -r llm_requirements.txt
 
+# Install requirements if they are included in the package
+python3 <<EOF
+import pkgutil
+import subprocess
+try:
+    data = pkgutil.get_data("morpheus_llm", "requirements.txt")
+    requirements = data.decode("utf-8")
+    subprocess.call(f"pip install {requirements}".split())
+except FileNotFoundError:
+    pass
+EOF
+
 pytest tests/morpheus_llm

dependencies.yaml

@@ -179,7 +179,6 @@ files:
   morpheus_llm:
     output: none
     includes:
-      - test_morpheus_llm_pip
       - test_morpheus_core_pip
 
   # pip dependencies that are used during the conda test stage for morpheus-core output
@@ -408,17 +407,17 @@ dependencies:
            - --extra-index-url https://download.pytorch.org/whl/cu124
            - torch==2.4.0+cu124
 
-  test_morpheus_llm_pip:
+
+  morpheus_llm_pip:
     common:
       - output_types: [requirements]
         packages:
            - milvus==2.3.5 # update to match pymilvus when available
            - pymilvus==2.3.6
            - langchain==0.1.16
            - langchain-nvidia-ai-endpoints==0.0.11
-           - faiss-cpu
            - google-search-results==2.4
-            - nemollm==0.3.5
+           - nemollm==0.3.5
 
   example-dfp-prod:
     common:
@@ -449,6 +448,7 @@ dependencies:
           - &transformers transformers=4.36.2 # newer versions are incompatible with our pinned version of huggingface_hub
           - anyio>=3.7
           - arxiv=1.4
+          - faiss-cpu
           - huggingface_hub=0.20.2 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762
           - jsonpatch>=1.33
           - newspaper3k=0.2
@@ -464,7 +464,6 @@ dependencies:
           - pip:
             - langchain==0.1.16
             - langchain-nvidia-ai-endpoints==0.0.11
-            - faiss-cpu
             - google-search-results==2.4
             - nemollm==0.3.5
 

python/morpheus_llm/MANIFEST.in

@@ -1,2 +1,3 @@
 include morpheus_llm/_version.py
+include morpheus_llm/requirements.txt
 recursive-include morpheus_llm *.cpython*.so py.typed *.pyi

python/morpheus_llm/morpheus_llm/requirements.txt

@@ -0,0 +1,6 @@
+milvus==2.3.5
+pymilvus==2.3.6
+langchain==0.1.16
+langchain-nvidia-ai-endpoints==0.0.11
+google-search-results==2.4
+nemollm==0.3.5