Refactor Morpheus unit tests and plugin to the conda recipe for per-lib testing

.github/workflows/ci_pipe.yml

@@ -29,6 +29,9 @@ on:
         description: 'Runs the conda-build stage to build the morpheus conda packages'
         required: true
         type: boolean
+      base_container:
+        required: true
+        type: string
       container:
         required: true
         type: string
@@ -204,13 +207,14 @@ jobs:
     name: Conda Package
     if: ${{ inputs.conda_run_build }}
     needs: [documentation, test]
-    runs-on: linux-amd64-cpu16
+    runs-on: linux-amd64-gpu-v100-latest-1
     timeout-minutes: 60
     container:
-      credentials:
-        username: '$oauthtoken'
-        password: ${{ secrets.NGC_API_KEY }}
-      image: ${{ inputs.container }}
+      image: ${{ inputs.base_container }}
+      options: --cap-add=sys_nice
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+        PARALLEL_LEVEL: '10'
     strategy:
       fail-fast: true
 

.github/workflows/pr.yaml

@@ -94,6 +94,7 @@ jobs:
       conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }}
       # Upload morpheus conda packages only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
+      base_container: rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.10
       container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-241004
       test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-241004
     secrets:

ci/conda/recipes/morpheus-libs/meta.yaml

@@ -49,8 +49,6 @@ outputs:
         - {{ compiler("cxx") }}
         - ccache
         - cmake =3.27
-        - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application
-        - cuda-version {{ cuda_compiler_version }}.*
         - ninja =1.11
         - pkg-config =0.29
         - sysroot_linux-64 =2.28
@@ -92,6 +90,7 @@ outputs:
         - elasticsearch ==8.9.0
         - feedparser =6.0.*
         - grpcio =1.62.*
+        - lxml
         - mlflow>=2.10.0,<3
         - mrc
         - networkx=2.8.8
@@ -112,21 +111,21 @@ outputs:
         - watchdog =3.0.*
         - websockets
     test:
+      imports:
+        - morpheus
       requires:
-        # test that cuml can be installed in the env
         - cuml {{ rapids_version }}.*
         - pynvml
         - pytest-asyncio
         - pytest-benchmark
         - pytest-cov
         - pytest =7.4.4
       source_files:
+        - dependencies.yaml
         - pyproject.toml
         - scripts/fetch_data.py
-      imports:
-        - morpheus
-      commands:
-          - echo # pytest will be added post re-factoring
+        - tests/*
+      script: morpheus_core_test.sh
 
   ############################### morpheus-dfp #############################
   - name: morpheus-dfp
@@ -145,9 +144,18 @@ outputs:
 
     requirements:
       build:
+        - {{ compiler("c") }}
+        - {{ compiler("cuda") }}
+        - {{ compiler("cxx") }}
         - ccache
         - cmake =3.27
+        - ninja =1.11
+        - pkg-config =0.29
       host:
+        # cuda version is pinned here. without this the resolver pulls
+        # in versions that are not compatible with morpheus-core
+        - cuda-cudart-dev {{ cuda_compiler_version }}.*
+        - cuda-version {{ cuda_compiler_version }}.*
         - {{ pin_subpackage('morpheus-core', exact=True) }}
         - pip
         - python {{ python }}
@@ -156,7 +164,22 @@ outputs:
       run:
         - {{ pin_subpackage('morpheus-core', exact=True) }}
 
-    #test: Tests will be added post test refactoring
+    test:
+      imports:
+        - morpheus_dfp
+      requires:
+        - cuml {{ rapids_version }}.*
+        - pynvml
+        - pytest-asyncio
+        - pytest-benchmark
+        - pytest-cov
+        - pytest =7.4.4
+      source_files:
+        - dependencies.yaml
+        - pyproject.toml
+        - scripts/fetch_data.py
+        - tests/*
+      script: morpheus_dfp_test.sh
 
   ############################### morpheus-llm #############################
   - name: morpheus-llm
@@ -176,14 +199,17 @@ outputs:
     requirements:
       build:
         - {{ compiler("c") }}
+        - {{ compiler("cuda") }}
         - {{ compiler("cxx") }}
         - ccache
         - cmake =3.27
         - ninja =1.11
         - pkg-config =0.29
       host:
-        # morpheus-core has to be at the top. changing that order will result in different
-        # package versions getting installed creating unexpected version conflicts.
+        # cuda version is pinned here. without this the resolver pulls
+        # in versions that are not compatible with morpheus-core
+        - cuda-cudart-dev {{ cuda_compiler_version }}.*
+        - cuda-version {{ cuda_compiler_version }}.*
         - {{ pin_subpackage('morpheus-core', exact=True) }}
         - cython 3.0.*
         - glog >=0.7.1,<0.8
@@ -197,7 +223,22 @@ outputs:
       run:
         - {{ pin_subpackage('morpheus-core', exact=True) }}
 
-    #test: Tests will be added post test refactoring
+    test:
+      imports:
+        - morpheus_llm
+      requires:
+        - cuml {{ rapids_version }}.*
+        - pynvml
+        - pytest-asyncio
+        - pytest-benchmark
+        - pytest-cov
+        - pytest =7.4.4
+      source_files:
+        - dependencies.yaml
+        - pyproject.toml
+        - scripts/fetch_data.py
+        - tests/*
+      script: morpheus_llm_test.sh
 
 about:
   home: https://github.com/nv-morpheus/Morpheus

ci/conda/recipes/morpheus-libs/morpheus_core_test.sh

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# morpheus tests are dependent on some pypi packages
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key morpheus_core --matrix "" | tee "core_requirements.txt"
+
+pip install -r core_requirements.txt
+
+pytest tests/morpheus

ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# morpheus tests are dependent on some pypi packages
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt"
+
+pip install -r dfp_requirements.txt
+
+pytest tests/morpheus_dfp

ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# llm library and tests are dependent on a number of pypi packages - fixme
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt"
+
+pip install -r llm_requirements.txt
+
+pytest tests/morpheus_llm

ci/scripts/github/build.sh

@@ -17,6 +17,7 @@
 set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
+source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh
 source ${WORKSPACE}/ci/scripts/github/cmake_all.sh
 
 rapids-dependency-file-generator \

ci/scripts/github/checks.sh

@@ -17,6 +17,7 @@
 set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
+source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh
 source ${WORKSPACE}/ci/scripts/github/cmake_all.sh
 
 rapids-dependency-file-generator \

ci/scripts/github/common.sh

@@ -26,7 +26,6 @@ mkdir -p ${WORKSPACE_TMP}
 source /opt/conda/etc/profile.d/conda.sh
 export MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)}
 cd ${MORPHEUS_ROOT}
-conda activate morpheus
 
 # For non-gpu hosts nproc will correctly report the number of cores we are able to use
 # On a GPU host however nproc will report the total number of cores and PARALLEL_LEVEL

ci/scripts/github/conda.sh

@@ -18,6 +18,7 @@ set -e
 
 CI_SCRIPT_ARGS="$@"
 source ${WORKSPACE}/ci/scripts/github/common.sh
+source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh
 source ${WORKSPACE}/ci/scripts/github/cmake_all.sh
 
 cd ${MORPHEUS_ROOT}

ci/scripts/github/conda_libs.sh

@@ -17,32 +17,36 @@
 set -e
 
 CI_SCRIPT_ARGS="$@"
-source ${WORKSPACE}/ci/scripts/github/common.sh
-
-cd ${MORPHEUS_ROOT}
 
-fetch_base_branch
+source ${WORKSPACE}/ci/scripts/github/common.sh
 
 # Its important that we are in the base environment for the build
 rapids-logger "Activating Base Conda Environment"
+conda activate base
 
-# Deactivate any extra environments (There can be a few on the stack)
-while [[ "${CONDA_SHLVL:-0}" -gt 1 ]]; do
-   echo "Deactivating conda environment ${CONDA_DEFAULT_ENV}"
-   conda deactivate
-done
+cd ${MORPHEUS_ROOT}
 
-# Ensure at least base is activated
-if [[ "${CONDA_DEFAULT_ENV}" != "base" ]]; then
-   echo "Activating base conda environment"
-   conda activate base
-fi
+fetch_base_branch
 
 # Print the info just to be sure base is active
 conda info
 
+# Pull down data needed for running the per-library unit tests
+rapids-logger "Pulling LFS assets"
+
+conda install git-lfs
+git lfs install
+${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation
+
+# Listing LFS-known files
+rapids-logger "Listing LFS-known files"
+
+git lfs ls-files
 rapids-logger "Building Morpheus Libraries"
 
+# Run nvidia-smi to check the test env
+/usr/bin/nvidia-smi
+
 # Run the conda build, and upload to conda forge if requested
 export MORPHEUS_PYTHON_BUILD_STUBS=OFF
 export CONDA_ARGS="--skip-existing"

ci/scripts/github/docs.sh

@@ -17,6 +17,7 @@
 set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
+source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh
 source ${WORKSPACE}/ci/scripts/github/cmake_all.sh
 
 rapids-dependency-file-generator \

ci/scripts/github/morpheus_env.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+conda activate morpheus

ci/scripts/github/test.sh

@@ -17,6 +17,7 @@
 set -e
 
 source ${WORKSPACE}/ci/scripts/github/common.sh
+source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh
 source ${WORKSPACE}/ci/scripts/github/cmake_all.sh
 /usr/bin/nvidia-smi
 

ci/scripts/run_ci_local.sh

@@ -59,9 +59,12 @@ GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp}
 CONTAINER_VER=${CONTAINER_VER:-241004}
-CUDA_VER=${CUDA_VER:-12.1}
+CUDA_VER=${CUDA_VER:-12.5}
+CUDA_FULL_VER=${CUDA_FULL_VER:-12.5.1}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
+# Configure the base docker img
+CONDA_CONTAINER="rapidsai/ci-conda:cuda${CUDA_FULL_VER}-ubuntu22.04-py3.10"
 BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}"
 TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}"
 
@@ -88,13 +91,18 @@ for STAGE in "${STAGES[@]}"; do
     DOCKER_RUN_ARGS+=("-v" "${LOCAL_CI_TMP}:/ci_tmp")
     DOCKER_RUN_ARGS+=("${ENV_LIST[@]}")
     DOCKER_RUN_ARGS+=("--env STAGE=${STAGE}")
-    if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then
+    if [[ "${STAGE}" == "conda_libs" || "${USE_BASE}" == "1" ]]; then
+        CONTAINER="${CONDA_CONTAINER}"
+    elif [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then
         CONTAINER="${TEST_CONTAINER}"
+    else
+        CONTAINER="${BUILD_CONTAINER}"
+    fi
+    if [[ "${STAGE}" == "test" ||  "${STAGE}" == "conda_libs" || "${USE_GPU}" == "1" ]]; then
         DOCKER_RUN_ARGS+=("--runtime=nvidia")
         DOCKER_RUN_ARGS+=("--gpus all")
         DOCKER_RUN_ARGS+=("--cap-add=sys_nice")
     else
-        CONTAINER="${BUILD_CONTAINER}"
         DOCKER_RUN_ARGS+=("--runtime=runc")
     fi