diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml index 0d4bb14c7d..4226827473 100644 --- a/.github/workflows/ci_pipe.yml +++ b/.github/workflows/ci_pipe.yml @@ -29,6 +29,9 @@ on: description: 'Runs the conda-build stage to build the morpheus conda packages' required: true type: boolean + base_container: + required: true + type: string container: required: true type: string @@ -204,13 +207,14 @@ jobs: name: Conda Package if: ${{ inputs.conda_run_build }} needs: [documentation, test] - runs-on: linux-amd64-cpu16 + runs-on: linux-amd64-gpu-v100-latest-1 timeout-minutes: 60 container: - credentials: - username: '$oauthtoken' - password: ${{ secrets.NGC_API_KEY }} - image: ${{ inputs.container }} + image: ${{ inputs.base_container }} + options: --cap-add=sys_nice + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + PARALLEL_LEVEL: '10' strategy: fail-fast: true diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1afe90625c..aea0e918ac 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -94,6 +94,7 @@ jobs: conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }} # Upload morpheus conda packages only for non PR branches. Use 'main' for main branch and 'dev' for all other branches conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }} + base_container: rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.10 container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-241004 test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-241004 secrets: diff --git a/ci/conda/recipes/morpheus-libs/meta.yaml b/ci/conda/recipes/morpheus-libs/meta.yaml index dffdffe53e..747f734371 100644 --- a/ci/conda/recipes/morpheus-libs/meta.yaml +++ b/ci/conda/recipes/morpheus-libs/meta.yaml @@ -49,8 +49,6 @@ outputs: - {{ compiler("cxx") }} - ccache - cmake =3.27 - - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application - - cuda-version {{ cuda_compiler_version }}.* - ninja =1.11 - pkg-config =0.29 - sysroot_linux-64 =2.28 @@ -92,6 +90,7 @@ outputs: - elasticsearch ==8.9.0 - feedparser =6.0.* - grpcio =1.62.* + - lxml - mlflow>=2.10.0,<3 - mrc - networkx=2.8.8 @@ -112,8 +111,9 @@ outputs: - watchdog =3.0.* - websockets test: + imports: + - morpheus requires: - # test that cuml can be installed in the env - cuml {{ rapids_version }}.* - pynvml - pytest-asyncio @@ -121,12 +121,11 @@ outputs: - pytest-cov - pytest =7.4.4 source_files: + - dependencies.yaml - pyproject.toml - scripts/fetch_data.py - imports: - - morpheus - commands: - - echo # pytest will be added post re-factoring + - tests/* + script: morpheus_core_test.sh ############################### morpheus-dfp ############################# - name: morpheus-dfp @@ -145,9 +144,18 @@ outputs: requirements: build: + - {{ compiler("c") }} + - {{ compiler("cuda") }} + - {{ compiler("cxx") }} - ccache - cmake =3.27 + - ninja =1.11 + - pkg-config =0.29 host: + # cuda version is pinned here. without this the resolver pulls + # in versions that are not compatible with morpheus-core + - cuda-cudart-dev {{ cuda_compiler_version }}.* + - cuda-version {{ cuda_compiler_version }}.* - {{ pin_subpackage('morpheus-core', exact=True) }} - pip - python {{ python }} @@ -156,7 +164,22 @@ outputs: run: - {{ pin_subpackage('morpheus-core', exact=True) }} - #test: Tests will be added post test refactoring + test: + imports: + - morpheus_dfp + requires: + - cuml {{ rapids_version }}.* + - pynvml + - pytest-asyncio + - pytest-benchmark + - pytest-cov + - pytest =7.4.4 + source_files: + - dependencies.yaml + - pyproject.toml + - scripts/fetch_data.py + - tests/* + script: morpheus_dfp_test.sh ############################### morpheus-llm ############################# - name: morpheus-llm @@ -176,14 +199,17 @@ outputs: requirements: build: - {{ compiler("c") }} + - {{ compiler("cuda") }} - {{ compiler("cxx") }} - ccache - cmake =3.27 - ninja =1.11 - pkg-config =0.29 host: - # morpheus-core has to be at the top. changing that order will result in different - # package versions getting installed creating unexpected version conflicts. + # cuda version is pinned here. without this the resolver pulls + # in versions that are not compatible with morpheus-core + - cuda-cudart-dev {{ cuda_compiler_version }}.* + - cuda-version {{ cuda_compiler_version }}.* - {{ pin_subpackage('morpheus-core', exact=True) }} - cython 3.0.* - glog >=0.7.1,<0.8 @@ -197,7 +223,22 @@ outputs: run: - {{ pin_subpackage('morpheus-core', exact=True) }} - #test: Tests will be added post test refactoring + test: + imports: + - morpheus_llm + requires: + - cuml {{ rapids_version }}.* + - pynvml + - pytest-asyncio + - pytest-benchmark + - pytest-cov + - pytest =7.4.4 + source_files: + - dependencies.yaml + - pyproject.toml + - scripts/fetch_data.py + - tests/* + script: morpheus_llm_test.sh about: home: https://github.com/nv-morpheus/Morpheus diff --git a/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh new file mode 100644 index 0000000000..d7379cda4c --- /dev/null +++ b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# morpheus tests are dependent on some pypi packages +rapids-dependency-file-generator \ + --output requirements \ + --file-key morpheus_core --matrix "" | tee "core_requirements.txt" + +pip install -r core_requirements.txt + +pytest tests/morpheus diff --git a/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh new file mode 100644 index 0000000000..2c38db4cb3 --- /dev/null +++ b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# morpheus tests are dependent on some pypi packages +rapids-dependency-file-generator \ + --output requirements \ + --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt" + +pip install -r dfp_requirements.txt + +pytest tests/morpheus_dfp diff --git a/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh new file mode 100644 index 0000000000..67e7b2f52c --- /dev/null +++ b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# llm library and tests are dependent on a number of pypi packages - fixme +rapids-dependency-file-generator \ + --output requirements \ + --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt" + +pip install -r llm_requirements.txt + +pytest tests/morpheus_llm diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh index 8f75893967..7060b064db 100755 --- a/ci/scripts/github/build.sh +++ b/ci/scripts/github/build.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh source ${WORKSPACE}/ci/scripts/github/cmake_all.sh rapids-dependency-file-generator \ diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index a00afe718f..020bab2c0d 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh source ${WORKSPACE}/ci/scripts/github/cmake_all.sh rapids-dependency-file-generator \ diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 5d7cf3f698..21147c1ccc 100755 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -26,7 +26,6 @@ mkdir -p ${WORKSPACE_TMP} source /opt/conda/etc/profile.d/conda.sh export MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)} cd ${MORPHEUS_ROOT} -conda activate morpheus # For non-gpu hosts nproc will correctly report the number of cores we are able to use # On a GPU host however nproc will report the total number of cores and PARALLEL_LEVEL diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh index 5be051fade..f0cc041a08 100755 --- a/ci/scripts/github/conda.sh +++ b/ci/scripts/github/conda.sh @@ -18,6 +18,7 @@ set -e CI_SCRIPT_ARGS="$@" source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh source ${WORKSPACE}/ci/scripts/github/cmake_all.sh cd ${MORPHEUS_ROOT} diff --git a/ci/scripts/github/conda_libs.sh b/ci/scripts/github/conda_libs.sh index 7969a30c83..467138a950 100755 --- a/ci/scripts/github/conda_libs.sh +++ b/ci/scripts/github/conda_libs.sh @@ -17,32 +17,36 @@ set -e CI_SCRIPT_ARGS="$@" -source ${WORKSPACE}/ci/scripts/github/common.sh - -cd ${MORPHEUS_ROOT} -fetch_base_branch +source ${WORKSPACE}/ci/scripts/github/common.sh # Its important that we are in the base environment for the build rapids-logger "Activating Base Conda Environment" +conda activate base -# Deactivate any extra environments (There can be a few on the stack) -while [[ "${CONDA_SHLVL:-0}" -gt 1 ]]; do - echo "Deactivating conda environment ${CONDA_DEFAULT_ENV}" - conda deactivate -done +cd ${MORPHEUS_ROOT} -# Ensure at least base is activated -if [[ "${CONDA_DEFAULT_ENV}" != "base" ]]; then - echo "Activating base conda environment" - conda activate base -fi +fetch_base_branch # Print the info just to be sure base is active conda info +# Pull down data needed for running the per-library unit tests +rapids-logger "Pulling LFS assets" + +conda install git-lfs +git lfs install +${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation + +# Listing LFS-known files +rapids-logger "Listing LFS-known files" + +git lfs ls-files rapids-logger "Building Morpheus Libraries" +# Run nvidia-smi to check the test env +/usr/bin/nvidia-smi + # Run the conda build, and upload to conda forge if requested export MORPHEUS_PYTHON_BUILD_STUBS=OFF export CONDA_ARGS="--skip-existing" diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh index 441c8495b0..bec4da24fb 100755 --- a/ci/scripts/github/docs.sh +++ b/ci/scripts/github/docs.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh source ${WORKSPACE}/ci/scripts/github/cmake_all.sh rapids-dependency-file-generator \ diff --git a/ci/scripts/github/morpheus_env.sh b/ci/scripts/github/morpheus_env.sh new file mode 100755 index 0000000000..4f667db06a --- /dev/null +++ b/ci/scripts/github/morpheus_env.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +conda activate morpheus diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh index 2e3388e611..c26b70011b 100755 --- a/ci/scripts/github/test.sh +++ b/ci/scripts/github/test.sh @@ -17,6 +17,7 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh +source ${WORKSPACE}/ci/scripts/github/morpheus_env.sh source ${WORKSPACE}/ci/scripts/github/cmake_all.sh /usr/bin/nvidia-smi diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 4a2bbe3f23..d84c096a0c 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -59,9 +59,12 @@ GIT_COMMIT=$(git log -n 1 --pretty=format:%H) LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp} CONTAINER_VER=${CONTAINER_VER:-241004} -CUDA_VER=${CUDA_VER:-12.1} +CUDA_VER=${CUDA_VER:-12.5} +CUDA_FULL_VER=${CUDA_FULL_VER:-12.5.1} DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} +# Configure the base docker img +CONDA_CONTAINER="rapidsai/ci-conda:cuda${CUDA_FULL_VER}-ubuntu22.04-py3.10" BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}" TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}" @@ -88,13 +91,18 @@ for STAGE in "${STAGES[@]}"; do DOCKER_RUN_ARGS+=("-v" "${LOCAL_CI_TMP}:/ci_tmp") DOCKER_RUN_ARGS+=("${ENV_LIST[@]}") DOCKER_RUN_ARGS+=("--env STAGE=${STAGE}") - if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then + if [[ "${STAGE}" == "conda_libs" || "${USE_BASE}" == "1" ]]; then + CONTAINER="${CONDA_CONTAINER}" + elif [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then CONTAINER="${TEST_CONTAINER}" + else + CONTAINER="${BUILD_CONTAINER}" + fi + if [[ "${STAGE}" == "test" || "${STAGE}" == "conda_libs" || "${USE_GPU}" == "1" ]]; then DOCKER_RUN_ARGS+=("--runtime=nvidia") DOCKER_RUN_ARGS+=("--gpus all") DOCKER_RUN_ARGS+=("--cap-add=sys_nice") else - CONTAINER="${BUILD_CONTAINER}" DOCKER_RUN_ARGS+=("--runtime=runc") fi diff --git a/dependencies.yaml b/dependencies.yaml index 5c2eb2a5b2..95809bb0ee 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -175,6 +175,25 @@ files: - checks - python + # pip dependencies that are used during the conda test stage for morpheus-llm output + morpheus_llm: + output: none + includes: + - test_morpheus_llm_pip + - test_morpheus_core_pip + + # pip dependencies that are used during the conda test stage for morpheus-core output + morpheus_core: + output: none + includes: + - test_morpheus_core_pip + + # pip dependencies that are used during the conda test stage for morpheus-dfp output + morpheus_dfp: + output: none + includes: + - test_morpheus_core_pip + channels: - conda-forge - huggingface @@ -382,6 +401,25 @@ dependencies: - pip: - pytest-kafka==0.6.0 + test_morpheus_core_pip: + common: + - output_types: [requirements] + packages: + - --extra-index-url https://download.pytorch.org/whl/cu124 + - torch==2.4.0+cu124 + + test_morpheus_llm_pip: + common: + - output_types: [requirements] + packages: + - milvus==2.3.5 # update to match pymilvus when available + - pymilvus==2.3.6 + - langchain==0.1.16 + - langchain-nvidia-ai-endpoints==0.0.11 + - faiss-cpu + - google-search-results==2.4 + - nemollm==0.3.5 + example-dfp-prod: common: - output_types: [conda] diff --git a/examples/gnn_fraud_detection_pipeline/run.py b/examples/gnn_fraud_detection_pipeline/run.py index 80cd9d5d0b..6a3268f174 100644 --- a/examples/gnn_fraud_detection_pipeline/run.py +++ b/examples/gnn_fraud_detection_pipeline/run.py @@ -27,10 +27,11 @@ from morpheus.stages.postprocess.serialize_stage import SerializeStage from morpheus.stages.preprocess.deserialize_stage import DeserializeStage from morpheus.utils.logger import configure_logging + # pylint: disable=no-name-in-module -from stages.classification_stage import ClassificationStage -from stages.graph_construction_stage import FraudGraphConstructionStage -from stages.graph_sage_stage import GraphSAGEStage +from .stages.classification_stage import ClassificationStage +from .stages.graph_construction_stage import FraudGraphConstructionStage +from .stages.graph_sage_stage import GraphSAGEStage CUR_DIR = os.path.dirname(__file__) diff --git a/examples/llm/cli.py b/examples/llm/cli.py index cf81557bfd..867d670345 100644 --- a/examples/llm/cli.py +++ b/examples/llm/cli.py @@ -20,11 +20,11 @@ # after_pipeline is, and thus is executed after after_pipeline is invoked. This avoids memory leak warnings at shutdown. # https://github.com/nv-morpheus/Morpheus/issues/1864 import pypdfium2 # pylint: disable=unused-import # noqa: F401 - from llm.agents import run as run_agents from llm.completion import run as run_completion from llm.rag import run as run_rag from llm.vdb_upload import run as run_vdb_upload + from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import parse_log_level diff --git a/examples/ransomware_detection/run.py b/examples/ransomware_detection/run.py index 0c06f21f95..a89c7c93f2 100644 --- a/examples/ransomware_detection/run.py +++ b/examples/ransomware_detection/run.py @@ -30,8 +30,9 @@ from morpheus.stages.postprocess.add_scores_stage import AddScoresStage from morpheus.stages.postprocess.serialize_stage import SerializeStage from morpheus.utils.logger import configure_logging -from stages.create_features import CreateFeaturesRWStage -from stages.preprocessing import PreprocessingRWStage + +from .stages.create_features import CreateFeaturesRWStage +from .stages.preprocessing import PreprocessingRWStage CUR_DIR = os.path.dirname(__file__) diff --git a/tests/_utils/__init__.py b/tests/_utils/__init__.py index b116dffb69..026cc57941 100644 --- a/tests/_utils/__init__.py +++ b/tests/_utils/__init__.py @@ -29,7 +29,7 @@ from .test_directories import TestDirectories -TEST_DIRS = TestDirectories() +TEST_DIRS = TestDirectories(os.path.dirname(os.path.dirname(__file__))) Results = collections.namedtuple('Results', ['total_rows', 'diff_rows', 'error_pct']) diff --git a/tests/_utils/test_directories.py b/tests/_utils/test_directories.py index 9fb9b5c10d..60c3b11b0e 100644 --- a/tests/_utils/test_directories.py +++ b/tests/_utils/test_directories.py @@ -20,9 +20,10 @@ class TestDirectories: - def __init__(self, cur_file=__file__) -> None: - self.tests_dir = os.path.dirname(os.path.dirname(cur_file)) + def __init__(self, tests_dir) -> None: + self.tests_dir = tests_dir self.morpheus_root = os.environ.get('MORPHEUS_ROOT', os.path.dirname(self.tests_dir)) + self.tests_dir = os.path.join(self.morpheus_root, "tests") self.data_dir = morpheus.DATA_DIR self.examples_dir = os.path.join(self.morpheus_root, 'examples') self.models_dir = os.path.join(self.morpheus_root, 'models') @@ -32,3 +33,4 @@ def __init__(self, cur_file=__file__) -> None: self.tests_data_dir = os.path.join(self.tests_dir, 'tests_data') self.mock_triton_servers_dir = os.path.join(self.tests_dir, 'mock_triton_server') self.mock_rest_server = os.path.join(self.tests_dir, 'mock_rest_server') + print(self) diff --git a/tests/conftest.py b/tests/conftest.py index 834426261d..2d7fef3ee3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1119,6 +1119,16 @@ def langchain_nvidia_ai_endpoints_fixture(fail_missing: bool): fail_missing=fail_missing) +@pytest.fixture(name="databricks", scope='session') +def databricks_fixture(fail_missing: bool): + """ + Fixture to ensure databricks is installed + """ + yield import_or_skip("databricks.connect", + reason=OPT_DEP_SKIP_REASON.format(package="databricks-connect"), + fail_missing=fail_missing) + + @pytest.mark.usefixtures("openai") @pytest.fixture(name="mock_chat_completion") def mock_chat_completion_fixture(): diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index 29c06efdc1..64e1defe98 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -43,9 +43,10 @@ def test_constructor( rwd_conf: dict, interested_plugins: typing.List[str]): mock_dask_client.return_value = mock_dask_client + from stages.create_features import CreateFeaturesRWStage + from common.data_models import FeatureConfig from common.feature_extractor import FeatureExtractor - from stages.create_features import CreateFeaturesRWStage n_workers = 12 threads_per_worker = 8 diff --git a/tests/examples/ransomware_detection/test_preprocessing.py b/tests/examples/ransomware_detection/test_preprocessing.py index ad9d3b74eb..55f2584146 100644 --- a/tests/examples/ransomware_detection/test_preprocessing.py +++ b/tests/examples/ransomware_detection/test_preprocessing.py @@ -116,9 +116,10 @@ def test_rollover_pending_snapshots_empty_results(self, assert len(stage._snapshot_dict) == 0 def test_merge_curr_and_prev_snapshots(self, config: Config, rwd_conf: dict, dataset_pandas: DatasetManager): - from common.data_models import SnapshotData from stages.preprocessing import PreprocessingRWStage + from common.data_models import SnapshotData + snapshot_ids = [5, 8, 10, 13] source_pid_process = "123_test.exe" df = dataset_pandas['examples/ransomware_detection/dask_results.csv'] diff --git a/tests/modules/__init__.py b/tests/morpheus/_modules/__init__.py similarity index 100% rename from tests/modules/__init__.py rename to tests/morpheus/_modules/__init__.py diff --git a/tests/modules/multiplexer.py b/tests/morpheus/_modules/multiplexer.py similarity index 100% rename from tests/modules/multiplexer.py rename to tests/morpheus/_modules/multiplexer.py diff --git a/tests/test_abp.py b/tests/morpheus/apps/test_abp.py similarity index 100% rename from tests/test_abp.py rename to tests/morpheus/apps/test_abp.py diff --git a/tests/test_abp_kafka.py b/tests/morpheus/apps/test_abp_kafka.py similarity index 100% rename from tests/test_abp_kafka.py rename to tests/morpheus/apps/test_abp_kafka.py diff --git a/tests/test_phishing.py b/tests/morpheus/apps/test_phishing.py similarity index 100% rename from tests/test_phishing.py rename to tests/morpheus/apps/test_phishing.py diff --git a/tests/test_phishing_kafka.py b/tests/morpheus/apps/test_phishing_kafka.py similarity index 100% rename from tests/test_phishing_kafka.py rename to tests/morpheus/apps/test_phishing_kafka.py diff --git a/tests/test_sid.py b/tests/morpheus/apps/test_sid.py similarity index 100% rename from tests/test_sid.py rename to tests/morpheus/apps/test_sid.py diff --git a/tests/test_sid_kafka.py b/tests/morpheus/apps/test_sid_kafka.py similarity index 100% rename from tests/test_sid_kafka.py rename to tests/morpheus/apps/test_sid_kafka.py diff --git a/tests/morpheus/conftest.py b/tests/morpheus/conftest.py new file mode 100644 index 0000000000..3443e3c91a --- /dev/null +++ b/tests/morpheus/conftest.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +import pytest + +from _utils import TEST_DIRS + + +@pytest.fixture(autouse=True) +def morpheus_sys_path(): + sys.path.append(os.path.join(TEST_DIRS.tests_dir, "morpheus")) diff --git a/tests/controllers/test_elasticsearch_controller.py b/tests/morpheus/controllers/test_elasticsearch_controller.py similarity index 100% rename from tests/controllers/test_elasticsearch_controller.py rename to tests/morpheus/controllers/test_elasticsearch_controller.py diff --git a/tests/controllers/test_rss_controller.py b/tests/morpheus/controllers/test_rss_controller.py similarity index 100% rename from tests/controllers/test_rss_controller.py rename to tests/morpheus/controllers/test_rss_controller.py diff --git a/tests/dfencoder/test_autoencoder.py b/tests/morpheus/dfencoder/test_autoencoder.py similarity index 100% rename from tests/dfencoder/test_autoencoder.py rename to tests/morpheus/dfencoder/test_autoencoder.py diff --git a/tests/dfencoder/test_dfencoder_distributed_e2e.py b/tests/morpheus/dfencoder/test_dfencoder_distributed_e2e.py similarity index 100% rename from tests/dfencoder/test_dfencoder_distributed_e2e.py rename to tests/morpheus/dfencoder/test_dfencoder_distributed_e2e.py diff --git a/tests/dfencoder/test_dfencoder_e2e.py b/tests/morpheus/dfencoder/test_dfencoder_e2e.py similarity index 100% rename from tests/dfencoder/test_dfencoder_e2e.py rename to tests/morpheus/dfencoder/test_dfencoder_e2e.py diff --git a/tests/dfencoder/test_encoder_dataframe.py b/tests/morpheus/dfencoder/test_encoder_dataframe.py similarity index 100% rename from tests/dfencoder/test_encoder_dataframe.py rename to tests/morpheus/dfencoder/test_encoder_dataframe.py diff --git a/tests/dfencoder/test_pkg.py b/tests/morpheus/dfencoder/test_pkg.py similarity index 100% rename from tests/dfencoder/test_pkg.py rename to tests/morpheus/dfencoder/test_pkg.py diff --git a/tests/dfencoder/test_scalers.py b/tests/morpheus/dfencoder/test_scalers.py similarity index 100% rename from tests/dfencoder/test_scalers.py rename to tests/morpheus/dfencoder/test_scalers.py diff --git a/tests/io/data_storage/test_file_system_storage.py b/tests/morpheus/io/data_storage/test_file_system_storage.py similarity index 100% rename from tests/io/data_storage/test_file_system_storage.py rename to tests/morpheus/io/data_storage/test_file_system_storage.py diff --git a/tests/io/data_storage/test_in_memory_storage.py b/tests/morpheus/io/data_storage/test_in_memory_storage.py similarity index 100% rename from tests/io/data_storage/test_in_memory_storage.py rename to tests/morpheus/io/data_storage/test_in_memory_storage.py diff --git a/tests/io/test_data_manager.py b/tests/morpheus/io/test_data_manager.py similarity index 100% rename from tests/io/test_data_manager.py rename to tests/morpheus/io/test_data_manager.py diff --git a/tests/io/test_data_record.py b/tests/morpheus/io/test_data_record.py similarity index 100% rename from tests/io/test_data_record.py rename to tests/morpheus/io/test_data_record.py diff --git a/tests/io/test_io_utils.py b/tests/morpheus/io/test_io_utils.py similarity index 100% rename from tests/io/test_io_utils.py rename to tests/morpheus/io/test_io_utils.py diff --git a/tests/io/test_loader_registry.py b/tests/morpheus/io/test_loader_registry.py similarity index 100% rename from tests/io/test_loader_registry.py rename to tests/morpheus/io/test_loader_registry.py diff --git a/tests/io/test_serializers.py b/tests/morpheus/io/test_serializers.py similarity index 100% rename from tests/io/test_serializers.py rename to tests/morpheus/io/test_serializers.py diff --git a/tests/messages/test_control_message.py b/tests/morpheus/messages/test_control_message.py similarity index 100% rename from tests/messages/test_control_message.py rename to tests/morpheus/messages/test_control_message.py diff --git a/tests/messages/test_message_meta.py b/tests/morpheus/messages/test_message_meta.py similarity index 100% rename from tests/messages/test_message_meta.py rename to tests/morpheus/messages/test_message_meta.py diff --git a/tests/test_messages.py b/tests/morpheus/messages/test_messages.py similarity index 100% rename from tests/test_messages.py rename to tests/morpheus/messages/test_messages.py diff --git a/tests/test_tensor_memory.py b/tests/morpheus/messages/test_tensor_memory.py similarity index 100% rename from tests/test_tensor_memory.py rename to tests/morpheus/messages/test_tensor_memory.py diff --git a/tests/morpheus/modules/__init__.py b/tests/morpheus/modules/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/modules/morpheus_module_test_pipeline.py b/tests/morpheus/modules/morpheus_module_test_pipeline.py similarity index 100% rename from tests/modules/morpheus_module_test_pipeline.py rename to tests/morpheus/modules/morpheus_module_test_pipeline.py diff --git a/tests/modules/test_file_batcher.py b/tests/morpheus/modules/test_file_batcher.py similarity index 100% rename from tests/modules/test_file_batcher.py rename to tests/morpheus/modules/test_file_batcher.py diff --git a/tests/modules/test_from_control_message.py b/tests/morpheus/modules/test_from_control_message.py similarity index 100% rename from tests/modules/test_from_control_message.py rename to tests/morpheus/modules/test_from_control_message.py diff --git a/tests/modules/test_morpheus_modules.py b/tests/morpheus/modules/test_morpheus_modules.py similarity index 100% rename from tests/modules/test_morpheus_modules.py rename to tests/morpheus/modules/test_morpheus_modules.py diff --git a/tests/modules/test_morpheus_modules_with_pipeline.py b/tests/morpheus/modules/test_morpheus_modules_with_pipeline.py similarity index 100% rename from tests/modules/test_morpheus_modules_with_pipeline.py rename to tests/morpheus/modules/test_morpheus_modules_with_pipeline.py diff --git a/tests/modules/test_payload_batcher.py b/tests/morpheus/modules/test_payload_batcher.py similarity index 100% rename from tests/modules/test_payload_batcher.py rename to tests/morpheus/modules/test_payload_batcher.py diff --git a/tests/modules/test_to_control_message.py b/tests/morpheus/modules/test_to_control_message.py similarity index 100% rename from tests/modules/test_to_control_message.py rename to tests/morpheus/modules/test_to_control_message.py diff --git a/tests/test_ip.py b/tests/morpheus/parsers/test_ip.py similarity index 100% rename from tests/test_ip.py rename to tests/morpheus/parsers/test_ip.py diff --git a/tests/test_splunk_notable_parser.py b/tests/morpheus/parsers/test_splunk_notable_parser.py similarity index 100% rename from tests/test_splunk_notable_parser.py rename to tests/morpheus/parsers/test_splunk_notable_parser.py diff --git a/tests/test_url_parser.py b/tests/morpheus/parsers/test_url_parser.py similarity index 100% rename from tests/test_url_parser.py rename to tests/morpheus/parsers/test_url_parser.py diff --git a/tests/test_windows_event_parser.py b/tests/morpheus/parsers/test_windows_event_parser.py similarity index 100% rename from tests/test_windows_event_parser.py rename to tests/morpheus/parsers/test_windows_event_parser.py diff --git a/tests/test_zeek.py b/tests/morpheus/parsers/test_zeek.py similarity index 100% rename from tests/test_zeek.py rename to tests/morpheus/parsers/test_zeek.py diff --git a/tests/pipeline/conftest.py b/tests/morpheus/pipeline/conftest.py similarity index 100% rename from tests/pipeline/conftest.py rename to tests/morpheus/pipeline/conftest.py diff --git a/tests/test_error_pipe.py b/tests/morpheus/pipeline/test_error_pipe.py similarity index 100% rename from tests/test_error_pipe.py rename to tests/morpheus/pipeline/test_error_pipe.py diff --git a/tests/test_file_in_out.py b/tests/morpheus/pipeline/test_file_in_out.py similarity index 100% rename from tests/test_file_in_out.py rename to tests/morpheus/pipeline/test_file_in_out.py diff --git a/tests/test_multi_port_pipeline.py b/tests/morpheus/pipeline/test_multi_port_pipeline.py similarity index 97% rename from tests/test_multi_port_pipeline.py rename to tests/morpheus/pipeline/test_multi_port_pipeline.py index cac48b0bd0..3b4b83a95b 100755 --- a/tests/test_multi_port_pipeline.py +++ b/tests/morpheus/pipeline/test_multi_port_pipeline.py @@ -18,9 +18,6 @@ import cudf -# When segment modules are imported, they're added to the module registry. -# To avoid flake8 warnings about unused code, the noqa flag is used during import. -import modules.multiplexer # noqa: F401 # pylint: disable=unused-import from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.pipeline.pipeline import Pipeline @@ -28,6 +25,10 @@ from morpheus.stages.input.in_memory_source_stage import InMemorySourceStage from morpheus.stages.output.in_memory_sink_stage import InMemorySinkStage +# When segment modules are imported, they're added to the module registry. +# To avoid flake8 warnings about unused code, the noqa flag is used during import. +import _modules.multiplexer # noqa: F401 # pylint: disable=unused-import # isort:skip + def _run_pipeline(config: Config, source_df: cudf.DataFrame, module_conf: dict, stage_input_ports: list[str]) -> InMemorySinkStage: diff --git a/tests/test_nonlinear_pipeline.py b/tests/morpheus/pipeline/test_nonlinear_pipeline.py similarity index 100% rename from tests/test_nonlinear_pipeline.py rename to tests/morpheus/pipeline/test_nonlinear_pipeline.py diff --git a/tests/pipeline/test_pipe_viz.py b/tests/morpheus/pipeline/test_pipe_viz.py similarity index 100% rename from tests/pipeline/test_pipe_viz.py rename to tests/morpheus/pipeline/test_pipe_viz.py diff --git a/tests/pipeline/test_pipeline.py b/tests/morpheus/pipeline/test_pipeline.py similarity index 100% rename from tests/pipeline/test_pipeline.py rename to tests/morpheus/pipeline/test_pipeline.py diff --git a/tests/pipeline/test_pipeline_error.py b/tests/morpheus/pipeline/test_pipeline_error.py similarity index 100% rename from tests/pipeline/test_pipeline_error.py rename to tests/morpheus/pipeline/test_pipeline_error.py diff --git a/tests/pipeline/test_pipeline_state.py b/tests/morpheus/pipeline/test_pipeline_state.py similarity index 100% rename from tests/pipeline/test_pipeline_state.py rename to tests/morpheus/pipeline/test_pipeline_state.py diff --git a/tests/pipeline/test_port_schema.py b/tests/morpheus/pipeline/test_port_schema.py similarity index 100% rename from tests/pipeline/test_port_schema.py rename to tests/morpheus/pipeline/test_port_schema.py diff --git a/tests/pipeline/test_preallocation_pipe.py b/tests/morpheus/pipeline/test_preallocation_pipe.py similarity index 100% rename from tests/pipeline/test_preallocation_pipe.py rename to tests/morpheus/pipeline/test_preallocation_pipe.py diff --git a/tests/pipeline/test_stage_decorator.py b/tests/morpheus/pipeline/test_stage_decorator.py similarity index 100% rename from tests/pipeline/test_stage_decorator.py rename to tests/morpheus/pipeline/test_stage_decorator.py diff --git a/tests/pipeline/test_stage_schema.py b/tests/morpheus/pipeline/test_stage_schema.py similarity index 100% rename from tests/pipeline/test_stage_schema.py rename to tests/morpheus/pipeline/test_stage_schema.py diff --git a/tests/morpheus/stages/databricks/conftest.py b/tests/morpheus/stages/databricks/conftest.py new file mode 100644 index 0000000000..3d602715e3 --- /dev/null +++ b/tests/morpheus/stages/databricks/conftest.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +# Override fixtures from parent setting autouse to True +@pytest.fixture(name="databricks", autouse=True, scope='session') +def databricks_fixture(databricks): + """ + All of the tests in this subdir require databricks + """ + yield databricks diff --git a/tests/test_databricks_deltalake_source_stage.py b/tests/morpheus/stages/databricks/test_databricks_deltalake_source_stage.py similarity index 100% rename from tests/test_databricks_deltalake_source_stage.py rename to tests/morpheus/stages/databricks/test_databricks_deltalake_source_stage.py diff --git a/tests/test_write_to_databricks_deltalake_stage.py b/tests/morpheus/stages/databricks/test_write_to_databricks_deltalake_stage.py similarity index 100% rename from tests/test_write_to_databricks_deltalake_stage.py rename to tests/morpheus/stages/databricks/test_write_to_databricks_deltalake_stage.py diff --git a/tests/test_add_classifications_stage.py b/tests/morpheus/stages/test_add_classifications_stage.py similarity index 100% rename from tests/test_add_classifications_stage.py rename to tests/morpheus/stages/test_add_classifications_stage.py diff --git a/tests/test_add_classifications_stage_pipe.py b/tests/morpheus/stages/test_add_classifications_stage_pipe.py similarity index 100% rename from tests/test_add_classifications_stage_pipe.py rename to tests/morpheus/stages/test_add_classifications_stage_pipe.py diff --git a/tests/test_add_scores_stage.py b/tests/morpheus/stages/test_add_scores_stage.py similarity index 100% rename from tests/test_add_scores_stage.py rename to tests/morpheus/stages/test_add_scores_stage.py diff --git a/tests/test_add_scores_stage_pipe.py b/tests/morpheus/stages/test_add_scores_stage_pipe.py similarity index 100% rename from tests/test_add_scores_stage_pipe.py rename to tests/morpheus/stages/test_add_scores_stage_pipe.py diff --git a/tests/test_appshield_source_stage.py b/tests/morpheus/stages/test_appshield_source_stage.py similarity index 100% rename from tests/test_appshield_source_stage.py rename to tests/morpheus/stages/test_appshield_source_stage.py diff --git a/tests/stages/test_deserialize_stage_pipe.py b/tests/morpheus/stages/test_deserialize_stage_pipe.py similarity index 100% rename from tests/stages/test_deserialize_stage_pipe.py rename to tests/morpheus/stages/test_deserialize_stage_pipe.py diff --git a/tests/stages/test_file_source_stage_pipe.py b/tests/morpheus/stages/test_file_source_stage_pipe.py similarity index 100% rename from tests/stages/test_file_source_stage_pipe.py rename to tests/morpheus/stages/test_file_source_stage_pipe.py diff --git a/tests/stages/test_filter_detections_stage.py b/tests/morpheus/stages/test_filter_detections_stage.py similarity index 100% rename from tests/stages/test_filter_detections_stage.py rename to tests/morpheus/stages/test_filter_detections_stage.py diff --git a/tests/test_filter_detections_stage_column_pipe.py b/tests/morpheus/stages/test_filter_detections_stage_column_pipe.py similarity index 100% rename from tests/test_filter_detections_stage_column_pipe.py rename to tests/morpheus/stages/test_filter_detections_stage_column_pipe.py diff --git a/tests/test_filter_detections_stage_pipe.py b/tests/morpheus/stages/test_filter_detections_stage_pipe.py similarity index 100% rename from tests/test_filter_detections_stage_pipe.py rename to tests/morpheus/stages/test_filter_detections_stage_pipe.py diff --git a/tests/stages/test_generate_viz_frames_stage.py b/tests/morpheus/stages/test_generate_viz_frames_stage.py similarity index 100% rename from tests/stages/test_generate_viz_frames_stage.py rename to tests/morpheus/stages/test_generate_viz_frames_stage.py diff --git a/tests/stages/test_group_by_column_stage.py b/tests/morpheus/stages/test_group_by_column_stage.py similarity index 100% rename from tests/stages/test_group_by_column_stage.py rename to tests/morpheus/stages/test_group_by_column_stage.py diff --git a/tests/test_http_client_sink_stage_pipe.py b/tests/morpheus/stages/test_http_client_sink_stage_pipe.py similarity index 100% rename from tests/test_http_client_sink_stage_pipe.py rename to tests/morpheus/stages/test_http_client_sink_stage_pipe.py diff --git a/tests/stages/test_http_client_source_stage_pipe.py b/tests/morpheus/stages/test_http_client_source_stage_pipe.py similarity index 100% rename from tests/stages/test_http_client_source_stage_pipe.py rename to tests/morpheus/stages/test_http_client_source_stage_pipe.py diff --git a/tests/test_http_server_sink_stage.py b/tests/morpheus/stages/test_http_server_sink_stage.py similarity index 100% rename from tests/test_http_server_sink_stage.py rename to tests/morpheus/stages/test_http_server_sink_stage.py diff --git a/tests/stages/test_http_server_source_stage.py b/tests/morpheus/stages/test_http_server_source_stage.py similarity index 100% rename from tests/stages/test_http_server_source_stage.py rename to tests/morpheus/stages/test_http_server_source_stage.py diff --git a/tests/stages/test_http_server_source_stage_pipe.py b/tests/morpheus/stages/test_http_server_source_stage_pipe.py similarity index 100% rename from tests/stages/test_http_server_source_stage_pipe.py rename to tests/morpheus/stages/test_http_server_source_stage_pipe.py diff --git a/tests/test_inference_stage.py b/tests/morpheus/stages/test_inference_stage.py similarity index 100% rename from tests/test_inference_stage.py rename to tests/morpheus/stages/test_inference_stage.py diff --git a/tests/test_kafka_source_stage_pipe.py b/tests/morpheus/stages/test_kafka_source_stage_pipe.py similarity index 100% rename from tests/test_kafka_source_stage_pipe.py rename to tests/morpheus/stages/test_kafka_source_stage_pipe.py diff --git a/tests/test_linear_modules_stage.py b/tests/morpheus/stages/test_linear_modules_stage.py similarity index 100% rename from tests/test_linear_modules_stage.py rename to tests/morpheus/stages/test_linear_modules_stage.py diff --git a/tests/stages/test_ml_flow_drift_stage.py b/tests/morpheus/stages/test_ml_flow_drift_stage.py similarity index 100% rename from tests/stages/test_ml_flow_drift_stage.py rename to tests/morpheus/stages/test_ml_flow_drift_stage.py diff --git a/tests/test_monitor_stage.py b/tests/morpheus/stages/test_monitor_stage.py similarity index 100% rename from tests/test_monitor_stage.py rename to tests/morpheus/stages/test_monitor_stage.py diff --git a/tests/test_multi_port_modules_stage.py b/tests/morpheus/stages/test_multi_port_modules_stage.py similarity index 98% rename from tests/test_multi_port_modules_stage.py rename to tests/morpheus/stages/test_multi_port_modules_stage.py index ce8388ae1c..31ac032546 100755 --- a/tests/test_multi_port_modules_stage.py +++ b/tests/morpheus/stages/test_multi_port_modules_stage.py @@ -19,10 +19,11 @@ import pytest +from morpheus.stages.general.multi_port_modules_stage import MultiPortModulesStage + # When segment modules are imported, they're added to the module registry. # To avoid flake8 warnings about unused code, the noqa flag is used during import. -import modules.multiplexer # noqa: F401 # pylint:disable=unused-import -from morpheus.stages.general.multi_port_modules_stage import MultiPortModulesStage +import _modules.multiplexer # noqa: F401 # pylint:disable=unused-import # isort:skip # pylint: disable=redefined-outer-name diff --git a/tests/test_multi_processing_stage.py b/tests/morpheus/stages/test_multi_processing_stage.py similarity index 100% rename from tests/test_multi_processing_stage.py rename to tests/morpheus/stages/test_multi_processing_stage.py diff --git a/tests/test_multi_segment.py b/tests/morpheus/stages/test_multi_segment.py similarity index 100% rename from tests/test_multi_segment.py rename to tests/morpheus/stages/test_multi_segment.py diff --git a/tests/stages/test_preprocess_ae_stage.py b/tests/morpheus/stages/test_preprocess_ae_stage.py similarity index 100% rename from tests/stages/test_preprocess_ae_stage.py rename to tests/morpheus/stages/test_preprocess_ae_stage.py diff --git a/tests/stages/test_preprocess_fil_stage.py b/tests/morpheus/stages/test_preprocess_fil_stage.py similarity index 100% rename from tests/stages/test_preprocess_fil_stage.py rename to tests/morpheus/stages/test_preprocess_fil_stage.py diff --git a/tests/stages/test_preprocess_nlp_stage.py b/tests/morpheus/stages/test_preprocess_nlp_stage.py similarity index 100% rename from tests/stages/test_preprocess_nlp_stage.py rename to tests/morpheus/stages/test_preprocess_nlp_stage.py diff --git a/tests/test_rss_source_stage_pipe.py b/tests/morpheus/stages/test_rss_source_stage_pipe.py similarity index 100% rename from tests/test_rss_source_stage_pipe.py rename to tests/morpheus/stages/test_rss_source_stage_pipe.py diff --git a/tests/test_serialize_stage.py b/tests/morpheus/stages/test_serialize_stage.py similarity index 100% rename from tests/test_serialize_stage.py rename to tests/morpheus/stages/test_serialize_stage.py diff --git a/tests/stages/test_timeseries_stage.py b/tests/morpheus/stages/test_timeseries_stage.py similarity index 100% rename from tests/stages/test_timeseries_stage.py rename to tests/morpheus/stages/test_timeseries_stage.py diff --git a/tests/test_trigger_stage.py b/tests/morpheus/stages/test_trigger_stage.py similarity index 100% rename from tests/test_trigger_stage.py rename to tests/morpheus/stages/test_trigger_stage.py diff --git a/tests/test_triton_inference_stage.py b/tests/morpheus/stages/test_triton_inference_stage.py similarity index 100% rename from tests/test_triton_inference_stage.py rename to tests/morpheus/stages/test_triton_inference_stage.py diff --git a/tests/stages/test_validation_stage.py b/tests/morpheus/stages/test_validation_stage.py similarity index 100% rename from tests/stages/test_validation_stage.py rename to tests/morpheus/stages/test_validation_stage.py diff --git a/tests/test_validation_stage_pipe.py b/tests/morpheus/stages/test_validation_stage_pipe.py similarity index 100% rename from tests/test_validation_stage_pipe.py rename to tests/morpheus/stages/test_validation_stage_pipe.py diff --git a/tests/test_write_to_elasticsearch_stage_pipe.py b/tests/morpheus/stages/test_write_to_elasticsearch_stage_pipe.py similarity index 100% rename from tests/test_write_to_elasticsearch_stage_pipe.py rename to tests/morpheus/stages/test_write_to_elasticsearch_stage_pipe.py diff --git a/tests/test_write_to_file_stage.py b/tests/morpheus/stages/test_write_to_file_stage.py similarity index 100% rename from tests/test_write_to_file_stage.py rename to tests/morpheus/stages/test_write_to_file_stage.py diff --git a/tests/test_write_to_kafka_stage_pipe.py b/tests/morpheus/stages/test_write_to_kafka_stage_pipe.py similarity index 100% rename from tests/test_write_to_kafka_stage_pipe.py rename to tests/morpheus/stages/test_write_to_kafka_stage_pipe.py diff --git a/tests/test_cli.py b/tests/morpheus/test_cli.py similarity index 100% rename from tests/test_cli.py rename to tests/morpheus/test_cli.py diff --git a/tests/test_config.py b/tests/morpheus/test_config.py similarity index 100% rename from tests/test_config.py rename to tests/morpheus/test_config.py diff --git a/tests/test_package.py b/tests/morpheus/test_package.py similarity index 100% rename from tests/test_package.py rename to tests/morpheus/test_package.py diff --git a/tests/test_column_info.py b/tests/morpheus/utils/test_column_info.py similarity index 100% rename from tests/test_column_info.py rename to tests/morpheus/utils/test_column_info.py diff --git a/tests/test_concat_df.py b/tests/morpheus/utils/test_concat_df.py similarity index 100% rename from tests/test_concat_df.py rename to tests/morpheus/utils/test_concat_df.py diff --git a/tests/utils/test_control_message_utils.py b/tests/morpheus/utils/test_control_message_utils.py similarity index 100% rename from tests/utils/test_control_message_utils.py rename to tests/morpheus/utils/test_control_message_utils.py diff --git a/tests/utils/test_cudf_subword_helper.py b/tests/morpheus/utils/test_cudf_subword_helper.py similarity index 100% rename from tests/utils/test_cudf_subword_helper.py rename to tests/morpheus/utils/test_cudf_subword_helper.py diff --git a/tests/test_directory_watcher.py b/tests/morpheus/utils/test_directory_watcher.py similarity index 100% rename from tests/test_directory_watcher.py rename to tests/morpheus/utils/test_directory_watcher.py diff --git a/tests/test_downloader.py b/tests/morpheus/utils/test_downloader.py similarity index 100% rename from tests/test_downloader.py rename to tests/morpheus/utils/test_downloader.py diff --git a/tests/test_env_config_value.py b/tests/morpheus/utils/test_env_config_value.py similarity index 100% rename from tests/test_env_config_value.py rename to tests/morpheus/utils/test_env_config_value.py diff --git a/tests/utils/test_execution_chain.py b/tests/morpheus/utils/test_execution_chain.py similarity index 100% rename from tests/utils/test_execution_chain.py rename to tests/morpheus/utils/test_execution_chain.py diff --git a/tests/utils/test_http_utils.py b/tests/morpheus/utils/test_http_utils.py similarity index 100% rename from tests/utils/test_http_utils.py rename to tests/morpheus/utils/test_http_utils.py diff --git a/tests/test_inference_worker.py b/tests/morpheus/utils/test_inference_worker.py similarity index 100% rename from tests/test_inference_worker.py rename to tests/morpheus/utils/test_inference_worker.py diff --git a/tests/test_logger.py b/tests/morpheus/utils/test_logger.py similarity index 100% rename from tests/test_logger.py rename to tests/morpheus/utils/test_logger.py diff --git a/tests/test_module_utils.py b/tests/morpheus/utils/test_module_utils.py similarity index 100% rename from tests/test_module_utils.py rename to tests/morpheus/utils/test_module_utils.py diff --git a/tests/utils/test_shared_process_pool.py b/tests/morpheus/utils/test_shared_process_pool.py similarity index 100% rename from tests/utils/test_shared_process_pool.py rename to tests/morpheus/utils/test_shared_process_pool.py diff --git a/tests/utils/test_sql_utils.py b/tests/morpheus/utils/test_sql_utils.py similarity index 100% rename from tests/utils/test_sql_utils.py rename to tests/morpheus/utils/test_sql_utils.py diff --git a/tests/examples/digital_fingerprinting/conftest.py b/tests/morpheus_dfp/conftest.py similarity index 100% rename from tests/examples/digital_fingerprinting/conftest.py rename to tests/morpheus_dfp/conftest.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_training.py b/tests/morpheus_dfp/modules/test_dfp_training.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_training.py rename to tests/morpheus_dfp/modules/test_dfp_training.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_file_batcher_stage.py b/tests/morpheus_dfp/stages/test_dfp_file_batcher_stage.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_file_batcher_stage.py rename to tests/morpheus_dfp/stages/test_dfp_file_batcher_stage.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_inference_stage.py b/tests/morpheus_dfp/stages/test_dfp_inference_stage.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_inference_stage.py rename to tests/morpheus_dfp/stages/test_dfp_inference_stage.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py b/tests/morpheus_dfp/stages/test_dfp_mlflow_model_writer.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py rename to tests/morpheus_dfp/stages/test_dfp_mlflow_model_writer.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_postprocessing_stage.py b/tests/morpheus_dfp/stages/test_dfp_postprocessing_stage.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_postprocessing_stage.py rename to tests/morpheus_dfp/stages/test_dfp_postprocessing_stage.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_preprocessing_stage.py b/tests/morpheus_dfp/stages/test_dfp_preprocessing_stage.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_preprocessing_stage.py rename to tests/morpheus_dfp/stages/test_dfp_preprocessing_stage.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py b/tests/morpheus_dfp/stages/test_dfp_rolling_window_stage.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py rename to tests/morpheus_dfp/stages/test_dfp_rolling_window_stage.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py b/tests/morpheus_dfp/stages/test_dfp_split_users_stage.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py rename to tests/morpheus_dfp/stages/test_dfp_split_users_stage.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_viz_postproc.py b/tests/morpheus_dfp/stages/test_dfp_viz_postproc.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_viz_postproc.py rename to tests/morpheus_dfp/stages/test_dfp_viz_postproc.py diff --git a/tests/examples/digital_fingerprinting/test_multifile_source.py b/tests/morpheus_dfp/stages/test_multifile_source.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_multifile_source.py rename to tests/morpheus_dfp/stages/test_multifile_source.py diff --git a/tests/examples/digital_fingerprinting/test_write_to_s3_stage.py b/tests/morpheus_dfp/stages/test_write_to_s3_stage.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_write_to_s3_stage.py rename to tests/morpheus_dfp/stages/test_write_to_s3_stage.py diff --git a/tests/test_dfp.py b/tests/morpheus_dfp/test_dfp.py similarity index 100% rename from tests/test_dfp.py rename to tests/morpheus_dfp/test_dfp.py diff --git a/tests/examples/digital_fingerprinting/test_dfp_file_to_df.py b/tests/morpheus_dfp/test_dfp_file_to_df.py similarity index 100% rename from tests/examples/digital_fingerprinting/test_dfp_file_to_df.py rename to tests/morpheus_dfp/test_dfp_file_to_df.py diff --git a/tests/test_dfp_kafka.py b/tests/morpheus_dfp/test_dfp_kafka.py similarity index 100% rename from tests/test_dfp_kafka.py rename to tests/morpheus_dfp/test_dfp_kafka.py diff --git a/tests/examples/digital_fingerprinting/utils/test_config_generator.py b/tests/morpheus_dfp/utils/test_config_generator.py similarity index 100% rename from tests/examples/digital_fingerprinting/utils/test_config_generator.py rename to tests/morpheus_dfp/utils/test_config_generator.py diff --git a/tests/llm/conftest.py b/tests/morpheus_llm/llm/conftest.py similarity index 100% rename from tests/llm/conftest.py rename to tests/morpheus_llm/llm/conftest.py diff --git a/tests/llm/nodes/conftest.py b/tests/morpheus_llm/llm/nodes/conftest.py similarity index 100% rename from tests/llm/nodes/conftest.py rename to tests/morpheus_llm/llm/nodes/conftest.py diff --git a/tests/llm/nodes/test_extractor_node.py b/tests/morpheus_llm/llm/nodes/test_extractor_node.py similarity index 100% rename from tests/llm/nodes/test_extractor_node.py rename to tests/morpheus_llm/llm/nodes/test_extractor_node.py diff --git a/tests/llm/nodes/test_langchain_agent_node.py b/tests/morpheus_llm/llm/nodes/test_langchain_agent_node.py similarity index 100% rename from tests/llm/nodes/test_langchain_agent_node.py rename to tests/morpheus_llm/llm/nodes/test_langchain_agent_node.py diff --git a/tests/llm/nodes/test_langchain_agent_node_pipe.py b/tests/morpheus_llm/llm/nodes/test_langchain_agent_node_pipe.py similarity index 100% rename from tests/llm/nodes/test_langchain_agent_node_pipe.py rename to tests/morpheus_llm/llm/nodes/test_langchain_agent_node_pipe.py diff --git a/tests/llm/nodes/test_llm_generate_node.py b/tests/morpheus_llm/llm/nodes/test_llm_generate_node.py similarity index 100% rename from tests/llm/nodes/test_llm_generate_node.py rename to tests/morpheus_llm/llm/nodes/test_llm_generate_node.py diff --git a/tests/llm/nodes/test_llm_generate_node_pipe.py b/tests/morpheus_llm/llm/nodes/test_llm_generate_node_pipe.py similarity index 100% rename from tests/llm/nodes/test_llm_generate_node_pipe.py rename to tests/morpheus_llm/llm/nodes/test_llm_generate_node_pipe.py diff --git a/tests/llm/nodes/test_llm_retriever_node.py b/tests/morpheus_llm/llm/nodes/test_llm_retriever_node.py similarity index 100% rename from tests/llm/nodes/test_llm_retriever_node.py rename to tests/morpheus_llm/llm/nodes/test_llm_retriever_node.py diff --git a/tests/llm/nodes/test_llm_retriever_node_pipe.py b/tests/morpheus_llm/llm/nodes/test_llm_retriever_node_pipe.py similarity index 100% rename from tests/llm/nodes/test_llm_retriever_node_pipe.py rename to tests/morpheus_llm/llm/nodes/test_llm_retriever_node_pipe.py diff --git a/tests/llm/nodes/test_manual_extractor_node.py b/tests/morpheus_llm/llm/nodes/test_manual_extractor_node.py similarity index 100% rename from tests/llm/nodes/test_manual_extractor_node.py rename to tests/morpheus_llm/llm/nodes/test_manual_extractor_node.py diff --git a/tests/llm/nodes/test_prompt_template_node.py b/tests/morpheus_llm/llm/nodes/test_prompt_template_node.py similarity index 100% rename from tests/llm/nodes/test_prompt_template_node.py rename to tests/morpheus_llm/llm/nodes/test_prompt_template_node.py diff --git a/tests/llm/nodes/test_prompt_template_node_pipe.py b/tests/morpheus_llm/llm/nodes/test_prompt_template_node_pipe.py similarity index 100% rename from tests/llm/nodes/test_prompt_template_node_pipe.py rename to tests/morpheus_llm/llm/nodes/test_prompt_template_node_pipe.py diff --git a/tests/llm/nodes/test_rag_node.py b/tests/morpheus_llm/llm/nodes/test_rag_node.py similarity index 100% rename from tests/llm/nodes/test_rag_node.py rename to tests/morpheus_llm/llm/nodes/test_rag_node.py diff --git a/tests/llm/nodes/test_rag_node_pipe.py b/tests/morpheus_llm/llm/nodes/test_rag_node_pipe.py similarity index 100% rename from tests/llm/nodes/test_rag_node_pipe.py rename to tests/morpheus_llm/llm/nodes/test_rag_node_pipe.py diff --git a/tests/llm/services/conftest.py b/tests/morpheus_llm/llm/services/conftest.py similarity index 100% rename from tests/llm/services/conftest.py rename to tests/morpheus_llm/llm/services/conftest.py diff --git a/tests/llm/services/test_llm_service.py b/tests/morpheus_llm/llm/services/test_llm_service.py similarity index 100% rename from tests/llm/services/test_llm_service.py rename to tests/morpheus_llm/llm/services/test_llm_service.py diff --git a/tests/llm/services/test_llm_service_pipe.py b/tests/morpheus_llm/llm/services/test_llm_service_pipe.py similarity index 100% rename from tests/llm/services/test_llm_service_pipe.py rename to tests/morpheus_llm/llm/services/test_llm_service_pipe.py diff --git a/tests/llm/services/test_nemo_llm_client.py b/tests/morpheus_llm/llm/services/test_nemo_llm_client.py similarity index 100% rename from tests/llm/services/test_nemo_llm_client.py rename to tests/morpheus_llm/llm/services/test_nemo_llm_client.py diff --git a/tests/llm/services/test_nemo_llm_service.py b/tests/morpheus_llm/llm/services/test_nemo_llm_service.py similarity index 100% rename from tests/llm/services/test_nemo_llm_service.py rename to tests/morpheus_llm/llm/services/test_nemo_llm_service.py diff --git a/tests/llm/services/test_nvfoundation_llm_service.py b/tests/morpheus_llm/llm/services/test_nvfoundation_llm_service.py similarity index 100% rename from tests/llm/services/test_nvfoundation_llm_service.py rename to tests/morpheus_llm/llm/services/test_nvfoundation_llm_service.py diff --git a/tests/llm/services/test_openai_chat_service.py b/tests/morpheus_llm/llm/services/test_openai_chat_service.py similarity index 100% rename from tests/llm/services/test_openai_chat_service.py rename to tests/morpheus_llm/llm/services/test_openai_chat_service.py diff --git a/tests/llm/task_handlers/test_simple_task_handler.py b/tests/morpheus_llm/llm/task_handlers/test_simple_task_handler.py similarity index 100% rename from tests/llm/task_handlers/test_simple_task_handler.py rename to tests/morpheus_llm/llm/task_handlers/test_simple_task_handler.py diff --git a/tests/llm/test_agents_simple_pipe.py b/tests/morpheus_llm/llm/test_agents_simple_pipe.py similarity index 100% rename from tests/llm/test_agents_simple_pipe.py rename to tests/morpheus_llm/llm/test_agents_simple_pipe.py diff --git a/tests/llm/test_completion_pipe.py b/tests/morpheus_llm/llm/test_completion_pipe.py similarity index 100% rename from tests/llm/test_completion_pipe.py rename to tests/morpheus_llm/llm/test_completion_pipe.py diff --git a/tests/llm/test_extractor_simple_task_handler_pipe.py b/tests/morpheus_llm/llm/test_extractor_simple_task_handler_pipe.py similarity index 100% rename from tests/llm/test_extractor_simple_task_handler_pipe.py rename to tests/morpheus_llm/llm/test_extractor_simple_task_handler_pipe.py diff --git a/tests/llm/test_llm.py b/tests/morpheus_llm/llm/test_llm.py similarity index 100% rename from tests/llm/test_llm.py rename to tests/morpheus_llm/llm/test_llm.py diff --git a/tests/llm/test_rag_standalone_pipe.py b/tests/morpheus_llm/llm/test_rag_standalone_pipe.py similarity index 100% rename from tests/llm/test_rag_standalone_pipe.py rename to tests/morpheus_llm/llm/test_rag_standalone_pipe.py diff --git a/tests/llm/test_vdb_upload_pipe.py b/tests/morpheus_llm/llm/test_vdb_upload_pipe.py similarity index 100% rename from tests/llm/test_vdb_upload_pipe.py rename to tests/morpheus_llm/llm/test_vdb_upload_pipe.py diff --git a/tests/test_faiss_vector_db_service.py b/tests/morpheus_llm/services/test_faiss_vector_db_service.py similarity index 95% rename from tests/test_faiss_vector_db_service.py rename to tests/morpheus_llm/services/test_faiss_vector_db_service.py index 248d25d78a..3f11de6a29 100644 --- a/tests/test_faiss_vector_db_service.py +++ b/tests/morpheus_llm/services/test_faiss_vector_db_service.py @@ -51,7 +51,7 @@ async def aembed_documents(self, texts: list[str]) -> list[list[float]]: @pytest.fixture(scope="function", name="faiss_simple_store_dir") -def faiss_simple_store_dir_fixture(tmpdir_path: Path): +def faiss_simple_store_dir_fixture(tmp_path: Path): from langchain_community.vectorstores.faiss import FAISS @@ -60,15 +60,15 @@ def faiss_simple_store_dir_fixture(tmpdir_path: Path): # create FAISS docstore for testing index_store = FAISS.from_texts([str(x) * x for x in range(3)], embeddings, ids=[chr(x + 97) for x in range(3)]) - index_store.save_local(str(tmpdir_path), index_name="index") + index_store.save_local(str(tmp_path), index_name="index") # create a second index for testing other_store = FAISS.from_texts([str(x) * x for x in range(3, 8)], embeddings, ids=[chr(x + 97) for x in range(3, 8)]) - other_store.save_local(str(tmpdir_path), index_name="other_index") + other_store.save_local(str(tmp_path), index_name="other_index") - return str(tmpdir_path) + return str(tmp_path) @pytest.fixture(scope="function", name="faiss_service") diff --git a/tests/test_milvus_vector_db_service.py b/tests/morpheus_llm/services/test_milvus_vector_db_service.py similarity index 100% rename from tests/test_milvus_vector_db_service.py rename to tests/morpheus_llm/services/test_milvus_vector_db_service.py diff --git a/tests/stages/arxiv/conftest.py b/tests/morpheus_llm/stages/arxiv/conftest.py similarity index 100% rename from tests/stages/arxiv/conftest.py rename to tests/morpheus_llm/stages/arxiv/conftest.py diff --git a/tests/stages/arxiv/test_arxiv_source.py b/tests/morpheus_llm/stages/arxiv/test_arxiv_source.py similarity index 100% rename from tests/stages/arxiv/test_arxiv_source.py rename to tests/morpheus_llm/stages/arxiv/test_arxiv_source.py diff --git a/tests/stages/arxiv/test_arxiv_source_pipe.py b/tests/morpheus_llm/stages/arxiv/test_arxiv_source_pipe.py similarity index 100% rename from tests/stages/arxiv/test_arxiv_source_pipe.py rename to tests/morpheus_llm/stages/arxiv/test_arxiv_source_pipe.py diff --git a/tests/stages/test_llm_engine_stage.py b/tests/morpheus_llm/stages/test_llm_engine_stage.py similarity index 100% rename from tests/stages/test_llm_engine_stage.py rename to tests/morpheus_llm/stages/test_llm_engine_stage.py diff --git a/tests/stages/test_llm_engine_stage_pipe.py b/tests/morpheus_llm/stages/test_llm_engine_stage_pipe.py similarity index 100% rename from tests/stages/test_llm_engine_stage_pipe.py rename to tests/morpheus_llm/stages/test_llm_engine_stage_pipe.py diff --git a/tests/test_milvus_write_to_vector_db_stage_pipe.py b/tests/morpheus_llm/stages/test_milvus_write_to_vector_db_stage_pipe.py similarity index 100% rename from tests/test_milvus_write_to_vector_db_stage_pipe.py rename to tests/morpheus_llm/stages/test_milvus_write_to_vector_db_stage_pipe.py