diff --git a/.devcontainer/cuda12.1-conda/devcontainer.json b/.devcontainer/cuda12.1-conda/devcontainer.json index fa5d24c6d8..ee34c67fa8 100644 --- a/.devcontainer/cuda12.1-conda/devcontainer.json +++ b/.devcontainer/cuda12.1-conda/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "12.1", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.06-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04" } }, "privileged": true, diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 07ec5ca9e9..1afe90625c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -94,8 +94,8 @@ jobs: conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }} # Upload morpheus conda packages only for non PR branches. Use 'main' for main branch and 'dev' for all other branches conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }} - container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240614 - test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240614 + container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-241004 + test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-241004 secrets: CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }} NGC_API_KEY: ${{ secrets.NGC_API_KEY }} diff --git a/CMakeLists.txt b/CMakeLists.txt index d50f872e85..bd9580ae12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ option(MORPHEUS_USE_IWYU "Enable running include-what-you-use as part of the bui set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Location to install the python directory") -set(MORPHEUS_RAPIDS_VERSION "24.02" CACHE STRING "Sets default versions for RAPIDS libraries.") +set(MORPHEUS_RAPIDS_VERSION "24.10" CACHE STRING "Sets default versions for RAPIDS libraries.") set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data") mark_as_advanced(MORPHEUS_CACHE_DIR) diff --git a/ci/conda/recipes/morpheus-libs/conda_build_config.yaml b/ci/conda/recipes/morpheus-libs/conda_build_config.yaml index 4b051dc074..9c88ef46cb 100644 --- a/ci/conda/recipes/morpheus-libs/conda_build_config.yaml +++ b/ci/conda/recipes/morpheus-libs/conda_build_config.yaml @@ -14,19 +14,19 @@ # limitations under the License. c_compiler_version: - - 11.2 + - 12.1 cxx_compiler_version: - - 11.2 + - 12.1 cuda_compiler: - cuda-nvcc cuda_compiler_version: - - 12.1 + - 12.5 python: - 3.10 rapids_version: - - 24.02 + - 24.10 diff --git a/ci/conda/recipes/morpheus-libs/meta.yaml b/ci/conda/recipes/morpheus-libs/meta.yaml index 17db27a5dd..dffdffe53e 100644 --- a/ci/conda/recipes/morpheus-libs/meta.yaml +++ b/ci/conda/recipes/morpheus-libs/meta.yaml @@ -53,7 +53,7 @@ outputs: - cuda-version {{ cuda_compiler_version }}.* - ninja =1.11 - pkg-config =0.29 - - sysroot_linux-64 =2.17 + - sysroot_linux-64 =2.28 host: # CUDA dependencies - cuda-cudart-dev {{ cuda_compiler_version }}.* @@ -65,7 +65,7 @@ outputs: # Non-CUDA dependencies - cudf {{ rapids_version }} - cython 3.0.* - - glog 0.6.* + - glog >=0.7.1,<0.8 - libcudf {{ rapids_version }} - librdkafka >=1.9.2,<1.10.0a0 - mrc {{ minor_version }} @@ -76,7 +76,7 @@ outputs: - rapidjson 1.1.0 - scikit-build 0.17.6 - versioneer-518 - - zlib 1.2.13 # required to build triton client + - zlib 1.3.1 # required to build triton client run: # Runtime only requirements. This + setup.py is the definitive runtime requirement list # This should be synced with `runtime` in dependencies.yaml @@ -91,7 +91,7 @@ outputs: - docker-py =5.0.* - elasticsearch ==8.9.0 - feedparser =6.0.* - - grpcio =1.59.* + - grpcio =1.62.* - mlflow>=2.10.0,<3 - mrc - networkx=2.8.8 @@ -101,8 +101,6 @@ outputs: - python - python-confluent-kafka >=1.9.2,<1.10.0a0 - python-graphviz - - pytorch-cuda - - pytorch * *cuda* - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed - requests - requests-cache =1.1.* @@ -188,14 +186,14 @@ outputs: # package versions getting installed creating unexpected version conflicts. - {{ pin_subpackage('morpheus-core', exact=True) }} - cython 3.0.* - - glog 0.6.* + - glog >=0.7.1,<0.8 - pip - pybind11-stubgen 0.10.5 - python {{ python }} - rapidjson 1.1.0 - scikit-build 0.17.6 - versioneer-518 - - zlib 1.2.13 # required to build triton client + - zlib 1.3.1 # required to build triton client run: - {{ pin_subpackage('morpheus-core', exact=True) }} diff --git a/ci/conda/recipes/morpheus/conda_build_config.yaml b/ci/conda/recipes/morpheus/conda_build_config.yaml index 4b051dc074..9c88ef46cb 100644 --- a/ci/conda/recipes/morpheus/conda_build_config.yaml +++ b/ci/conda/recipes/morpheus/conda_build_config.yaml @@ -14,19 +14,19 @@ # limitations under the License. c_compiler_version: - - 11.2 + - 12.1 cxx_compiler_version: - - 11.2 + - 12.1 cuda_compiler: - cuda-nvcc cuda_compiler_version: - - 12.1 + - 12.5 python: - 3.10 rapids_version: - - 24.02 + - 24.10 diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index fc55e54f30..23327bcbc7 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -56,7 +56,7 @@ outputs: - libtool # Needed for DOCA build - ninja =1.11 - pkg-config =0.29 # for mrc cmake - - sysroot_linux-64 =2.17 + - sysroot_linux-64 =2.28 host: # CUDA dependencies - cuda-cudart-dev {{ cuda_compiler_version }}.* @@ -68,7 +68,7 @@ outputs: # Non-CUDA dependencies - cudf {{ rapids_version }} - cython 3.0.* - - glog 0.6.* + - glog >=0.7.1,<0.8 - libcudf {{ rapids_version }} - librdkafka >=1.9.2,<1.10.0a0 - mrc {{ minor_version }} @@ -80,7 +80,7 @@ outputs: - rdma-core >=48 # Needed for DOCA. - scikit-build 0.17.6 - versioneer-518 - - zlib 1.2.13 # required to build triton client + - zlib 1.3.1 # required to build triton client run: # Runtime only requirements. This + setup.py is the definitive runtime requirement list # This should be synced with `runtime` in dependencies.yaml @@ -95,7 +95,7 @@ outputs: - docker-py =5.0.* - elasticsearch ==8.9.0 - feedparser =6.0.* - - grpcio =1.59.* + - grpcio =1.62.* - libwebp>=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 - mlflow>=2.10.0,<3 - mrc @@ -106,8 +106,6 @@ outputs: - python - python-confluent-kafka >=1.9.2,<1.10.0a0 - python-graphviz - - pytorch-cuda - - pytorch * *cuda* - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed - requests - requests-cache =1.1.* diff --git a/ci/runner/Dockerfile b/ci/runner/Dockerfile index 40b035c402..e3d7347268 100644 --- a/ci/runner/Dockerfile +++ b/ci/runner/Dockerfile @@ -16,8 +16,8 @@ # Args used in FROM commands must come first ARG FROM_IMAGE="rapidsai/ci-conda" ARG CUDA_PKG_VER=12-0 -ARG CUDA_SHORT_VER=12.1 -ARG CUDA_VER=12.1.1 +ARG CUDA_SHORT_VER=12.5 +ARG CUDA_VER=12.5.1 ARG LINUX_DISTRO=ubuntu ARG LINUX_VER=22.04 ARG PROJ_NAME=morpheus diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index ce4311b9a6..4a2bbe3f23 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -58,7 +58,7 @@ GIT_BRANCH=$(git branch --show-current) GIT_COMMIT=$(git log -n 1 --pretty=format:%H) LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp} -CONTAINER_VER=${CONTAINER_VER:-240614} +CONTAINER_VER=${CONTAINER_VER:-241004} CUDA_VER=${CUDA_VER:-12.1} DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index db88a38401..b4f428c64e 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -41,13 +41,11 @@ rapids_find_package(ZLIB if(MORPHEUS_BUILD_BENCHMARKS) # google benchmark - # - Expects package to pre-exist in the build environment # ================ - rapids_find_package(benchmark REQUIRED - GLOBAL_TARGETS benchmark::benchmark + include(${rapids-cmake-dir}/cpm/gbench.cmake) + rapids_cpm_gbench( BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports - FIND_ARGS CONFIG ) endif() @@ -65,13 +63,11 @@ morpheus_utils_configure_glog() if(MORPHEUS_BUILD_TESTS) # google test - # - Expects package to pre-exist in the build environment # =========== - rapids_find_package(GTest REQUIRED - GLOBAL_TARGETS GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main + include(${rapids-cmake-dir}/cpm/gtest.cmake) + rapids_cpm_gtest( BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports - FIND_ARGS CONFIG ) endif() diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml similarity index 76% rename from conda/environments/all_cuda-121_arch-x86_64.yaml rename to conda/environments/all_cuda-125_arch-x86_64.yaml index 201b487b98..0281cf6b68 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -22,18 +22,18 @@ dependencies: - clangdev=16 - click>=8 - cmake=3.27 -- cuda-cudart-dev=12.1 -- cuda-cudart=12.1 -- cuda-nvcc=12.1 -- cuda-nvml-dev=12.1 -- cuda-nvrtc-dev=12.1 -- cuda-nvrtc=12.1 -- cuda-nvtx-dev=12.1 -- cuda-nvtx=12.1 -- cuda-tools=12.1 -- cuda-version=12.1 -- cudf=24.02 -- cuml=24.02.* +- cuda-cudart-dev=12.5 +- cuda-cudart=12.5 +- cuda-nvcc=12.5 +- cuda-nvml-dev=12.5 +- cuda-nvrtc-dev=12.5 +- cuda-nvrtc=12.5 +- cuda-nvtx-dev=12.5 +- cuda-nvtx=12.5 +- cuda-sanitizer-api +- cuda-version=12.5 +- cudf=24.10 +- cuml=24.10.* - cupy - cxx-compiler - cython=3.0 @@ -45,23 +45,28 @@ dependencies: - exhale=0.3.6 - feedparser=6.0 - flake8 -- gcc_linux-64=11.2 - git-lfs -- glog=0.6 -- grpcio-status=1.59 -- grpcio=1.59 -- gxx_linux-64=11.2 +- glog>=0.7.1,<0.8 +- grpcio +- grpcio-status +- gtest=1.14 +- gxx=12.1 - huggingface_hub=0.20.2 - include-what-you-use=0.20 - ipython - isort - jsonpatch>=1.33 - kfp -- libcudf=24.02 +- libcublas-dev +- libcudf=24.10 +- libcufft-dev +- libcurand-dev +- libcusolver-dev - librdkafka>=1.9.2,<1.10.0a0 - libtool - libwebp=1.3.2 -- mlflow>=2.10.0,<3 +- libzlib >=1.3.1,<2 +- mlflow - mrc=24.10 - myst-parser=0.18.1 - nbsphinx @@ -81,6 +86,7 @@ dependencies: - pre-commit - pybind11-stubgen=0.10.5 - pydantic +- pylibcudf=24.10 - pylint=3.0.3 - pypdf=3.17.4 - pypdfium2=4.30 @@ -92,22 +98,20 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 -- pytorch-cuda -- pytorch=*=*cuda* - rapidjson=1.1.0 -- rapids-dask-dependency=24.02 +- rapids-dask-dependency=24.10 - rdma-core>=48 - requests - requests-cache=1.1 - requests-toolbelt=1.0 -- s3fs=2023.12.2 +- s3fs - scikit-build=0.17.6 - scikit-learn=1.3.2 - sentence-transformers=2.7 - sphinx - sphinx_rtd_theme - sqlalchemy<2.0 -- sysroot_linux-64=2.17 +- sysroot_linux-64>=2.28 - tqdm=4 - transformers=4.36.2 - tritonclient=2.34 @@ -120,15 +124,15 @@ dependencies: - watchdog=3.0 - websockets - yapf=0.40.1 -- zlib=1.2.13 - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - databricks-cli < 0.100 - databricks-connect - dgl==2.0.0 - dglgo - - faiss-gpu==1.7.* + - faiss-cpu - google-search-results==2.4 - langchain-nvidia-ai-endpoints==0.0.11 - langchain==0.1.16 @@ -136,4 +140,5 @@ dependencies: - nemollm==0.3.5 - pymilvus==2.3.6 - pytest-kafka==0.6.0 -name: all_cuda-121_arch-x86_64 + - torch==2.4.0+cu124 +name: all_cuda-125_arch-x86_64 diff --git a/conda/environments/dev_cuda-121_arch-x86_64.yaml b/conda/environments/dev_cuda-125_arch-x86_64.yaml similarity index 74% rename from conda/environments/dev_cuda-121_arch-x86_64.yaml rename to conda/environments/dev_cuda-125_arch-x86_64.yaml index 2b00aaeab2..af599fb7de 100644 --- a/conda/environments/dev_cuda-121_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-125_arch-x86_64.yaml @@ -19,14 +19,14 @@ dependencies: - clangdev=16 - click>=8 - cmake=3.27 -- cuda-cudart-dev=12.1 -- cuda-nvcc=12.1 -- cuda-nvml-dev=12.1 -- cuda-nvrtc-dev=12.1 -- cuda-nvtx-dev=12.1 -- cuda-tools=12.1 -- cuda-version=12.1 -- cudf=24.02 +- cuda-cudart-dev=12.5 +- cuda-nvcc=12.5 +- cuda-nvml-dev=12.5 +- cuda-nvrtc-dev=12.5 +- cuda-nvtx-dev=12.5 +- cuda-sanitizer-api +- cuda-version=12.5 +- cudf=24.10 - cupy - cxx-compiler - cython=3.0 @@ -38,20 +38,25 @@ dependencies: - exhale=0.3.6 - feedparser=6.0 - flake8 -- gcc_linux-64=11.2 - git-lfs -- glog=0.6 -- grpcio-status=1.59 -- grpcio=1.59 -- gxx_linux-64=11.2 +- glog>=0.7.1,<0.8 +- grpcio +- grpcio-status +- gtest=1.14 +- gxx=12.1 - include-what-you-use=0.20 - ipython - isort -- libcudf=24.02 +- libcublas-dev +- libcudf=24.10 +- libcufft-dev +- libcurand-dev +- libcusolver-dev - librdkafka>=1.9.2,<1.10.0a0 - libtool - libwebp=1.3.2 -- mlflow>=2.10.0,<3 +- libzlib >=1.3.1,<2 +- mlflow - mrc=24.10 - myst-parser=0.18.1 - nbsphinx @@ -66,6 +71,7 @@ dependencies: - pre-commit - pybind11-stubgen=0.10.5 - pydantic +- pylibcudf=24.10 - pylint=3.0.3 - pypdfium2=4.30 - pytest-asyncio @@ -76,10 +82,8 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 -- pytorch-cuda -- pytorch=*=*cuda* - rapidjson=1.1.0 -- rapids-dask-dependency=24.02 +- rapids-dask-dependency=24.10 - rdma-core>=48 - requests - requests-cache=1.1 @@ -88,7 +92,7 @@ dependencies: - sphinx - sphinx_rtd_theme - sqlalchemy<2.0 -- sysroot_linux-64=2.17 +- sysroot_linux-64>=2.28 - tqdm=4 - tritonclient=2.34 - typing_utils=0.1 @@ -100,11 +104,12 @@ dependencies: - watchdog=3.0 - websockets - yapf=0.40.1 -- zlib=1.2.13 - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 - pymilvus==2.3.6 - pytest-kafka==0.6.0 -name: dev_cuda-121_arch-x86_64 + - torch==2.4.0+cu124 +name: dev_cuda-125_arch-x86_64 diff --git a/conda/environments/examples_cuda-121_arch-x86_64.yaml b/conda/environments/examples_cuda-125_arch-x86_64.yaml similarity index 85% rename from conda/environments/examples_cuda-121_arch-x86_64.yaml rename to conda/environments/examples_cuda-125_arch-x86_64.yaml index 66b30db78e..ffcae28e4a 100644 --- a/conda/environments/examples_cuda-121_arch-x86_64.yaml +++ b/conda/environments/examples_cuda-125_arch-x86_64.yaml @@ -15,21 +15,21 @@ dependencies: - beautifulsoup4=4.12 - boto3 - click>=8 -- cudf=24.02 -- cuml=24.02.* +- cudf=24.10 +- cuml=24.10.* - cupy - datacompy=0.10 - dill=0.3.7 - docker-py=5.0 - elasticsearch==8.9.0 - feedparser=6.0 -- grpcio-status=1.59 -- grpcio=1.59 +- grpcio +- grpcio-status - huggingface_hub=0.20.2 - jsonpatch>=1.33 - kfp - libwebp=1.3.2 -- mlflow>=2.10.0,<3 +- mlflow - mrc=24.10 - networkx=2.8.8 - newspaper3k=0.2 @@ -48,13 +48,11 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 -- pytorch-cuda -- pytorch=*=*cuda* -- rapids-dask-dependency=24.02 +- rapids-dask-dependency=24.10 - requests - requests-cache=1.1 - requests-toolbelt=1.0 -- s3fs=2023.12.2 +- s3fs - scikit-learn=1.3.2 - sentence-transformers=2.7 - sqlalchemy<2.0 @@ -65,17 +63,19 @@ dependencies: - watchdog=3.0 - websockets - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - databricks-cli < 0.100 - databricks-connect - dgl==2.0.0 - dglgo - - faiss-gpu==1.7.* + - faiss-cpu - google-search-results==2.4 - langchain-nvidia-ai-endpoints==0.0.11 - langchain==0.1.16 - milvus==2.3.5 - nemollm==0.3.5 - pymilvus==2.3.6 -name: examples_cuda-121_arch-x86_64 + - torch==2.4.0+cu124 +name: examples_cuda-125_arch-x86_64 diff --git a/conda/environments/model-utils_cuda-121_arch-x86_64.yaml b/conda/environments/model-utils_cuda-125_arch-x86_64.yaml similarity index 88% rename from conda/environments/model-utils_cuda-121_arch-x86_64.yaml rename to conda/environments/model-utils_cuda-125_arch-x86_64.yaml index 761f19aaa0..2957c36473 100644 --- a/conda/environments/model-utils_cuda-121_arch-x86_64.yaml +++ b/conda/environments/model-utils_cuda-125_arch-x86_64.yaml @@ -9,7 +9,7 @@ channels: - nvidia/label/dev - pytorch dependencies: -- cuml=24.02.* +- cuml=24.10.* - jupyterlab - matplotlib - onnx @@ -20,4 +20,4 @@ dependencies: - seqeval=1.2.2 - transformers=4.36.2 - xgboost -name: model-utils_cuda-121_arch-x86_64 +name: model-utils_cuda-125_arch-x86_64 diff --git a/conda/environments/runtime_cuda-121_arch-x86_64.yaml b/conda/environments/runtime_cuda-125_arch-x86_64.yaml similarity index 75% rename from conda/environments/runtime_cuda-121_arch-x86_64.yaml rename to conda/environments/runtime_cuda-125_arch-x86_64.yaml index ea6a442b3a..2551739061 100644 --- a/conda/environments/runtime_cuda-121_arch-x86_64.yaml +++ b/conda/environments/runtime_cuda-125_arch-x86_64.yaml @@ -12,21 +12,21 @@ dependencies: - appdirs - beautifulsoup4=4.12 - click>=8 -- cuda-cudart=12.1 -- cuda-nvrtc=12.1 -- cuda-nvtx=12.1 -- cuda-version=12.1 -- cudf=24.02 +- cuda-cudart=12.5 +- cuda-nvrtc=12.5 +- cuda-nvtx=12.5 +- cuda-version=12.5 +- cudf=24.10 - cupy - datacompy=0.10 - dill=0.3.7 - docker-py=5.0 - elasticsearch==8.9.0 - feedparser=6.0 -- grpcio-status=1.59 -- grpcio=1.59 +- grpcio +- grpcio-status - libwebp=1.3.2 -- mlflow>=2.10.0,<3 +- mlflow - mrc=24.10 - networkx=2.8.8 - numpydoc=1.5 @@ -36,9 +36,7 @@ dependencies: - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz - python=3.10 -- pytorch-cuda -- pytorch=*=*cuda* -- rapids-dask-dependency=24.02 +- rapids-dask-dependency=24.10 - requests - requests-cache=1.1 - scikit-learn=1.3.2 @@ -49,8 +47,10 @@ dependencies: - watchdog=3.0 - websockets - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 - pymilvus==2.3.6 -name: runtime_cuda-121_arch-x86_64 + - torch==2.4.0+cu124 +name: runtime_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index e8f5525696..5c2eb2a5b2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -14,12 +14,13 @@ # limitations under the License. # Dependency list for https://github.com/rapidsai/dependency-file-generator + files: # Includes all dependencies together in a single file all: output: conda matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - benchmark_cpp @@ -45,7 +46,7 @@ files: dev: output: conda matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - benchmark_cpp @@ -67,7 +68,7 @@ files: build: output: none matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - benchmark_cpp @@ -87,7 +88,7 @@ files: test: output: none matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - benchmark_cpp @@ -108,7 +109,7 @@ files: docs: output: none matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - benchmark_cpp @@ -131,7 +132,7 @@ files: runtime: output: conda matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - cudatoolkit @@ -145,7 +146,7 @@ files: examples: output: conda matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - cve-mitigation @@ -161,7 +162,7 @@ files: model-utils: output: conda matrix: - cuda: ["12.1"] + cuda: ["12.5"] arch: [x86_64] includes: - model-training-tuning @@ -196,26 +197,30 @@ dependencies: - output_types: [conda] matrices: - matrix: - cuda: "12.1" + cuda: "12.5" packages: - - cuda-cudart=12.1 - - cuda-nvrtc=12.1 - - cuda-nvtx=12.1 - - cuda-version=12.1 + - cuda-cudart=12.5 + - cuda-nvrtc=12.5 + - cuda-nvtx=12.5 + - cuda-version=12.5 cudatoolkit-dev: specific: - output_types: [conda] matrices: - matrix: - cuda: "12.1" + cuda: "12.5" packages: - - cuda-cudart-dev=12.1 - - cuda-nvml-dev=12.1 - - cuda-nvrtc-dev=12.1 - - cuda-nvtx-dev=12.1 - - cuda-tools=12.1 - - cuda-version=12.1 + - cuda-cudart-dev=12.5 + - cuda-nvml-dev=12.5 + - cuda-nvrtc-dev=12.5 + - cuda-nvtx-dev=12.5 + - cuda-sanitizer-api + - cuda-version=12.5 + - libcublas-dev # required by matx + - libcufft-dev # required by matx + - libcusolver-dev # required by matx + - libcurand-dev # required by matx @@ -226,22 +231,21 @@ dependencies: - output_types: [conda] packages: # Compilers - - cuda-nvcc=12.1 + - cuda-nvcc=12.5 - cxx-compiler - - gcc_linux-64=11.2 - - gxx_linux-64=11.2 + - gxx=12.1 # Non-Compiler Dependencies - automake=1.16.5 # Needed for DOCA build - c-ares=1.32 # 1.33 causes an undefined symbol error - ccache - cmake=3.27 - - cuda-cudart-dev=12.1 - - cuda-version=12.1 + - cuda-cudart-dev=12.5 + - cuda-version=12.5 - libtool # Needed for DOCA build - ninja=1.11 - pkg-config=0.29 # for mrc cmake - - sysroot_linux-64=2.17 + - sysroot_linux-64>=2.28 # Build dependencies for Morpheus on the host arch. Mirrors the `host` section in # ci/conda/recipes/morpheus/meta.yaml @@ -250,19 +254,21 @@ dependencies: - output_types: [conda] packages: # Include: cudatoolkit-dev - - cudf=24.02 + - cudf=24.10 - cython=3.0 - - glog=0.6 - - libcudf=24.02 + - glog>=0.7.1,<0.8 + - gtest=1.14 + - libcudf=24.10 - librdkafka>=1.9.2,<1.10.0a0 + - libzlib >=1.3.1,<2 - mrc=24.10 - nlohmann_json=3.11 - pybind11-stubgen=0.10.5 + - pylibcudf=24.10 - rapidjson=1.1.0 - rdma-core>=48 # Needed for DOCA. - scikit-build=0.17.6 - versioneer-518 - - zlib=1.2.13 checks: common: @@ -323,29 +329,26 @@ dependencies: - appdirs - beautifulsoup4=4.12 - click>=8 - # - cuda-version=12.1 ## - - cudf=24.02 + # - cuda-version=12.5 ## + - cudf=24.10 - cupy # Version determined from cudf - datacompy=0.10 - dill=0.3.7 - docker-py=5.0 - elasticsearch==8.9.0 - feedparser=6.0 - - grpcio=1.59 - - grpcio-status=1.59 + - grpcio + - grpcio-status # - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 ## - - mlflow>=2.10.0,<3 + - mlflow #>=2.10.0,<3 - mrc=24.10 - networkx=2.8.8 - numpydoc=1.5 - pydantic - # - python ## - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz - - pytorch-cuda - - pytorch=*=*cuda* - pluggy=1.3 - - rapids-dask-dependency=24.02 # provides dask and distributed + - rapids-dask-dependency=24.10 # provides dask and distributed - requests - requests-cache=1.1 - scikit-learn=1.3.2 @@ -357,10 +360,12 @@ dependencies: - websockets - pip - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 # update to match pymilvus when available - pymilvus==2.3.6 + - torch==2.4.0+cu124 test_python_morpheus: common: @@ -385,13 +390,13 @@ dependencies: - boto3 - kfp - papermill=2.4.0 - - s3fs=2023.12.2 + - s3fs example-gnn: common: - output_types: [conda] packages: - - &cuml cuml=24.02.* + - &cuml cuml=24.10.* - pip - pip: - --find-links https://data.dgl.ai/wheels/cu121/repo.html @@ -421,7 +426,7 @@ dependencies: - pip: - langchain==0.1.16 - langchain-nvidia-ai-endpoints==0.0.11 - - faiss-gpu==1.7.* + - faiss-cpu - google-search-results==2.4 - nemollm==0.3.5 diff --git a/docs/README.md b/docs/README.md index 469303430e..4fe4c43e58 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,7 +22,7 @@ Additional packages required for building the documentation are defined in `./co ## Install Additional Dependencies From the root of the Morpheus repo: ```bash -conda env update --solver=libmamba -n morpheus --file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune +conda env update --solver=libmamba -n morpheus --file conda/environments/dev_cuda-125_arch-x86_64.yaml --prune ``` ## Build Morpheus and Documentation diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index ea1ecbaa34..b31edbfc64 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -177,7 +177,7 @@ git submodule update --init --recursive 1. Create the Morpheus Conda environment ```bash - conda env create --solver=libmamba -n morpheus --file conda/environments/dev_cuda-121_arch-x86_64.yaml + conda env create --solver=libmamba -n morpheus --file conda/environments/dev_cuda-125_arch-x86_64.yaml conda activate morpheus ``` diff --git a/examples/developer_guide/3_simple_cpp_stage/README.md b/examples/developer_guide/3_simple_cpp_stage/README.md index 6e62534325..51573b0ad4 100644 --- a/examples/developer_guide/3_simple_cpp_stage/README.md +++ b/examples/developer_guide/3_simple_cpp_stage/README.md @@ -21,5 +21,5 @@ limitations under the License. |-------------|-----------|-------| | Conda | ✔ | | | Morpheus Docker Container | ✔ | | -| Morpheus Release Container | ✔ | Requires adding development packages to the container's Conda environment via `conda env update --solver=libmamba -n morpheus --file /workspace/conda/environments/dev_cuda-121_arch-x86_64.yaml` | +| Morpheus Release Container | ✔ | Requires adding development packages to the container's Conda environment via `conda env update --solver=libmamba -n morpheus --file /workspace/conda/environments/dev_cuda-125_arch-x86_64.yaml` | | Dev Container | ✔ | | diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/README.md b/examples/developer_guide/4_rabbitmq_cpp_stage/README.md index 1fba854fde..988381e1c6 100644 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/README.md +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/README.md @@ -25,7 +25,7 @@ This example adds two flags to the `read_simple.py` script. A `--use_cpp` flag w |-------------|-----------|-------| | Conda | ✔ | | | Morpheus Docker Container | ✔ | Requires launching the RabbitMQ container on the host | -| Morpheus Release Container | ✔ | Requires launching the RabbitMQ container on the host, and adding development packages to the container's Conda environment via `conda env update --solver=libmamba -n morpheus --file /workspace/conda/environments/dev_cuda-121_arch-x86_64.yaml` | +| Morpheus Release Container | ✔ | Requires launching the RabbitMQ container on the host, and adding development packages to the container's Conda environment via `conda env update --solver=libmamba -n morpheus --file /workspace/conda/environments/dev_cuda-125_arch-x86_64.yaml` | | Dev Container | ✘ | | ## Installing Pika diff --git a/examples/digital_fingerprinting/production/Dockerfile b/examples/digital_fingerprinting/production/Dockerfile index 8b9a884da0..d2e330c944 100644 --- a/examples/digital_fingerprinting/production/Dockerfile +++ b/examples/digital_fingerprinting/production/Dockerfile @@ -28,7 +28,7 @@ WORKDIR /workspace/examples/digital_fingerprinting/ # Install DFP dependencies RUN source activate morpheus \ - && /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file /workspace/conda/environments/examples_cuda-121_arch-x86_64.yaml + && /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file /workspace/conda/environments/examples_cuda-125_arch-x86_64.yaml # Set the tracking URI for mlflow ENV MLFLOW_TRACKING_URI="http://mlflow:5000" diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md index b002e7fa47..e43755094e 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md @@ -47,7 +47,7 @@ Install additional required dependencies: ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-125_arch-x86_64.yaml ``` diff --git a/examples/doca/vdb_realtime/README.md b/examples/doca/vdb_realtime/README.md index 6633957106..64dabdb459 100644 --- a/examples/doca/vdb_realtime/README.md +++ b/examples/doca/vdb_realtime/README.md @@ -98,7 +98,7 @@ export NGC_API_KEY="" Then install basic requirements: ```bash -conda env update --solver=libmamba -n morpheus --file conda/environments/examples_cuda-121_arch-x86_64.yaml --prune +conda env update --solver=libmamba -n morpheus --file conda/environments/examples_cuda-125_arch-x86_64.yaml --prune ``` Run the RAG example to query the Milvus database: diff --git a/examples/gnn_fraud_detection_pipeline/README.md b/examples/gnn_fraud_detection_pipeline/README.md index 8bb1ab1570..3945eced97 100644 --- a/examples/gnn_fraud_detection_pipeline/README.md +++ b/examples/gnn_fraud_detection_pipeline/README.md @@ -17,7 +17,7 @@ limitations under the License. # GNN Fraud Detection Pipeline ## Supported Environments -All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-121_arch-x86_64.yaml` or `conda/environments/examples_cuda-121_arch-x86_64.yaml` environment files. Refer to the [Requirements](#requirements) section for more information. +All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-125_arch-x86_64.yaml` or `conda/environments/examples_cuda-125_arch-x86_64.yaml` environment files. Refer to the [Requirements](#requirements) section for more information. | Environment | Supported | Notes | |-------------|-----------|-------| | Conda | ✔ | | @@ -32,7 +32,7 @@ Prior to running the GNN fraud detection pipeline, additional requirements must ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-125_arch-x86_64.yaml ``` ## Running diff --git a/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py b/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py index 013034dcef..3a5845f6e6 100644 --- a/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py +++ b/examples/gnn_fraud_detection_pipeline/stages/classification_stage.py @@ -46,7 +46,7 @@ def __init__(self, c: Config, model_xgb_file: str): super().__init__(c) - self._xgb_model = cuml.ForestInference.load(model_xgb_file, output_class=True) + self._xgb_model = cuml.ForestInference.load(model_xgb_file, output_class=True, model_type="xgboost") self._needed_columns.update({'node_id': TypeId.INT64, 'prediction': TypeId.FLOAT32}) @property diff --git a/examples/llm/agents/README.md b/examples/llm/agents/README.md index d8c2944b26..2721452a93 100644 --- a/examples/llm/agents/README.md +++ b/examples/llm/agents/README.md @@ -35,7 +35,7 @@ limitations under the License. - [Run example (Kafka Pipeline)](#run-example-kafka-pipeline) ## Supported Environments -All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-121_arch-x86_64.yaml` or `conda/environments/examples_cuda-121_arch-x86_64.yaml` environment files. Refer to the [Install Dependencies](#install-dependencies) section for more information. +All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-125_arch-x86_64.yaml` or `conda/environments/examples_cuda-125_arch-x86_64.yaml` environment files. Refer to the [Install Dependencies](#install-dependencies) section for more information. | Environment | Supported | Notes | |-------------|-----------|-------| | Conda | ✔ | | @@ -106,7 +106,7 @@ Install the required dependencies. ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-125_arch-x86_64.yaml ``` diff --git a/examples/llm/completion/README.md b/examples/llm/completion/README.md index c619546c47..e72ffe1ce6 100644 --- a/examples/llm/completion/README.md +++ b/examples/llm/completion/README.md @@ -31,7 +31,7 @@ limitations under the License. - [Running the Morpheus Pipeline](#running-the-morpheus-pipeline) ## Supported Environments -All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-121_arch-x86_64.yaml` or `conda/environments/examples_cuda-121_arch-x86_64.yaml` environment files. Refer to the [Install Dependencies](#install-dependencies) section for more information. +All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-125_arch-x86_64.yaml` or `conda/environments/examples_cuda-125_arch-x86_64.yaml` environment files. Refer to the [Install Dependencies](#install-dependencies) section for more information. | Environment | Supported | Notes | |-------------|-----------|-------| | Conda | ✔ | | @@ -80,7 +80,7 @@ Install the required dependencies. ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-125_arch-x86_64.yaml ``` diff --git a/examples/llm/rag/README.md b/examples/llm/rag/README.md index f21531ac05..f7c0863b5e 100644 --- a/examples/llm/rag/README.md +++ b/examples/llm/rag/README.md @@ -18,7 +18,7 @@ limitations under the License. # Retrieval Augmented Generation (RAG) Pipeline ## Supported Environments -All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-121_arch-x86_64.yaml` or `conda/environments/examples_cuda-121_arch-x86_64.yaml` environment files. This example also requires the [VDB upload](../vdb_upload/README.md) pipeline to have been run previously. +All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-125_arch-x86_64.yaml` or `conda/environments/examples_cuda-125_arch-x86_64.yaml` environment files. This example also requires the [VDB upload](../vdb_upload/README.md) pipeline to have been run previously. | Environment | Supported | Notes | |-------------|-----------|-------| | Conda | ✔ | | diff --git a/examples/llm/vdb_upload/README.md b/examples/llm/vdb_upload/README.md index 7348a9cde6..b892de2159 100644 --- a/examples/llm/vdb_upload/README.md +++ b/examples/llm/vdb_upload/README.md @@ -34,7 +34,7 @@ limitations under the License. - [Exporting and Deploying a Different Model from Hugging Face](#exporting-and-deploying-a-different-model-from-hugging-face) ## Supported Environments -All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-121_arch-x86_64.yaml` or `conda/environments/examples_cuda-121_arch-x86_64.yaml` environment files. +All environments require additional Conda packages which can be installed with either the `conda/environments/all_cuda-125_arch-x86_64.yaml` or `conda/environments/examples_cuda-125_arch-x86_64.yaml` environment files. | Environment | Supported | Notes | |-------------|-----------|-------| | Conda | ✔ | | diff --git a/external/utilities b/external/utilities index fb2c9503fb..85f8f7af2e 160000 --- a/external/utilities +++ b/external/utilities @@ -1 +1 @@ -Subproject commit fb2c9503fbfdd08503013f712b8bc1e4d9869933 +Subproject commit 85f8f7af2e8d9bc7bde978cd40c40297b1116957 diff --git a/manifest.yaml b/manifest.yaml index f81e333671..8646890c13 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -23,7 +23,7 @@ repos: -DMORPHEUS_PYTHON_INPLACE_BUILD=ON python: - name: morpheus - sub_dir: "" + sub_dir: python/morpheus git: host: github tag: branch-24.10 diff --git a/models/training-tuning-scripts/fraud-detection-models/README.md b/models/training-tuning-scripts/fraud-detection-models/README.md index 025d871ceb..219121b91a 100644 --- a/models/training-tuning-scripts/fraud-detection-models/README.md +++ b/models/training-tuning-scripts/fraud-detection-models/README.md @@ -26,7 +26,7 @@ Install packages for training GNN model. ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./conda/environments/model-utils-121_arch-x86_64.yaml + --file ./conda/environments/model-utils-125_arch-x86_64.yaml ``` ### Options for training and tuning models. diff --git a/python/morpheus/morpheus/_lib/cmake/libmorpheus.cmake b/python/morpheus/morpheus/_lib/cmake/libmorpheus.cmake index 62c33b96c4..7d07b41bbd 100644 --- a/python/morpheus/morpheus/_lib/cmake/libmorpheus.cmake +++ b/python/morpheus/morpheus/_lib/cmake/libmorpheus.cmake @@ -83,8 +83,9 @@ target_link_libraries(morpheus $<$:ZLIB::ZLIB> PUBLIC $ - cudf::cudf CUDA::nvtx3 + cudf::cudf + glog::glog mrc::pymrc RDKAFKA::RDKAFKA TritonClient::httpclient_static diff --git a/python/morpheus/morpheus/_lib/cudf_helpers.pyx b/python/morpheus/morpheus/_lib/cudf_helpers.pyx index 0940fd8f18..2345978b92 100644 --- a/python/morpheus/morpheus/_lib/cudf_helpers.pyx +++ b/python/morpheus/morpheus/_lib/cudf_helpers.pyx @@ -19,18 +19,32 @@ from cudf.core.dtypes import StructDtype from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector +from pylibcudf.libcudf.io.types cimport column_name_info +from pylibcudf.libcudf.io.types cimport table_metadata +from pylibcudf.libcudf.io.types cimport table_with_metadata +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.types cimport size_type from cudf._lib.column cimport Column -from cudf._lib.cpp.io.types cimport column_name_info -from cudf._lib.cpp.io.types cimport table_metadata -from cudf._lib.cpp.io.types cimport table_with_metadata -from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.cpp.types cimport size_type from cudf._lib.utils cimport data_from_unique_ptr -from cudf._lib.utils cimport get_column_names from cudf._lib.utils cimport table_view_from_table +cdef vector[string] get_column_names(object tbl, object index): + cdef vector[string] column_names + if index is not False: + if isinstance(tbl._index, cudf.core.multiindex.MultiIndex): + for idx_name in tbl._index.names: + column_names.push_back(str.encode(idx_name)) + else: + if tbl._index.name is not None: + column_names.push_back(str.encode(tbl._index.name)) + + for col_name in tbl._column_names: + column_names.push_back(str.encode(col_name)) + + return column_names + cdef extern from "morpheus/objects/table_info.hpp" namespace "morpheus" nogil: diff --git a/python/morpheus/morpheus/_lib/src/utilities/matx_util.cu b/python/morpheus/morpheus/_lib/src/utilities/matx_util.cu index a1dc626242..b5bf6c6b22 100644 --- a/python/morpheus/morpheus/_lib/src/utilities/matx_util.cu +++ b/python/morpheus/morpheus/_lib/src/utilities/matx_util.cu @@ -274,7 +274,7 @@ struct MatxUtil__MatxThreshold auto output_tensor = matx::make_tensor(static_cast(output_data), output_shape); // Convert max value to bool - (output_tensor = matx::rmax(input_tensor, {1}) > (InputT)threshold).run(stream.value()); + (output_tensor = matx::max(input_tensor, {1}) > (InputT)threshold).run(stream.value()); } /** @@ -362,7 +362,7 @@ struct MatxUtil__MatxReduceMax auto output_slice = output_tensor.template Slice<1>({output_idx, 0}, {matx::matxDropDim, matx::matxEnd}); - (output_slice = matx::rmax(input_slice.Permute({1, 0}))).run(stream.value()); + (output_slice = matx::max(input_slice.Permute({1, 0}))).run(stream.value()); } }; } // namespace diff --git a/python/morpheus/morpheus/_lib/tests/CMakeLists.txt b/python/morpheus/morpheus/_lib/tests/CMakeLists.txt index 0dab45cbc6..215b595576 100644 --- a/python/morpheus/morpheus/_lib/tests/CMakeLists.txt +++ b/python/morpheus/morpheus/_lib/tests/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(test_cuda target_link_libraries(test_cuda PRIVATE + glog::glog GTest::gtest GTest::gtest_main matx::matx diff --git a/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py b/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py index 2f81401e94..8bc1be6829 100644 --- a/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py +++ b/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py @@ -19,6 +19,8 @@ import urllib.parse import mlflow +import mlflow.types +import pandas as pd import requests from mlflow.exceptions import MlflowException from mlflow.models.signature import ModelSignature @@ -28,7 +30,7 @@ from mlflow.tracking import MlflowClient from mlflow.types import ColSpec from mlflow.types import Schema -from mlflow.types.utils import _infer_pandas_column +from mlflow.types.utils import _infer_pandas_column as _mlflow_infer_pandas_column from mlflow.types.utils import _infer_schema import cudf @@ -39,6 +41,19 @@ logger = logging.getLogger(__name__) +def _infer_pandas_column(col: pd.Series): + try: + return _mlflow_infer_pandas_column(col) + except MlflowException as ex: + # mlflow internally calls `pd.api.types.is_string_dtype(col)` which was changed in Pandas 2. + # passing the series such as `[None]` will return False when the dtype is string, so + # it is necessary to pass the dtype of the series rather than the series itself for backwards + # compatibility. + if "Failed to infer schema for pandas.Series" in ex.message and pd.api.types.is_string_dtype(col.dtype): + return mlflow.types.DataType.string + raise ex + + class MLFlowModelWriterController: """ Controller class for writing machine learning models to MLflow with optional permissions and configurations. diff --git a/python/morpheus/morpheus/parsers/event_parser.py b/python/morpheus/morpheus/parsers/event_parser.py index 998232c130..a82785b48a 100644 --- a/python/morpheus/morpheus/parsers/event_parser.py +++ b/python/morpheus/morpheus/parsers/event_parser.py @@ -98,7 +98,7 @@ def parse_raw_event(self, text: cudf.Series, event_regex: typing.Dict[str, any]) # Applies regex pattern for each expected output column to raw data for col in event_specific_columns: regex_pattern = event_regex.get(col) - extracted_gdf = text.str.extract(regex_pattern) + extracted_gdf = text.str.extract(regex_pattern).reset_index() if not extracted_gdf.empty: parsed_gdf[col] = extracted_gdf[0] diff --git a/python/morpheus/morpheus/parsers/ip.py b/python/morpheus/morpheus/parsers/ip.py index 814d46f9dd..1fcb75ee81 100644 --- a/python/morpheus/morpheus/parsers/ip.py +++ b/python/morpheus/morpheus/parsers/ip.py @@ -39,9 +39,9 @@ def ip_to_int(values): >>> ip.ip_to_int(cudf.Series(["192.168.0.1","10.0.0.1"])) 0 3232235521 1 167772161 - dtype: int64 + dtype: uint32 """ - return cudf.Series(values.str.ip2int()) + return values.str.ip2int() def int_to_ip(values): @@ -52,7 +52,7 @@ def int_to_ip(values): Parameters ---------- values : cudf.Series - Integer representations of IP addresses + uint32 representations of IP addresses Returns ------- @@ -63,12 +63,12 @@ def int_to_ip(values): -------- >>> import morpheus.parsers.ip as ip >>> import cudf - >>> ip.int_to_ip(cudf.Series([3232235521, 167772161])) + >>> ip.int_to_ip(cudf.Series([3232235521, 167772161], dtype=cudf.api.types.dtype("uint32"))) 0 192.168.0.1 1 10.0.0.1 dtype: object """ - return cudf.Series(values._column.int2ip()) + return cudf.Series._from_column(values._column.int2ip()) def is_ip(ips: str): diff --git a/python/morpheus/morpheus/stages/input/arxiv_source.py b/python/morpheus/morpheus/stages/input/arxiv_source.py index 0e987da229..1e24fb855f 100644 --- a/python/morpheus/morpheus/stages/input/arxiv_source.py +++ b/python/morpheus/morpheus/stages/input/arxiv_source.py @@ -37,7 +37,7 @@ IMPORT_ERROR_MESSAGE = ( "ArxivSource requires additional dependencies to be installed. Install them by running the following command: " "`conda env update --solver=libmamba -n morpheus" - "--file conda/environments/all_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/all_cuda-125_arch-x86_64.yaml --prune`") @register_stage("from-arxiv") @@ -47,7 +47,7 @@ class ArxivSource(PreallocatorMixin, SingleOutputSource): This stage requires several additional dependencies to be installed. Install them by running the following command: `conda env update --solver=libmamba -n morpheus " - "--file conda/environments/all_cuda-121_arch-x86_64.yaml --prune` + "--file conda/environments/all_cuda-125_arch-x86_64.yaml --prune` Parameters ---------- diff --git a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py index 97ead48749..7e62870138 100644 --- a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py @@ -281,8 +281,8 @@ def write_batch(msg: ControlMessage): out_df = cudf.DataFrame() out_df["dt"] = (df["timestamp"] - time0).astype(np.int32) - out_df["src"] = df["src_ip"].str.ip_to_int().astype(np.int32) - out_df["dst"] = df["dest_ip"].str.ip_to_int().astype(np.int32) + out_df["src"] = df["src_ip"].str.ip_to_int().astype(np.uint32) + out_df["dst"] = df["dest_ip"].str.ip_to_int().astype(np.uint32) out_df["lvl"] = df["secret_keys"].astype(np.int32) out_df["data"] = df["data"] diff --git a/python/morpheus_llm/morpheus_llm/error.py b/python/morpheus_llm/morpheus_llm/error.py index 2505d987dd..02a2694ed3 100644 --- a/python/morpheus_llm/morpheus_llm/error.py +++ b/python/morpheus_llm/morpheus_llm/error.py @@ -15,4 +15,4 @@ IMPORT_ERROR_MESSAGE = ( "{package} not found. Install it and other additional dependencies by running the following command:\n" "`conda env update --solver=libmamba -n morpheus " - "--file conda/environments/examples_cuda-121_arch-x86_64.yaml`") + "--file conda/environments/examples_cuda-125_arch-x86_64.yaml`") diff --git a/tests/_utils/__init__.py b/tests/_utils/__init__.py index c11afb671b..b116dffb69 100644 --- a/tests/_utils/__init__.py +++ b/tests/_utils/__init__.py @@ -25,7 +25,6 @@ import numpy as np import pytest - from morpheus.io.deserializers import read_file_to_df from .test_directories import TestDirectories diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 9aa0bd105a..148dbb3d44 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -200,5 +200,5 @@ You can use the same Dev container created here to run the Production DFP benchm ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-125_arch-x86_64.yaml ``` diff --git a/tests/conftest.py b/tests/conftest.py index 84f894f707..6eafa0d80c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,7 +49,7 @@ OPT_DEP_SKIP_REASON = ( "This test requires the {package} package to be installed, to install this run:\n" - "`conda env update --solver=libmamba -n morpheus --file conda/environments/examples_cuda-121_arch-x86_64.yaml`") + "`conda env update --solver=libmamba -n morpheus --file conda/environments/examples_cuda-125_arch-x86_64.yaml`") def pytest_addoption(parser: pytest.Parser): diff --git a/tests/dfencoder/test_autoencoder.py b/tests/dfencoder/test_autoencoder.py index 43a1b7574b..bd02907f92 100755 --- a/tests/dfencoder/test_autoencoder.py +++ b/tests/dfencoder/test_autoencoder.py @@ -503,4 +503,4 @@ def test_auto_encoder_num_only_convergence(train_ae: autoencoder.AutoEncoder): for loss in train_ae.logger.train_fts.values()], axis=0) / len(train_ae.logger.train_fts) # Make sure the model converges with numerical feats only - assert avg_loss[-1] < avg_loss[0] / 2 + assert avg_loss[-1] < avg_loss[0] * 0.75 diff --git a/tests/examples/llm/common/conftest.py b/tests/examples/llm/common/conftest.py index 259b535304..591ed21cba 100644 --- a/tests/examples/llm/common/conftest.py +++ b/tests/examples/llm/common/conftest.py @@ -60,5 +60,5 @@ def langchain_fixture(fail_missing: bool): skip_reason = ("Tests for the WebScraperStage require the langchain package to be installed, to install this run:\n" "`conda env update --solver=libmamba -n morpheus " - "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-125_arch-x86_64.yaml --prune`") yield import_or_skip("langchain", reason=skip_reason, fail_missing=fail_missing) diff --git a/tests/stages/arxiv/conftest.py b/tests/stages/arxiv/conftest.py index 0865faada0..6166cb1250 100644 --- a/tests/stages/arxiv/conftest.py +++ b/tests/stages/arxiv/conftest.py @@ -23,7 +23,7 @@ SKIP_REASON = ("Tests for the arxiv_source require a number of packages not installed in the Morpheus development " "environment. To install these run:\n" "`conda env update --solver=libmamba -n morpheus " - "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-125_arch-x86_64.yaml --prune`") @pytest.fixture(name="arxiv", autouse=True, scope='session') diff --git a/tests/test_ip.py b/tests/test_ip.py index a8875939b1..5899c810ae 100644 --- a/tests/test_ip.py +++ b/tests/test_ip.py @@ -19,13 +19,13 @@ def test_ip_to_int(): input_df = cudf.Series(["5.79.97.178", "94.130.74.45"]) - expected = cudf.Series([89088434, 1585596973]) + expected = cudf.Series([89088434, 1585596973], dtype=cudf.api.types.dtype("uint32")) actual = ip.ip_to_int(input_df) assert actual.equals(expected) def test_int_to_ip(): - input_df = cudf.Series([89088434, 1585596973]) + input_df = cudf.Series([89088434, 1585596973], dtype=cudf.api.types.dtype("uint32")) expected = cudf.Series(["5.79.97.178", "94.130.74.45"]) actual = ip.int_to_ip(input_df) assert actual.equals(expected)