Update github workflow to build/upload the morpheus_dfp conda package

Signed-off-by: Anuradha Karuppiah <[email protected]>
nv-morpheus · Sep 23, 2024 · ff3837f · ff3837f
1 parent 620deba
commit ff3837f
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 7 deletions.
diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml
@@ -21,8 +21,8 @@ on:
       run_check:
         required: true
         type: boolean
-      conda_core_run_build:
-        description: 'Runs the stage to build the morpheus-core conda package'
+      conda_lib_build:
+        description: 'Runs the stage to build the conda packages for all morpheus libraries - core, dfp etc.'
         required: true
         type: boolean
       conda_upload_label:
@@ -206,7 +206,7 @@ jobs:
 
   package-core:
     name: Package Core
-    if: ${{ inputs.conda_core_run_build }}
+    if: ${{ inputs.conda_lib_build }}
     needs: [documentation, test]
     runs-on: linux-amd64-cpu16
     timeout-minutes: 60
@@ -242,6 +242,44 @@ jobs:
           CONDA_PKG_LABEL: "${{ inputs.conda_upload_label }}"
         run: ./morpheus/ci/scripts/github/conda_core.sh $SCRIPT_ARGS
 
+  package-dfp:
+    name: Package DFP library
+    if: ${{ inputs.conda_lib_build }}
+    needs: [package-core]
+    runs-on: linux-amd64-cpu16
+    timeout-minutes: 60
+    container:
+      credentials:
+        username: '$oauthtoken'
+        password: ${{ secrets.NGC_API_KEY }}
+      image: ${{ inputs.container }}
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: false
+          path: 'morpheus'
+          fetch-depth: 0
+          submodules: 'recursive'
+
+      - name: Get AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
+
+      - name: Build morpheus-dfp conda package
+        shell: bash
+        env:
+          CONDA_TOKEN: "${{ secrets.CONDA_TOKEN }}"
+          SCRIPT_ARGS: "${{ inputs.conda_upload_label != '' && 'upload' || '' }}"
+          CONDA_PKG_LABEL: "${{ inputs.conda_upload_label }}"
+        run: ./morpheus/ci/scripts/github/conda_dfp.sh $SCRIPT_ARGS
+
   package:
     name: Package All
     if: ${{ inputs.conda_run_build }}

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -68,7 +68,7 @@ jobs:
       is_main_branch: ${{ github.ref_name == 'main' }}
       is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }}
       has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }}
-      has_conda_core_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-core-build') || false }}
+      has_conda_lib_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-lib-build') || false }}
       has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }}
       pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }}
 
@@ -90,9 +90,9 @@ jobs:
     with:
       # Run checks for any PR branch
       run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }}
-      # Build morpheus-core conda package. This is done for main/dev branches and
-      # for PRs with the conda-core-build label
-      conda_core_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_core_build_label) }}
+      # Build conda packages for all the morpheus libraries - core, dfp, llm. This is done for
+      # main/dev branches and for PRs with the conda-lib-build label
+      conda_lib_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_lib_build_label) }}
       # Upload morpheus-core conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       # Run morpheus conda package, with all components. This is done for main/dev