Merge branch 'main' into ashors/llama-download-cmd

.github/workflows/cicd-main.yml

@@ -41,6 +41,7 @@ jobs:
     outputs:
       test_to_run: ${{ steps.test_to_run.outputs.main }}
       all: ${{ steps.all.outputs.main }}
+      run_ci: ${{ steps.evaluate.outputs.run_ci }}
     steps:
       - name: Parse test_to_run
         id: test_to_run
@@ -51,9 +52,42 @@ jobs:
         id: all
         run: |
           echo "main=${{ contains(fromJSON(steps.test_to_run.outputs.main), 'all') }}" | tee -a "$GITHUB_OUTPUT"
-  
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+          files_yaml: |
+            doc:
+              - '**.md'
+              - docs/**
+            src:
+              - '!**.md'
+              - '!docs/**'
+
+      - name: Evaluate conditions
+        id: evaluate
+        env:
+          DOCS_ONLY: ${{ steps.changed-files.outputs.doc_any_changed == 'true' && steps.changed-files.outputs.src_any_changed == 'false' }}
+          CHANGED_DOCS: ${{ steps.changed-files.outputs.doc_all_changed_files }}
+          CHANGED_SRC: ${{ steps.changed-files.outputs.src_all_changed_files }}
+          LABEL: ${{ github.event.label.name == 'Run CICD' }}
+          MAIN_BRANCH: ${{ github.ref == 'refs/heads/main' }}          
+        run: |
+          # Some output that's helpful for debugging
+          echo "Docs changed: $CHANGED_DOCS"
+          echo "Src changed: $CHANGED_SRC"
+          
+          echo "docs_only: $DOCS_ONLY"
+          echo "label: $LABEL"
+          echo "main_branch: $MAIN_BRANCH"
+          
+          # Run CI only (on main or if label is attached) and if it's not only docs
+          echo run_ci=$([[ ("$LABEL" = "true" || "$MAIN_BRANCH" = "true") && "$DOCS_ONLY" = "false" ]] && echo "true" || echo "false") | tee -a "$GITHUB_OUTPUT"
+
   build-container:
-    if: ${{ github.event.label.name == 'Run CICD' || github.ref == 'refs/heads/main' }}
+    if: ${{ needs.pre-flight.outputs.run_ci == 'true' }}
+    needs: [pre-flight]
     uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/[email protected]
     with:
       image-name: nemo_aligner_container
@@ -67,7 +101,7 @@ jobs:
     name: ${{ matrix.test_case }}
     needs: [build-container, pre-flight]
     uses: ./.github/workflows/_run_test.yml
-    if: contains(fromJSON(needs.pre-flight.outputs.test_to_run), 'unit') || needs.pre-flight.outputs.all == 'true'
+    if: ${{ needs.pre-flight.outputs.run_ci == 'true' }}
     strategy:
       matrix:
         test_case:
@@ -85,7 +119,7 @@ jobs:
     name: ${{ matrix.test_case }}
     needs: [build-container, pre-flight]
     uses: ./.github/workflows/_run_test.yml
-    if: contains(fromJSON(needs.pre-flight.outputs.test_to_run), 'functional') || needs.pre-flight.outputs.all == 'true'
+    if: ${{ needs.pre-flight.outputs.run_ci == 'true' }}
     strategy:
       matrix:
         test_case:
@@ -102,3 +136,24 @@ jobs:
       TIMEOUT: 8
       SCRIPT: |
         bash /opt/NeMo-Aligner/tests/functional/test_cases/${{ matrix.test_case }}
+
+  CI_QA_Gate:
+    name: CI quality check
+    if: always()
+    runs-on: ubuntu-latest
+    needs: 
+      - Unit_Tests
+      - Functional_Tests
+    steps:
+      - name: main
+        env:
+          JOB_RESULTS: ${{ toJSON(needs) }}
+          ALL_SUCCESS: ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'skipped') }}
+          CI_SKIP: ${{ github.event.label.name == 'Skip CICD' }}
+        run: |
+         
+          SUMMARY=$(echo $JOB_RESULTS | jq 'to_entries[] | .key + ": " + .value.result' | tr -d '"')
+          echo '🤖: CICD Result' >> $GITHUB_STEP_SUMMARY
+          echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY
+          
+          test "$ALL_SUCCESS" = "true" || test "$CI_SKIP" = "true"

.github/workflows/release-freeze.yml

@@ -3,20 +3,25 @@ name: "Code freeze"
 on:
   workflow_dispatch:
     inputs:
-      type_of_release:
+      release-type:
         type: choice
         description: Type of release
         options: 
         - major
         - minor
-
+      freeze-commit:
+        type: string
+        description: Commit SHA to use for cut-off
+        required: false
+        default: main
 jobs:
   code-freeze:
-    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_code_freeze.yml@v0.8.0
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_code_freeze.yml@v0.17.3
     with:
-      name_of_library: NeMo-Aligner
-      type_of_release: ${{ inputs.type_of_release }}
-      python_package: nemo_aligner
+      library-name: NeMo-Aligner
+      python-package: nemo_aligner
+      release-type: ${{ inputs.release-type }}
+      freeze-commit: ${{ inputs.freeze-commit }}
     secrets:
       SLACK_RELEASE_ENDPOINT: ${{ secrets.SLACK_RELEASE_ENDPOINT }}
       SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}

.github/workflows/release.yaml

@@ -25,10 +25,10 @@ on:
         required: true
         default: true
         type: boolean
-
+          
 jobs:
   release:
-    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_release_library.yml@v0.15.0
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_release_library.yml@v0.17.3
     with:
       release-ref: ${{ inputs.release-ref }}
       image-name: nemo_aligner_container
@@ -46,5 +46,5 @@ jobs:
       TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
       TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
       SLACK_RELEASE_ENDPOINT: ${{ secrets.SLACK_RELEASE_ENDPOINT }}
-      PAT: ${{ secrets.PAT }}
       SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+      PAT: ${{ secrets.PAT }}

docs/user-guide/dpo.rst

@@ -46,7 +46,7 @@ To start, we must first get a pretrained model to align. There are two models we
                   --in-folder ./model_checkpoint \
                   --out-file ./mcore_gpt.nemo
 
-    .. tab-item:: LLaMa3 7B
+    .. tab-item:: LLaMa3 8B
         :sync: key2
 
         #. Download the `Llama 3 8B LLM model and tokenizer <https://huggingface.co/meta-llama/Meta-Llama-3-8B>`__ into the models folder. You can use the Hugging Face CLI for this: