chore: add Made for UDS badge verification task and workflow (#1149)

.github/scripts/uds_verification_report.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+
+import os
+import re
+
+
+def remove_ansi_escape_sequences(text):
+    ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
+    return ansi_escape.sub("", text)
+
+
+# Capabilities that affect the entire capability, not just a single package
+def uds_capability_wide_errors(text: str) -> bool:
+    if "Not all pods have the istio sidecar" in text:
+        return True
+    return False
+
+
+# CI environment variable enables GitHub annotations
+def print_package_info(
+    package_name,
+    failures_count,
+    errors_count,
+    warnings_count,
+    failure_descriptions,
+    error_descriptions,
+    warning_descriptions,
+    uds_capability_wide_errors_count,
+):
+    if uds_capability_wide_errors_count >= 1:
+        errors_count -= uds_capability_wide_errors_count
+    if package_name:
+        print("-----------------------------")
+        if os.getenv("CI") == "true":
+            print(f"::group::{package_name}")
+        print(f"Package: {package_name}\n")
+        if failures_count > 0:
+            if os.getenv("CI") == "true":
+                print("::error::", end="")
+            print(f"⛔ Failures: {failures_count}")
+        else:
+            if errors_count > 0:
+                if os.getenv("CI") == "true":
+                    print("::error::", end="")
+                print(f"❌ Errors: {errors_count}")
+            if warnings_count > 0:
+                if os.getenv("CI") == "true":
+                    print("::warning::", end="")
+                print(f"⚠️  Warnings: {warnings_count}")
+        if failures_count > 0:
+            print("\n⛔ Failure Descriptions:")
+            for desc in failure_descriptions:
+                print(f"  - {desc}")
+        else:
+            if errors_count > 0:
+                print("\n❌ Error Descriptions:")
+                for desc in error_descriptions:
+                    print(f"  - {desc}")
+            if warnings_count > 0:
+                print("\n⚠️  Warning Descriptions:")
+                for desc in warning_descriptions:
+                    print(f"  - {desc}")
+        if os.getenv("CI") == "true":
+            print("::endgroup::")
+
+
+def main():
+    # Read data from the specified file instead of stdin
+    file_path = os.path.join(
+        os.getenv("GITHUB_WORKSPACE", ""), "reports/intermediate-report.txt"
+    )
+    with open(file_path, mode="r", encoding="utf-8", errors="ignore") as file:
+        data = file.read()
+    # Remove ANSI escape sequences
+    clean_data = remove_ansi_escape_sequences(data)
+    # Initialize variables
+    package_name = ""
+    failures_count = 0
+    errors_count = 0
+    warnings_count = 0
+    uds_capability_wide_errors_count = 0
+    failure_descriptions = []
+    error_descriptions = []
+    warning_descriptions = []
+    uds_capability_wide_error_descriptions = []
+    previous_package_name = None
+
+    # Process each line
+    for line in clean_data.splitlines():
+        # Remove leading and trailing whitespace
+        line = line.strip()
+
+        # Match and extract the package name
+        match = re.match(r"^ℹ️\s+Package\s+Name:\s+(.*)$", line)
+        if match:
+            # Print the previous package's info before starting a new one
+            if previous_package_name is not None:
+                print_package_info(
+                    previous_package_name,
+                    failures_count,
+                    errors_count,
+                    warnings_count,
+                    failure_descriptions,
+                    error_descriptions,
+                    warning_descriptions,
+                    uds_capability_wide_errors_count,
+                )
+            # Reset variables for the new package
+            package_name = match.group(1)
+            failures_count = 0
+            errors_count = 0
+            warnings_count = 0
+            failure_descriptions = []
+            error_descriptions = []
+            warning_descriptions = []
+            previous_package_name = package_name
+            continue
+
+        if uds_capability_wide_errors(line):
+            uds_capability_wide_errors_count = 1
+            uds_capability_wide_error_descriptions = [
+                "Not all pods have the istio sidecar"
+            ]
+            continue
+        else:
+            # Match and extract counts for failures, errors, and warnings
+            match = re.match(r"^(❌|⚠️|⛔)\s+(\d+)\s+([a-z]+)\s+found$", line)
+            if match:
+                count = int(match.group(2))
+                type_ = match.group(3)
+                if type_ == "errors":
+                    errors_count = count
+                elif type_ == "warnings":
+                    warnings_count = count
+                elif type_ == "failures":
+                    failures_count = count
+                continue
+
+            # Match and collect issue descriptions
+            match = re.match(r"^(❌|⚠️|⛔)\s+(.*)$", line)
+            if match:
+                emoji = match.group(1)
+                description = match.group(2)
+                if emoji == "❌":
+                    error_descriptions.append(description)
+                elif emoji == "⚠️":
+                    warning_descriptions.append(description)
+                elif emoji == "⛔":
+                    failure_descriptions.append(description)
+                continue
+
+    # Print the last package's information
+    if previous_package_name is not None:
+        print_package_info(
+            previous_package_name,
+            failures_count,
+            errors_count,
+            warnings_count,
+            failure_descriptions,
+            error_descriptions,
+            warning_descriptions,
+            uds_capability_wide_errors_count,
+        )
+        if uds_capability_wide_errors_count >= 1:
+            print("-----------------------------")
+            if os.getenv("CI") == "true":
+                print("::group::UDS Capability-Wide Issues")
+                print("::error::", end="")
+            print("UDS Capability Issues")
+            print("\n❌ Error Descriptions:")
+            for desc in uds_capability_wide_error_descriptions:
+                print(f"  - {desc}")
+            if os.getenv("CI") == "true":
+                print("::endgroup::")
+
+
+if __name__ == "__main__":
+    main()
+    # Print the final ending separator
+    print("-----------------------------")

.github/workflows/e2e-llama-cpp-python.yaml

@@ -32,6 +32,7 @@ on:
 
       # Ignore local development files
       - "!.pre-commit-config.yaml"
+      - "!tasks.yaml"
 
       # Ignore non e2e tests changes
       - "!tests/pytest/**"

.github/workflows/e2e-playwright.yaml

@@ -34,6 +34,7 @@ on:
 
       # Ignore local development files
       - "!.pre-commit-config.yaml"
+      - "!tasks.yaml"
 
       # Ignore non e2e tests changes
       - "!tests/pytest/**"
@@ -120,7 +121,7 @@ jobs:
         - name: UI/API/Supabase E2E Playwright Tests
           run: |
             cp src/leapfrogai_ui/.env.example src/leapfrogai_ui/.env
-            rm src/leapfrogai_ui/tests/global.teardown.ts 
+            rm src/leapfrogai_ui/tests/global.teardown.ts
             mkdir -p src/leapfrogai_ui/playwright/.auth
             SERVICE_ROLE_KEY=$(uds zarf tools kubectl get secret -n leapfrogai supabase-bootstrap-jwt -o jsonpath={.data.service-key} | base64 -d)
             echo "::add-mask::$SERVICE_ROLE_KEY"

.github/workflows/e2e-text-backend-full-cpu.yaml

@@ -32,6 +32,7 @@ on:
 
       # Ignore local development files
       - "!.pre-commit-config.yaml"
+      - "!tasks.yaml"
 
       # Ignore non e2e tests changes
       - "!tests/pytest/**"

.github/workflows/e2e-text-embeddings.yaml

@@ -32,6 +32,7 @@ on:
 
       # Ignore local development files
       - "!.pre-commit-config.yaml"
+      - "!tasks.yaml"
 
       # Ignore non e2e tests changes
       - "!tests/pytest/**"

.github/workflows/e2e-vllm.yaml

@@ -32,6 +32,7 @@ on:
 
       # Ignore local development files
       - "!.pre-commit-config.yaml"
+      - "!tasks.yaml"
 
       # Ignore non e2e tests changes
       - "!tests/pytest/**"

.github/workflows/e2e-whisper.yaml

@@ -32,6 +32,7 @@ on:
 
       # Ignore local development files
       - "!.pre-commit-config.yaml"
+      - "!tasks.yaml"
 
       # Ignore non e2e tests changes
       - "!tests/pytest/**"

.github/workflows/nightly-uds-badge-verification.yaml

@@ -0,0 +1,94 @@
+name: nightly-uds-badge-verification
+
+on:
+  schedule:
+    - cron: "0 11 * * *" # Runs daily at 3 AM PST
+  workflow_dispatch: # trigger manually as needed
+  pull_request:
+    paths:
+      - .github/workflows/nightly-uds-badge-verification.yaml
+      - tasks.yaml
+
+concurrency:
+  group: nightly-uds-badge-verification-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  SNAPSHOT_VERSION: snapshot-latest
+
+permissions:
+  contents: read
+  packages: read
+  id-token: write # This is needed for OIDC federation.
+
+jobs:
+  uds-badge-verification:
+    runs-on: ai-ubuntu-big-boy-8-core
+    name: nightly_uds_badge_verification
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: main
+
+      - name: Setup UDS Cluster
+        uses: ./.github/actions/uds-cluster
+        with:
+          registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }}
+          registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }}
+          ghToken: ${{ secrets.GITHUB_TOKEN }}
+          chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }}
+
+      - name: Print the Commit SHA
+        run: |
+          COMMIT_SHA=$(git rev-parse HEAD)
+          echo "The latest commit on the main branch is: $COMMIT_SHA"
+
+      # Set UDS CPU bundle refs and repositories to snapshot-latest
+      - name: Mutation of the UDS Bundle
+        run: |
+          uds zarf tools yq -i '.metadata.version = "${{ env.SNAPSHOT_VERSION }}"' bundles/latest/cpu/uds-bundle.yaml
+
+          uds zarf tools yq -i '.packages[].ref |= sub("^[^ ]+-upstream$", "${{ env.SNAPSHOT_VERSION }}-upstream")' bundles/latest/cpu/uds-bundle.yaml
+
+          uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml
+
+      - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }})
+        run: |
+          cd bundles/latest/cpu
+          uds create . --confirm && \
+          uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \
+          rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \
+          docker system prune -af
+
+      # Workaround for handling emojis in the upstream badge verification UDS task
+      - name: Set Locale to UTF-8
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y locales
+          sudo locale-gen en_US.UTF-8
+          export LANG=en_US.UTF-8
+          export LANGUAGE=en_US:en
+          export LC_ALL=en_US.UTF-8
+
+      # Setup Python for the report cleaning script in the next step
+      - name: Set up Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Run UDS Badge Verification Task
+        run: |
+          uds run nightly-uds-badge-verification --no-progress
+
+      - name: Archive UDS Badge Verification Report
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        with:
+          name: uds-badge-verification-report
+          path: reports
+          retention-days: 7

.github/workflows/uds-lint.yaml

@@ -46,3 +46,11 @@ jobs:
         run: |
           check-jsonschema bundles/latest/gpu/uds-bundle.yaml --schemafile uds.schema.json
           check-jsonschema bundles/latest/cpu/uds-bundle.yaml --schemafile uds.schema.json
+
+      - name: Download UDS Tasks Schema
+        run: curl -o tasks.schema.json https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json
+
+      - name: Validate tasks.yaml
+        if: always()
+        run: |
+          check-jsonschema tasks.yaml --schemafile tasks.schema.json

.gitignore

@@ -34,6 +34,7 @@ node_modules
 package.json
 package-lock.json
 **/*.schema.json
+reports
 
 # local model and tokenizer files
 *.bin

.pre-commit-config.yaml

@@ -137,3 +137,26 @@ repos:
         files: "uds-bundle.yaml"
         types: [yaml]
         args: ["--schemafile", "uds-v0.14.0.schema.json"]
+
+  # UDS TASKS CHECK
+  - repo: local
+    hooks:
+      - id: download-schema
+        name: "Download UDS Tasks Schema"
+        entry: |
+          bash -c 'FILE="tasks-v0.14.0.schema.json"
+          if [ -f "$(git rev-parse --show-toplevel)/$FILE" ]; then
+            echo "$FILE already exists in the root of the git project, skipping download."
+          else
+            curl -o "$FILE" https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json
+          fi'
+        language: system
+
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.14.0
+    hooks:
+      - id: check-jsonschema
+        name: "Validate UDS Bundles Against Schema"
+        files: "tasks.yaml"
+        types: [yaml]
+        args: ["--schemafile", "tasks-v0.14.0.schema.json"]