-
Notifications
You must be signed in to change notification settings - Fork 29
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: update registry1 weekly to 0.13.1, API, and testing workflows (#…
…1101) Co-authored-by: Gregory Horvath <[email protected]>
- Loading branch information
1 parent
6880569
commit 022e98d
Showing
26 changed files
with
332 additions
and
186 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,8 @@ | ||
| name: e2e-registry1-weekly | ||
| name: weekly-registry1-flavor-test | ||
|
|
||
| on: | ||
| schedule: | ||
| - cron: "0 0 * * 6" # Run every Sunday at 12 AM EST | ||
| - cron: "0 8 * * 0" # Run every Sunday at 12 AM PST | ||
| workflow_dispatch: # trigger manually as needed | ||
| pull_request: | ||
| types: | ||
|
|
@@ -12,79 +12,110 @@ on: | |
| - ready_for_review # don't run on draft PRs | ||
| - milestoned # allows us to trigger on bot PRs | ||
| paths: | ||
| - .github/workflows/e2e-registry1-weekly.yaml | ||
| - .github/workflows/weekly-registry1-flavor-test.yaml | ||
| - bundles/latest/** | ||
|
|
||
| concurrency: | ||
| group: e2e-registry1-weekly-${{ github.ref }} | ||
| group: weekly-registry1-flavor-test-${{ github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| defaults: | ||
| run: | ||
| shell: bash | ||
|
|
||
| jobs: | ||
| test-flavors: | ||
| registry1-flavor-test: | ||
| runs-on: ai-ubuntu-big-boy-8-core | ||
| name: e2e_registry1_weekly | ||
| name: weekly_registry1_flavor_test | ||
| if: ${{ !github.event.pull_request.draft }} | ||
|
|
||
| permissions: | ||
| contents: read | ||
| packages: write | ||
| packages: read | ||
| id-token: write # This is needed for OIDC federation. | ||
|
|
||
| steps: | ||
| - name: Checkout Repo | ||
| # Checkout main just to see the latest release in the release-please manifest | ||
| - name: Checkout Repo (main) | ||
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | ||
| with: | ||
| # x-release-please-start-version | ||
| ref: "caf4f9c3093a55a003b49fcbf05c03221be6a232" # 0.12.2 w/ integration tests turned-on | ||
| # x-release-please-end | ||
| ref: main | ||
|
|
||
| - name: Setup Python | ||
| uses: ./.github/actions/python | ||
| - name: Get Latest Release Version | ||
| id: get_version | ||
| run: | | ||
| LFAI_VERSION=$(jq -r '.["."]' .github/.release-please-manifest.json) | ||
| echo "LFAI_VERSION=$LFAI_VERSION" >> $GITHUB_OUTPUT | ||
| - name: Install API and SDK Dev Dependencies | ||
| run : | | ||
| make install | ||
| ################ | ||
| # LATEST RELEASE | ||
| ################ | ||
|
|
||
| - name: Checkout Repo | ||
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | ||
| with: | ||
| fetch-tags: true | ||
| ref: v${{ steps.get_version.outputs.LFAI_VERSION }} | ||
|
|
||
| - name: Setup UDS Cluster | ||
| uses: ./.github/actions/uds-cluster | ||
| - name: Setup UDS Environment | ||
| uses: defenseunicorns/uds-common/.github/actions/setup@24c8a2a48eeb33773b76b3587c489cb17496c9e0 # v0.12.0 | ||
| with: | ||
| registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} | ||
| registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} | ||
| ghToken: ${{ secrets.GITHUB_TOKEN }} | ||
| udsCliVersion: 0.14.0 | ||
| chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} | ||
|
|
||
| - name: Create UDS Cluster | ||
| shell: bash | ||
| - name: Setup Python | ||
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c #v5.0.0 | ||
| with: | ||
| python-version-file: "pyproject.toml" | ||
|
|
||
| - name: Install Python Dependencies | ||
| run: pip install ".[dev]" "src/leapfrogai_api" "src/leapfrogai_sdk" --no-cache-dir | ||
|
|
||
| - name: Mutation of the Zarf Packages | ||
| run: | | ||
| UDS_CONFIG=.github/config/uds-config.yaml make create-uds-cpu-cluster | ||
| uds zarf tools yq -i ' | ||
| .components[].images[0] |= sub(":v[0-9\.]+$", ":v${{ steps.get_version.outputs.LFAI_VERSION }}") | ||
| ' packages/api/zarf.yaml | ||
| uds zarf tools yq -i '.api.image.tag = "v${{ steps.get_version.outputs.LFAI_VERSION }}"' packages/api/values/registry1-values.yaml | ||
| - name: Setup Playwright | ||
| - name: Print the Modified Zarf Packages | ||
| run: | | ||
| npm --prefix src/leapfrogai_ui ci | ||
| npx --prefix src/leapfrogai_ui playwright install | ||
| cat packages/api/zarf.yaml | ||
| cat packages/api/values/registry1-values.yaml | ||
| - name: Create Registry1 Packages | ||
| - name: Create Registry1 Zarf Packages | ||
| run: | | ||
| LOCAL_VERSION=registry1 FLAVOR=registry1 make build-api | ||
| uds zarf package create packages/api --set image_version="${{ steps.get_version.outputs.LFAI_VERSION }}" --flavor registry1 -a amd64 --confirm | ||
| # Mutate UDS bundle definition to use Registry1 packages | ||
| - name: Mutation to Registry1 Bundle | ||
| # TODO: fix bundle path | ||
| # Mutate non-Registry1 packages to be the current tagged version | ||
| - name: Mutation of the UDS Bundle | ||
| run: | | ||
| uds zarf tools yq -i '.packages[1] |= del(.repository)' bundles/latest/cpu/uds-bundle.yaml | ||
| uds zarf tools yq -i '.packages[1] |= .ref = "registry1"' bundles/latest/cpu/uds-bundle.yaml | ||
| uds zarf tools yq -i '.packages[1] |= .path = "../../../packages/api"' bundles/latest/cpu/uds-bundle.yaml | ||
| uds zarf tools yq -i '.metadata.version = "registry1"' bundles/latest/cpu/uds-bundle.yaml | ||
| - name: Create and Deploy Bundle | ||
| uds zarf tools yq -i '.packages[].ref |= sub("^[^ ]+-upstream$", "${{ steps.get_version.outputs.LFAI_VERSION }}-upstream")' bundles/latest/cpu/uds-bundle.yaml | ||
| uds zarf tools yq -i '.packages[1] |= del(.repository)' bundles/latest/cpu/uds-bundle.yaml | ||
| uds zarf tools yq -i '.packages[1] |= .ref = "${{ steps.get_version.outputs.LFAI_VERSION }}"' bundles/latest/cpu/uds-bundle.yaml | ||
| uds zarf tools yq -i '.packages[1] |= .path = "../../../"' bundles/latest/cpu/uds-bundle.yaml | ||
| - name: Print the Modified UDS Bundle | ||
| run: | | ||
| cat bundles/latest/cpu/uds-config.yaml | ||
| cat bundles/latest/cpu/uds-bundle.yaml | ||
| - name: Create UDS Cluster | ||
| shell: bash | ||
| run: | | ||
| UDS_CONFIG=.github/config/uds-config.yaml make create-uds-cpu-cluster | ||
| - name: Create and Deploy Registry1 Bundle | ||
| run: | | ||
| cd bundles/latest/cpu | ||
| uds create . --confirm && \ | ||
| uds deploy uds-bundle-leapfrogai-amd64-registry1.tar.zst --confirm --no-progress && \ | ||
| uds deploy uds-bundle-leapfrogai-amd64-registry1.tar.zst --confirm --no-progress --log-level debug && \ | ||
| rm -rf uds-bundle-leapfrogai-amd64-registry1.tar.zst && \ | ||
| docker system prune -af | ||
|
|
@@ -107,32 +138,19 @@ jobs: | |
| echo "ANON_KEY is set: ${{ steps.generate_secrets.outputs.ANON_KEY != '' }}" | ||
| echo "SERVICE_KEY is set: ${{ steps.generate_secrets.outputs.SERVICE_KEY != '' }}" | ||
| - name: Run Integration Tests | ||
| env: | ||
| SUPABASE_ANON_KEY: ${{ steps.generate_secrets.outputs.ANON_KEY }} | ||
| SUPABASE_PASS: ${{ steps.generate_secrets.outputs.FAKE_PASSWORD }} | ||
| SUPABASE_EMAIL: [email protected] | ||
| SUPABASE_URL: https://supabase-kong.uds.dev | ||
| # Turn off NIAH tests that are not applicable for integration testing using the Repeater model | ||
| LFAI_RUN_NIAH_TESTS: "false" | ||
| run: | | ||
| uds zarf connect --name=llama-cpp-python-model --namespace=leapfrogai --local-port=50051 --remote-port=50051 & | ||
| while ! nc -z localhost 50051; do sleep 1; done | ||
| make test-user-pipeline | ||
| env $(cat .env | xargs) python -m pytest -v -s tests/integration/api | ||
| # Backends | ||
| - name: Run Backend E2E Tests | ||
| env: | ||
| ANON_KEY: ${{ steps.generate_secrets.outputs.ANON_KEY }} | ||
| SERVICE_KEY: ${{ steps.generate_secrets.outputs.SERVICE_KEY }} | ||
| LEAPFROGAI_MODEL: llama-cpp-python | ||
| run: | | ||
| python -m pytest -vvv -s ./tests/e2e | ||
| - name: Setup Playwright | ||
| run: | | ||
| python -m pytest ./tests/e2e/test_llama.py -vv | ||
| python -m pytest ./tests/e2e/test_text_embeddings.py -vv | ||
| python -m pytest ./tests/e2e/test_whisper.py -vv | ||
| python -m pytest ./tests/e2e/test_supabase.py -vv | ||
| python -m pytest ./tests/e2e/test_api.py -vv | ||
| npm --prefix src/leapfrogai_ui ci | ||
| npx --prefix src/leapfrogai_ui playwright install | ||
| - name: Run Playwright E2E Tests | ||
| env: | ||
|
|
@@ -156,3 +174,12 @@ jobs: | |
| name: playwright-report | ||
| path: src/leapfrogai_ui/e2e-report/ | ||
| retention-days: 30 | ||
|
|
||
| - name: Get Cluster Debug Information | ||
| id: debug | ||
| if: ${{ !cancelled() }} | ||
| uses: defenseunicorns/uds-common/.github/actions/debug-output@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 | ||
|
|
||
| - name: Get Cluster Debug Information | ||
| if: ${{ !cancelled() && steps.debug.conclusion == 'success' }} | ||
| uses: defenseunicorns/uds-common/.github/actions/save-logs@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 | ||
Oops, something went wrong.