From 68c93d89f6c45281412bdecc990db56a382df222 Mon Sep 17 00:00:00 2001 From: Kevin DeShawn <126115026+KevDevSha@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:55:43 -0500 Subject: [PATCH] Use aciton.yaml and omit default runner (#18) * Use aciton.yaml and omit default runner * add latest as default tag * latest * ci_repo_ref * Update README.md * Update action.yaml * add env * add python version * test out auto ref and add description * revert * delete workflows * description * Update action.yaml * Update action.yaml * Update README.md * Required at top --- .github/actions/pytest-cpu/action.yaml | 105 ++++++++++++++++++++ .github/actions/pytest-gpu/action.yaml | 127 +++++++++++++++++++++++++ .github/workflows/pytest-cpu.yaml | 98 ------------------- .github/workflows/pytest-gpu.yaml | 117 ----------------------- README.md | 9 +- 5 files changed, 240 insertions(+), 216 deletions(-) create mode 100644 .github/actions/pytest-cpu/action.yaml create mode 100644 .github/actions/pytest-gpu/action.yaml delete mode 100644 .github/workflows/pytest-cpu.yaml delete mode 100644 .github/workflows/pytest-gpu.yaml diff --git a/.github/actions/pytest-cpu/action.yaml b/.github/actions/pytest-cpu/action.yaml new file mode 100644 index 0000000..5a8f4af --- /dev/null +++ b/.github/actions/pytest-cpu/action.yaml @@ -0,0 +1,105 @@ +name: Pytest CPU +inputs: + composer_package_name: + required: false + type: string + container: + required: true + type: string + name: + required: true + type: string + pip_deps: + required: true + type: string + pytest_command: + required: true + type: string + pytest_markers: + required: true + type: string + pytest_wandb_entity: + required: false + type: string + pytest_wandb_project: + required: false + type: string + safe_directory: + required: true + type: string + wandb_api_key: + required: false + description: "wandb API key - Pass in as repo secret" + aws_access_key_id: + required: false + description: "aws access key id - Pass in as repo secret" + aws_secret_access_key: + required: false + description: "aws secret access key - Pass in as repo secret" + slack_notifications_bot_token: + required: false + description: "Slack notifications bot token - Pass in as repo secret" + code_eval_device: + required: false + description: "CodeEval device - Pass in as repo secret" + code_eval_url: + required: false + description: "CodeEval URL - Pass in as repo secret" + code_eval_apikey: + required: false + description: "CodeEval API key - Pass in as repo secret" + gcs_key: + required: false + description: "GCS key - Pass in as repo secret" + gcs_secret: + required: false + description: "GCS secret - Pass in as repo secret" + azure_account_name: + required: false + description: "Azure account name - Pass in as repo secret" + azure_account_access_key: + required: false + description: "Azure account access key - Pass in as repo secret" +runs: + using: composite + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup + shell: bash + run: | + set -ex + export PATH=/composer-python:$PATH + export COMPOSER_PACKAGE_NAME='${{ inputs.composer_package_name }}' + python -m pip install --upgrade 'pip<23' wheel + python -m pip install --upgrade .${{ inputs.pip_deps }} + - name: Run Tests + id: tests + shell: bash + run: | + set -ex + export PATH=/composer-python:$PATH + export WANDB_API_KEY='${{ inputs.wandb_api_key }}' + export WANDB_ENTITY='${{ inputs.pytest_wandb_entity }}' + export WANDB_PROJECT='${{ inputs.pytest_wandb_project }}' + export AWS_ACCESS_KEY_ID='${{ inputs.aws_access_key_id }}' + export AWS_SECRET_ACCESS_KEY='${{ inputs.aws_secret_access_key }}' + export CODE_EVAL_DEVICE='${{ inputs.code_eval_device }}' + export CODE_EVAL_URL='${{ inputs.code_eval_url }}' + export CODE_EVAL_APIKEY='${{ inputs.code_eval_apikey }}' + export GCS_KEY='${{ inputs.gcs_key }}' + export GCS_SECRET='${{ inputs.gcs_secret }}' + export AZURE_ACCOUNT_NAME='${{ inputs.azure_account_name }}' + export AZURE_ACCOUNT_ACCESS_KEY='${{ inputs.azure_account_access_key }}' + export COMMON_ARGS="-v --durations=20 -m '${{ inputs.pytest_markers }}' -o tmp_path_retention_policy=failed" + + # Necessary to run git diff for doctests + git config --global --add safe.directory /__w/${{ inputs.safe_directory }}/${{ inputs.safe_directory }} + make test PYTEST='${{ inputs.pytest_command }}' EXTRA_ARGS="$COMMON_ARGS --codeblocks" + make test-dist PYTEST='${{ inputs.pytest_command }}' EXTRA_ARGS="$COMMON_ARGS" WORLD_SIZE=2 + + python -m coverage combine + - uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.sha }}-${{ inputs.name }} + path: .coverage diff --git a/.github/actions/pytest-gpu/action.yaml b/.github/actions/pytest-gpu/action.yaml new file mode 100644 index 0000000..9822e8d --- /dev/null +++ b/.github/actions/pytest-gpu/action.yaml @@ -0,0 +1,127 @@ +name: Pytest GPU +inputs: + name: + required: true + type: string + container: + required: true + type: string + gpu_num: + required: true + type: number + git_repo: + required: true + type: string + ci_repo_gpu_test_ref: + description: "CI repo version of the GPU tests to run" + required: true + type: string + pip_deps: + required: true + type: string + pytest_command: + required: true + type: string + pytest_markers: + required: true + type: string + mcloud_api_key: + required: true + description: "MosaicML API key - Pass in as repo secret" + python_version: + required: false + type: string + default: 3.9 + mcloud_timeout: + required: false + type: number + default: 2700 + composer_package_name: + required: false + type: string + gha_timeout: + required: false + type: number + default: 60 + git_ssh_clone: + required: false + type: boolean + default: false + slack_notifications_bot_token: + required: false + description: "Slack notifications bot token - Pass in as repo secret" + code_eval_device: + required: false + description: "CodeEval device - Pass in as repo secret" + code_eval_url: + required: false + description: "CodeEval URL - Pass in as repo secret" + code_eval_apikey: + required: false + description: "CodeEval API key - Pass in as repo secret" +runs: + using: composite + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + with: + repository: "mosaicml/ci-testing" + ref: ${{ inputs.ci_repo_gpu_test_ref }} + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: Setup MCLI + shell: bash + run: | + set -ex + python -m pip install mosaicml-cli + mcli version + - name: Submit Run + id: tests + shell: bash + env: + MOSAICML_API_KEY: ${{ inputs.mcloud_api_key }} + SLACK_NOTIFICATIONS_BOT_TOKEN: ${{ inputs.slack_notifications_bot_token }} + CODE_EVAL_DEVICE: ${{ inputs.code_eval_device }} + CODE_EVAL_URL: ${{ inputs.code_eval_url }} + CODE_EVAL_APIKEY: ${{ inputs.code_eval_apikey }} + run: | + set -ex + + PR_NUMBER="$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" + REF_ARGS="" + + # Use the PR number if it exists, commit SHA for protected branches and the branch name otherwise + if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then + if [[ "$GITHUB_REF" =~ "refs/heads/dev" || "$GITHUB_REF" =~ "refs/heads/main" || \ + "$GITHUB_REF" =~ "refs/heads/release" ]]; then + REF_ARGS="--git_commit $GITHUB_SHA" + else + REF_ARGS="--git_branch $GITHUB_REF_NAME" + fi + else + REF_ARGS="--pr_number $PR_NUMBER" + fi + + python .github/mcli/mcli_pytest.py \ + --image '${{ inputs.container }}' \ + --git_repo '${{ inputs.git_repo }}' \ + --pip_deps '${{ inputs.pip_deps }}' \ + --pip_package_name '${{ inputs.composer_package_name }}' \ + --pytest_markers '${{ inputs.pytest_markers }}' \ + --pytest_command '${{ inputs.pytest_command }}' \ + --timeout ${{ inputs.mcloud_timeout }} \ + --gpu_num ${{ inputs.gpu_num }} \ + --git_ssh_clone ${{ inputs.git_ssh_clone }} \ + ${REF_ARGS} diff --git a/.github/workflows/pytest-cpu.yaml b/.github/workflows/pytest-cpu.yaml deleted file mode 100644 index f36a156..0000000 --- a/.github/workflows/pytest-cpu.yaml +++ /dev/null @@ -1,98 +0,0 @@ -name: Pytest CPU -on: - workflow_call: - inputs: - composer_package_name: - required: false - type: string - container: - required: true - type: string - name: - required: true - type: string - pip_deps: - required: true - type: string - pytest-command: - required: true - type: string - pytest-markers: - required: true - type: string - pytest-wandb-entity: - required: false - type: string - pytest-wandb-project: - required: false - type: string - safe_directory: - required: true - type: string - secrets: - wandb-api-key: - required: false - aws-access-key-id: - required: false - aws-secret-access-key: - required: false - slack-notifications-bot-token: - required: false - code-eval-device: - required: false - code-eval-url: - required: false - code-eval-apikey: - required: false - gcs-key: - required: false - gcs-secret: - required: false - azure-account-name: - required: false - azure-account-access-key: - required: false -jobs: - pytest-cpu: - timeout-minutes: 30 - runs-on: linux-ubuntu-latest - container: ${{ inputs.container }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Setup - run: | - set -ex - export PATH=/composer-python:$PATH - export COMPOSER_PACKAGE_NAME='${{ inputs.composer_package_name }}' - python -m pip install --upgrade 'pip<23' wheel - python -m pip install --upgrade .${{ inputs.pip_deps }} - - name: Run Tests - id: tests - run: | - set -ex - export PATH=/composer-python:$PATH - export WANDB_API_KEY='${{ secrets.wandb-api-key }}' - export WANDB_ENTITY='${{ inputs.pytest-wandb-entity }}' - export WANDB_PROJECT='${{ inputs.pytest-wandb-project }}' - export AWS_ACCESS_KEY_ID='${{ secrets.aws-access-key-id }}' - export AWS_SECRET_ACCESS_KEY='${{ secrets.aws-secret-access-key }}' - export CODE_EVAL_DEVICE='${{ secrets.code-eval-device }}' - export CODE_EVAL_URL='${{ secrets.code-eval-url }}' - export CODE_EVAL_APIKEY='${{ secrets.code-eval-apikey }}' - export GCS_KEY='${{ secrets.gcs-key }}' - export GCS_SECRET='${{ secrets.gcs-secret }}' - export AZURE_ACCOUNT_NAME='${{ secrets.azure-account-name }}' - export AZURE_ACCOUNT_ACCESS_KEY='${{ secrets.azure-account-access-key }}' - export COMMON_ARGS="-v --durations=20 -m '${{ inputs.pytest-markers }}' -o tmp_path_retention_policy=failed" - - # Necessary to run git diff for doctests - git config --global --add safe.directory /__w/${{ inputs.safe_directory }}/${{ inputs.safe_directory }} - make test PYTEST='${{ inputs.pytest-command }}' EXTRA_ARGS="$COMMON_ARGS --codeblocks" - make test-dist PYTEST='${{ inputs.pytest-command }}' EXTRA_ARGS="$COMMON_ARGS" WORLD_SIZE=2 - - python -m coverage combine - - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.sha }}-${{ inputs.name }} - path: .coverage diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml deleted file mode 100644 index a7808d5..0000000 --- a/.github/workflows/pytest-gpu.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: Pytest GPU -on: - workflow_call: - inputs: - container: - required: true - type: string - gha-timeout: - required: false - type: number - default: 60 - git_repo: - required: true - type: string - name: - required: true - type: string - pip_deps: - required: true - type: string - pytest-command: - required: true - type: string - pytest-markers: - required: true - type: string - python-version: - required: false - type: string - default: 3.9 - mcloud-timeout: - required: false - type: number - default: 2700 - composer_package_name: - required: false - type: string - git-ssh-clone: - required: false - type: boolean - default: false - gpu_num: - required: true - type: number - secrets: - mcloud-api-key: - required: true - slack-notifications-bot-token: - required: false - code-eval-device: - required: false - code-eval-url: - required: false - code-eval-apikey: - required: false -jobs: - pytest-gpu: - timeout-minutes: ${{ inputs.gha-timeout }} - runs-on: linux-ubuntu-latest - env: - MOSAICML_API_KEY: ${{ secrets.mcloud-api-key }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - with: - repository: "mosaicml/ci-testing" - ref: "v0.0.9" - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python-version }} - - name: Cache pip - uses: actions/cache@v3 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - name: Setup MCLI - run: | - set -ex - python -m pip install mosaicml-cli - mcli version - - name: Submit Run - id: tests - run: | - set -ex - - PR_NUMBER="$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" - REF_ARGS="" - - # Use the PR number if it exists, commit SHA for protected branches and the branch name otherwise - if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then - if [[ "$GITHUB_REF" =~ "refs/heads/dev" || "$GITHUB_REF" =~ "refs/heads/main" || \ - "$GITHUB_REF" =~ "refs/heads/release" ]]; then - REF_ARGS="--git_commit $GITHUB_SHA" - else - REF_ARGS="--git_branch $GITHUB_REF_NAME" - fi - else - REF_ARGS="--pr_number $PR_NUMBER" - fi - - python .github/mcli/mcli_pytest.py \ - --image '${{ inputs.container }}' \ - --git_repo '${{ inputs.git_repo }}' \ - --pip_deps '${{ inputs.pip_deps }}' \ - --pip_package_name '${{ inputs.composer_package_name }}' \ - --pytest_markers '${{ inputs.pytest-markers }}' \ - --pytest_command '${{ inputs.pytest-command }}' \ - --timeout ${{ inputs.mcloud-timeout }} \ - --gpu_num ${{ inputs.gpu_num }} \ - --git_ssh_clone ${{ inputs.git-ssh-clone }} \ - ${REF_ARGS} diff --git a/README.md b/README.md index 4825230..144475c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # ci-testing + This repository contains code for CI/CD and testing of MosaicML repos including [Composer](https://github.com/mosaicml/composer) and [Foundry](https://github.com/mosaicml/llm-foundry). # Release Instructions -Update the version in `.github/workflows/pytest-gpu.yaml` when releasing a new version of `mosaicml/ci-testing`. + +1. When bumping patch/minor/major, also select `Set as the latest release`. + +## GPU Tests + +You can select the GPU test version you'd like to use by setting the `ci_repo_gpu_test_ref` input in your calling workflow to the desired repo release/branch. +