Add the support of using WholeGraph distributed embedding to store/update sparse_emb #4432
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Graphstorm CI | |
| # Test execution order - [lint] -> [pytest] -> [end2end_cpu, end2end, end2end_mgpu] | |
| name: continuous-integration | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - ci_dev | |
| paths-ignore: | |
| - '.github/workflow_scripts/gsprocessing_pytest.sh' | |
| - '.github/workflow_scripts/gsprocessing_lint.sh' | |
| - 'graphstorm-processing/**' | |
| - '.github/workflows/gsprocessing-workflow.yml' | |
| - 'docs/**' | |
| pull_request_target: | |
| types: [ labeled, opened, reopened, synchronize, ready_for_review ] | |
| paths-ignore: | |
| - '.github/workflow_scripts/gsprocessing_pytest.sh' | |
| - '.github/workflow_scripts/gsprocessing_lint.sh' | |
| - 'graphstorm-processing/**' | |
| - '.github/workflows/gsprocessing-workflow.yml' | |
| - 'docs/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.number || github.event.pull_request.head.sha }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| pull-requests: write | |
| contents: read | |
| defaults: | |
| run: | |
| shell: bash | |
| # All test scripts split in individual .sh files and moved to .github/workflow scripts | |
| env: | |
| COMMAND-PYTEST: chmod +x ./pytest_check.sh && ./pytest_check.sh | |
| COMMAND-LINT: chmod +x ./lint_check.sh && ./lint_check.sh | |
| COMMAND-E2E-CPU-TEST: chmod +x ./e2e_cpu_check.sh && ./e2e_cpu_check.sh | |
| COMMAND-E2E: chmod +x ./e2e_check.sh && ./e2e_check.sh | |
| COMMAND-E2E-MGPU: chmod +x ./e2e_mgpu_check.sh && ./e2e_mgpu_check.sh | |
| jobs: | |
| lint_check: | |
| if: ${{ github.event_name == 'push' || | |
| (github.event.label.name != 'draft' | |
| && github.event_name == 'pull_request_target' | |
| && contains(github.event.pull_request.labels.*.name, 'ready'))}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role | |
| role-duration-seconds: 14400 | |
| aws-region: us-east-1 | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python 3.x | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install pytest | |
| pip3 install boto3 | |
| - name: Submit Job (for Push) | |
| if: ${{ github.event_name == 'push' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Push" | |
| python3 ./submitJob.py --job-type CI-LINT-PUSH --name Graphstorm-LintTest-Push-'${{ github.ref }}' \ | |
| --command "${{ env.COMMAND-LINT }}" \ | |
| --remote https://github.com/'${{ github.repository }}' \ | |
| --source-ref '${{ github.ref }}' \ | |
| --wait | |
| - name: Submit Job (for Pull Request) | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Pull Request" | |
| python3 ./submitJob.py --job-type CI-LINT-PUSH --name Graphstorm-LintTest-PR#-'${{ github.event.number }}' \ | |
| --command "${{ env.COMMAND-LINT }}" \ | |
| --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ | |
| --source-ref '${{ github.event.pull_request.head.sha }}' \ | |
| --wait | |
| pytest_check: | |
| if: ${{ github.event.label.name != 'draft' }} | |
| needs: lint_check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role | |
| aws-region: us-east-1 | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python 3.x | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install pytest | |
| pip3 install boto3 | |
| - name: Submit Job (for Push) | |
| if: ${{ github.event_name == 'push' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Push" | |
| python3 ./submitJob.py --job-type CI-CPU-PUSH --name Graphstorm-PyTest-Push-'${{ github.ref }}' \ | |
| --command "${{ env.COMMAND-PYTEST }}" \ | |
| --remote https://github.com/'${{ github.repository }}' \ | |
| --source-ref '${{ github.ref }}' \ | |
| --wait | |
| - name: Submit Job (for Pull Request) | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Pull Request" | |
| python3 ./submitJob.py --job-type CI-CPU --name Graphstorm-PyTest-PR#-'${{ github.event.number }}' \ | |
| --command "${{ env.COMMAND-PYTEST }}" \ | |
| --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ | |
| --source-ref '${{ github.event.pull_request.head.sha }}' \ | |
| --wait | |
| e2e_cpu_check: | |
| if: ${{ github.event.label.name != 'draft' }} | |
| needs: [pytest_check, lint_check] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role | |
| aws-region: us-east-1 | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python 3.x | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install pytest | |
| pip3 install boto3 | |
| - name: Submit Job (for Push) | |
| if: ${{ github.event_name == 'push' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Push" | |
| python3 ./submitJob.py --job-type CI-CPU-CHECK-PUSH --name Graphstorm-E2ETest-CPU-Push-'${{ github.ref }}' \ | |
| --command "${{ env.COMMAND-E2E-CPU-TEST }}" \ | |
| --remote https://github.com/'${{ github.repository }}' \ | |
| --source-ref '${{ github.ref }}' \ | |
| --wait | |
| - name: Submit Job (for Pull Request) | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Pull Request" | |
| python3 ./submitJob.py --job-type CI-CPU-CHECK --name Graphstorm-E2ETest-CPU-PR#-'${{ github.event.number }}' \ | |
| --command "${{ env.COMMAND-E2E-CPU-TEST }}" \ | |
| --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ | |
| --source-ref '${{ github.event.pull_request.head.sha }}' \ | |
| --wait | |
| e2e_check: | |
| if: ${{ github.event.label.name != 'draft' }} | |
| needs: [pytest_check, lint_check] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role | |
| aws-region: us-east-1 | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python 3.x | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install pytest | |
| pip3 install boto3 | |
| - name: Submit Job (for Push) | |
| if: ${{ github.event_name == 'push' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Push" | |
| python3 ./submitJob.py --job-type CI-CPU-PUSH --name Graphstorm-E2ETest-Push-'${{ github.ref }}' \ | |
| --command "${{ env.COMMAND-E2E }}" \ | |
| --remote https://github.com/'${{ github.repository }}' \ | |
| --source-ref '${{ github.ref }}' \ | |
| --wait | |
| - name: Submit Job (for Pull Request) | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Pull Request" | |
| python3 ./submitJob.py --job-type CI-CPU --name Graphstorm-E2ETest-PR#-'${{ github.event.number }}' \ | |
| --command "${{ env.COMMAND-E2E }}" \ | |
| --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ | |
| --source-ref '${{ github.event.pull_request.head.sha }}' \ | |
| --wait | |
| e2e_mgpu_check: | |
| if: ${{ github.event.label.name != 'draft' }} | |
| needs: [pytest_check, lint_check] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role | |
| aws-region: us-east-1 | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python 3.x | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install pytest | |
| pip3 install boto3 | |
| - name: Submit Job (for Push) | |
| if: ${{ github.event_name == 'push' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Push" | |
| python3 ./submitJob.py --job-type CI-GPU-PUSH --name Graphstorm-E2E-MGPUTest-Push-'${{ github.ref }}' \ | |
| --command "${{ env.COMMAND-E2E-MGPU }}" \ | |
| --remote https://github.com/'${{ github.repository }}' \ | |
| --source-ref '${{ github.ref }}' \ | |
| --wait | |
| - name: Submit Job (for Pull Request) | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| shell: bash | |
| run: | | |
| echo "Start submitting job - Pull Request" | |
| python3 ./submitJob.py --job-type CI-GPU --name Graphstorm-E2E-MGPUTest-PR#-'${{ github.event.number }}' \ | |
| --command "${{ env.COMMAND-E2E-MGPU }}" \ | |
| --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ | |
| --source-ref '${{ github.event.pull_request.head.sha }}' \ | |
| --wait | |