diff --git a/.github/workflow_scripts/gsprocessing_lint.sh b/.github/workflow_scripts/gsprocessing_lint.sh new file mode 100644 index 0000000000..e43da86db0 --- /dev/null +++ b/.github/workflow_scripts/gsprocessing_lint.sh @@ -0,0 +1,9 @@ +# Move to parent directory +cd ../../ + +set -ex + +pip install pylint==2.17.5 + +pylint --rcfile=./tests/lint/pylintrc ./graphstorm-processing/graphstorm_processing/ + diff --git a/.github/workflow_scripts/gsprocessing_pytest.sh b/.github/workflow_scripts/gsprocessing_pytest.sh new file mode 100644 index 0000000000..820f1a2c14 --- /dev/null +++ b/.github/workflow_scripts/gsprocessing_pytest.sh @@ -0,0 +1,4 @@ +cd /usr/lib/spark/graphstorm/graphstorm-processing/ +pip install . +python3 -m pytest . + diff --git a/.github/workflow_scripts/lint_check.sh b/.github/workflow_scripts/lint_check.sh index 874193bd25..3eecaf93c3 100644 --- a/.github/workflow_scripts/lint_check.sh +++ b/.github/workflow_scripts/lint_check.sh @@ -18,5 +18,3 @@ pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/run/ pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/utils.py pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/sagemaker/ - -pylint --rcfile=./tests/lint/pylintrc ./graphstorm-processing/graphstorm_processing/ diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index eac0de0d0e..8aa5d49651 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -7,9 +7,18 @@ on: branches: - main - ci_dev + paths-ignore: + - '.github/workflow_scripts/gsprocessing_pytest.sh' + - '.github/workflow_scripts/gsprocessing_lint.sh' + - 'graphstorm-processing/*' + - '.github/workflows/gsprocessing-workflow.yml' pull_request_target: types: [ labeled, opened, reopened, synchronize, ready_for_review ] - + paths-ignore: + - '.github/workflow_scripts/gsprocessing_pytest.sh' + - '.github/workflow_scripts/gsprocessing_lint.sh' + - 'graphstorm-processing/*' + - '.github/workflows/gsprocessing-workflow.yml' permissions: id-token: write @@ -250,5 +259,5 @@ jobs: --command "${{ env.COMMAND-E2E-MGPU }}" \ --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ --source-ref '${{ github.event.pull_request.head.sha }}' \ - --wait + --wait diff --git a/.github/workflows/gsprocessing-workflow.yml b/.github/workflows/gsprocessing-workflow.yml new file mode 100644 index 0000000000..4fc17f708b --- /dev/null +++ b/.github/workflows/gsprocessing-workflow.yml @@ -0,0 +1,122 @@ +# Graphstorm CI for GSProcessing Only +# Test execution order - [lint] -> [gsprocessing-pytest] +name: gsprocessing_workflow + +on: + push: + branches: + - main + - ci_dev + paths: + - '.github/workflow_scripts/gsprocessing_pytest.sh' + - '.github/workflow_scripts/gsprocessing_lint.sh' + - 'graphstorm-processing/*' + - '.github/workflows/gsprocessing-workflow.yml' + pull_request_target: + types: [ labeled, opened, reopened, synchronize, ready_for_review ] + paths: + - '.github/workflow_scripts/gsprocessing_pytest.sh' + - '.github/workflow_scripts/gsprocessing_lint.sh' + - 'graphstorm-processing/*' + - '.github/workflows/gsprocessing-workflow.yml' + +permissions: + id-token: write + pull-requests: write + contents: read + +defaults: + run: + shell: bash + +# All test scripts split in individual .sh files and moved to .github/workflow scripts +env: + COMMAND-LINT: bash gsprocessing_lint.sh + COMMAND-GSProcessing-Pytest: bash gsprocessing_pytest.sh + +jobs: + gsp_lint: + if: ${{ github.event_name == 'push' || + (github.event.label.name != 'draft' + && github.event_name == 'pull_request_target' + && contains(github.event.pull_request.labels.*.name, 'ready')) }} + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role + aws-region: us-east-1 + - name: Checkout repository + uses: actions/checkout@v3 + - name: Set up Python 3.x + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip + pip3 install boto3 + - name: Submit Job (for Push) + if: ${{ github.event_name == 'push' }} + shell: bash + run: | + echo "Start submitting job - Push" + python3 ./submitJob.py --job-type CI-LINT-PUSH --name Graphstorm-LintTest-Push-'${{ github.ref }}' \ + --command "${{ env.COMMAND-LINT }}" \ + --remote https://github.com/'${{ github.repository }}' \ + --source-ref '${{ github.ref }}' \ + --wait + - name: Submit Job (for Pull Request) + if: ${{ github.event_name == 'pull_request_target' }} + shell: bash + run: | + echo "Start submitting job - Pull Request" + python3 ./submitJob.py --job-type CI-LINT --name Graphstorm-LintTest-PR#-'${{ github.event.number }}' \ + --command "${{ env.COMMAND-LINT }}" \ + --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ + --source-ref '${{ github.event.pull_request.head.sha }}' \ + --wait + + gsprocessing: + if: ${{ github.event.label.name != 'draft' }} + needs: gsp_lint + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role + aws-region: us-east-1 + - name: Checkout repository + uses: actions/checkout@v3 + - name: Set up Python 3.x + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip + pip3 install boto3 + - name: Submit Job (for Push) + if: ${{ github.event_name == 'push' }} + shell: bash + run: | + echo "Start submitting job - Push" + python3 ./submitJob.py --job-type CI-GSProcessing-CHECK --name Graphstorm-GSProcessing-Push-'${{ github.ref }}' \ + --command "${{ env.COMMAND-GSProcessing-Pytest }}" \ + --remote https://github.com/'${{ github.repository }}' \ + --source-ref '${{ github.ref }}' \ + --wait + - name: Submit Job (for Pull Request) + if: ${{ github.event_name == 'pull_request_target' }} + shell: bash + run: | + echo "Start submitting job - Pull Request" + python3 ./submitJob.py --job-type CI-GSProcessing-CHECK --name Graphstorm-GSProcessing-PR#-'${{ github.event.number }}' \ + --command "${{ env.COMMAND-GSProcessing-Pytest }}" \ + --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \ + --source-ref '${{ github.event.pull_request.head.sha }}' \ + --wait diff --git a/submitJob.py b/submitJob.py index 916a47cf05..048a86e999 100644 --- a/submitJob.py +++ b/submitJob.py @@ -43,6 +43,10 @@ 'CI-CPU-CHECK': { 'job_definition': 'graphstorm-definition-check', 'job_queue': 'graphstorm-queue-lint', + }, + 'CI-GSProcessing-CHECK': { + 'job_definition': 'graphstorm-gsprocessing-definition', + 'job_queue': 'graphstorm-queue-v2', } }