Add first version for the workflow on GSProcessing (#407)

*Issue #, if available:*

*Description of changes:*

Will delete the PR after testing

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

.github/workflow_scripts/gsprocessing_lint.sh

@@ -0,0 +1,9 @@
+# Move to parent directory
+cd ../../
+
+set -ex
+
+pip install pylint==2.17.5
+
+pylint --rcfile=./tests/lint/pylintrc ./graphstorm-processing/graphstorm_processing/
+

.github/workflow_scripts/gsprocessing_pytest.sh

@@ -0,0 +1,4 @@
+cd /usr/lib/spark/graphstorm/graphstorm-processing/
+pip install .
+python3 -m pytest .
+

.github/workflow_scripts/lint_check.sh

@@ -18,5 +18,3 @@ pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/run/
 pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/utils.py
 
 pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/sagemaker/
-
-pylint --rcfile=./tests/lint/pylintrc ./graphstorm-processing/graphstorm_processing/

.github/workflows/continuous-integration.yml

@@ -7,9 +7,18 @@ on:
     branches:
       - main
       - ci_dev
+    paths-ignore:
+      - '.github/workflow_scripts/gsprocessing_pytest.sh'
+      - '.github/workflow_scripts/gsprocessing_lint.sh'
+      - 'graphstorm-processing/*'
+      - '.github/workflows/gsprocessing-workflow.yml'
   pull_request_target:
     types: [ labeled, opened, reopened, synchronize, ready_for_review ]
-
+    paths-ignore:
+      - '.github/workflow_scripts/gsprocessing_pytest.sh'
+      - '.github/workflow_scripts/gsprocessing_lint.sh'
+      - 'graphstorm-processing/*'
+      - '.github/workflows/gsprocessing-workflow.yml'    
 
 permissions:
   id-token: write
@@ -250,5 +259,5 @@ jobs:
                                --command "${{ env.COMMAND-E2E-MGPU }}" \
                                --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                                --source-ref '${{ github.event.pull_request.head.sha }}' \
-                               --wait    
+                               --wait
 

.github/workflows/gsprocessing-workflow.yml

@@ -0,0 +1,122 @@
+# Graphstorm CI for GSProcessing Only
+# Test execution order - [lint] -> [gsprocessing-pytest]
+name: gsprocessing_workflow
+
+on:
+  push:
+    branches:
+      - main
+      - ci_dev
+    paths:
+      - '.github/workflow_scripts/gsprocessing_pytest.sh'
+      - '.github/workflow_scripts/gsprocessing_lint.sh'
+      - 'graphstorm-processing/*'
+      - '.github/workflows/gsprocessing-workflow.yml'
+  pull_request_target:
+    types: [ labeled, opened, reopened, synchronize, ready_for_review ]
+    paths:
+      - '.github/workflow_scripts/gsprocessing_pytest.sh'
+      - '.github/workflow_scripts/gsprocessing_lint.sh'
+      - 'graphstorm-processing/*'
+      - '.github/workflows/gsprocessing-workflow.yml'
+
+permissions:
+  id-token: write
+  pull-requests: write
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+# All test scripts split in individual .sh files and moved to .github/workflow scripts
+env:
+  COMMAND-LINT: bash gsprocessing_lint.sh
+  COMMAND-GSProcessing-Pytest: bash gsprocessing_pytest.sh
+
+jobs:
+  gsp_lint:
+    if: ${{ github.event_name == 'push' ||
+      (github.event.label.name != 'draft'
+      && github.event_name == 'pull_request_target'
+      && contains(github.event.pull_request.labels.*.name, 'ready')) }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
+        aws-region: us-east-1
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python3 -m pip install --upgrade pip
+        pip3 install boto3
+    - name: Submit Job (for Push)
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      run: |
+        echo "Start submitting job - Push"  
+        python3 ./submitJob.py --job-type CI-LINT-PUSH --name Graphstorm-LintTest-Push-'${{ github.ref }}' \
+                               --command "${{ env.COMMAND-LINT }}" \
+                               --remote https://github.com/'${{ github.repository }}' \
+                               --source-ref '${{ github.ref }}' \
+                               --wait
+    - name: Submit Job (for Pull Request)
+      if: ${{ github.event_name == 'pull_request_target' }}
+      shell: bash
+      run: |
+        echo "Start submitting job - Pull Request"
+        python3 ./submitJob.py --job-type CI-LINT --name Graphstorm-LintTest-PR#-'${{ github.event.number }}' \
+                               --command "${{ env.COMMAND-LINT }}" \
+                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
+                               --source-ref '${{ github.event.pull_request.head.sha }}' \
+                               --wait
+
+  gsprocessing:
+    if: ${{ github.event.label.name != 'draft' }}
+    needs: gsp_lint
+    runs-on: ubuntu-latest
+    steps:
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
+        aws-region: us-east-1
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python3 -m pip install --upgrade pip
+        pip3 install boto3
+    - name: Submit Job (for Push)
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      run: |
+        echo "Start submitting job - Push"  
+        python3 ./submitJob.py --job-type CI-GSProcessing-CHECK --name Graphstorm-GSProcessing-Push-'${{ github.ref }}' \
+                               --command "${{ env.COMMAND-GSProcessing-Pytest }}" \
+                               --remote https://github.com/'${{ github.repository }}' \
+                               --source-ref '${{ github.ref }}' \
+                               --wait
+    - name: Submit Job (for Pull Request)
+      if: ${{ github.event_name == 'pull_request_target' }}
+      shell: bash
+      run: |
+        echo "Start submitting job - Pull Request"
+        python3 ./submitJob.py --job-type CI-GSProcessing-CHECK --name Graphstorm-GSProcessing-PR#-'${{ github.event.number }}' \
+                               --command "${{ env.COMMAND-GSProcessing-Pytest }}" \
+                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
+                               --source-ref '${{ github.event.pull_request.head.sha }}' \
+                               --wait

submitJob.py

@@ -43,6 +43,10 @@
     'CI-CPU-CHECK': {
         'job_definition': 'graphstorm-definition-check',
         'job_queue': 'graphstorm-queue-lint',
+    },
+    'CI-GSProcessing-CHECK': {
+        'job_definition': 'graphstorm-gsprocessing-definition',
+        'job_queue': 'graphstorm-queue-v2',
     }
 }