chore: cicd improvements (#3585)

* chore: cicd improvements

* chore: verify formatting & lint errors to only check for added/modified files
- chore: add json prettification on save in vscode settings

* chore: remove test command while building image

* fix: formatting in features.json

* chore: update eslint in verify gha

* chore: update latest image sha (#3622)

* chore: update latest image sha

* chore: add echo while setting a variable in run script

* chore: update ut prod deployment workflow

---------

Co-authored-by: Sai Sankeerth <[email protected]>

* chore: re-introduce execution of tests while building image
- introduce a flag to skip tests
- modify artifact building workflow to know when to skip tests

* fix: update the correct workflows to skip tests in docker for hotfix release

* fix: execute tests when dockerfile, package json is updated

* chore: send slack message when prod deployment fails

* chore: move slack notification to separate workflow

* fix: format error

* chore: add checkout step

* chore: move notification to a separate job inside workflow
- add capability to understand overall failure or just tests failure for notifying

* chore: add missing lint step

* fix: reusable notify workflow

* chore: update notification job name

---------

Co-authored-by: Sai Sankeerth <[email protected]>

.github/workflows/build-push-docker-image.yml

@@ -23,6 +23,15 @@ on:
         type: string
       build_type:
         type: string
+      use_merge_sha:
+        type: boolean
+        default: false
+      skip_tests:
+        type: boolean
+        default: false
+        description: if this option is true, we would skip tests while building docker image
+      workflow_url:
+        type: string
     secrets:
       DOCKERHUB_PROD_TOKEN:
         required: true
@@ -31,14 +40,61 @@ env:
   DOCKERHUB_USERNAME: rudderlabs
 
 jobs:
+  get_sha:
+    runs-on: ubuntu-latest
+    name: Get SHA information
+    outputs:
+      sha: ${{steps.getSHA.outputs.SHA}}
+    steps:
+      - name: Checkout SHA
+        id: getSHA
+        run: |
+          if ${{inputs.use_merge_sha}} == true; then
+            sha=$(echo ${{github.sha}})
+          else
+            sha=$(echo ${{ github.event.pull_request.head.sha }})
+          fi
+          echo "SHA: $sha"
+          echo "SHA=$sha" >> $GITHUB_OUTPUT
+
+  get_changed_files:
+    runs-on: ubuntu-latest
+    name: Get Changed files
+    outputs:
+      should_execute_tests: ${{ steps.processing.outputs.should_execute_tests }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+      - id: files
+        uses: Ana06/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          format: 'json'
+      - id: processing
+        run: |
+          readarray -t modified_files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.modified }}')"
+          echo "Modified files: $modified_files"
+          found=false
+          for modified_file in "${modified_files[@]}"; do
+            if [[ "$modified_file" == "Dockerfile" || "$modified_file" == "docker-compose.yml" || "$modified_file" == "Dockerfile" || "$modified_file" == "Dockerfile-ut-func" ]]; then
+              found=true
+              break
+            fi
+          done
+          echo "Match Found: $found"
+          echo "::set-output name=should_execute_tests::$found"
+
   build-transformer-image-arm64:
     name: Build Transformer Docker Image ARM64
     runs-on: [self-hosted, Linux, ARM64]
+    needs: [get_sha, get_changed_files]
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx
@@ -62,6 +118,7 @@ jobs:
           # cache-to: type=gha,mode=max
 
       - name: Run Tests
+        if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }}
         run: |
           docker run ${{ inputs.build_tag }} npm run test:js:ci
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
@@ -85,11 +142,12 @@ jobs:
   build-transformer-image-amd64:
     name: Build Transformer Docker Image AMD64
     runs-on: [self-hosted, Linux, X64]
+    needs: [get_sha, get_changed_files]
     steps:
       - name: Checkout
         uses: actions/[email protected]
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
 
       - name: Setup Docker Buildx
@@ -113,6 +171,7 @@ jobs:
           # cache-to: type=gha,mode=max
 
       - name: Run Tests
+        if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }}
         run: |
           docker run ${{ inputs.build_tag }} npm run test:js:ci
           docker run ${{ inputs.build_tag }} npm run test:ts:ci

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -13,10 +13,22 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  get_workflow_url:
+    runs-on: ubuntu-latest
+    steps:
+      - id: get_url
+        run: |
+          curl -s https://api.github.com/repos/${{ github.repository }}/actions/workflows/${{ github.workflow }}/runs/${{ github.run_id }} | jq -r .html_url >> workflow_url.txt
+          echo "::set-output name=workflow_url::$(cat workflow_url.txt)"
+    outputs:
+      url: ${{ steps.get_url.outputs.workflow_url }}
+
   coverage:
     name: Code Coverage
     runs-on: ubuntu-latest
-
+    needs: [get_workflow_url]
+    outputs:
+      tests_run_outcome: ${{steps.run_tests.outcome}}
     steps:
       - name: Checkout
         uses: actions/[email protected]
@@ -33,6 +45,8 @@ jobs:
         run: npm ci
 
       - name: Run Tests
+        id: run_tests
+        continue-on-error: true
         run: |
           # Supress logging in tests
           LOG_LEVEL=100 npm run test:js:ci
@@ -70,3 +84,13 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+  notify:
+    name: slack notification on failure
+    needs: [get_workflow_url, coverage]
+    if: needs.coverage.outputs.tests_run_outcome == 'failure' || failure()
+    uses: ./.github/workflows/slack-notify.yml
+    with:
+      workflow_url: ${{ needs.get_workflow_url.outputs.url }}
+      should_notify: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
+    secrets: inherit

.github/workflows/prepare-for-dev-deploy.yml

@@ -60,6 +60,7 @@ jobs:
       dockerfile: Dockerfile
       load_target: development
       push_target: production
+      use_merge_sha: true
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -57,6 +57,8 @@ jobs:
       load_target: development
       push_target: production
       build_type: dt
+      use_merge_sha: true
+      skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-prod-ut-deploy.yml

@@ -60,6 +60,8 @@ jobs:
       load_target: development
       push_target: production
       build_type: ut
+      use_merge_sha: true
+      skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}}
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/prepare-for-staging-deploy.yml

@@ -52,6 +52,7 @@ jobs:
       dockerfile: Dockerfile
       load_target: development
       push_target: production
+      use_merge_sha: true
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 

.github/workflows/slack-notify.yml

@@ -0,0 +1,51 @@
+name: Notify workflow failure
+
+on:
+  workflow_call:
+    inputs:
+      should_notify:
+        type: boolean
+        default: true
+      workflow_url:
+        type: string
+        required: true
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.should_notify }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: notify
+        uses: slackapi/[email protected]
+        continue-on-error: true
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          PROJECT_NAME: 'Rudder Transformer'
+        with:
+          channel-id: ${{ secrets.SLACK_INTEGRATION_DEV_CHANNEL_ID }}
+          payload: |
+            {
+              "text": "*<prod release tests failed>*\nCC: <!subteam^S02AEQL26CT>",
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":siren2:  prod release tests - Failed  :siren2:"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "*<${{inputs.workflow_url}}|failed workflow>*\nCC: <!subteam^S02AEQL26CT>"
+                  }
+                }
+              ]
+            }

.github/workflows/verify.yml

@@ -28,13 +28,29 @@ jobs:
       - name: Install Dependencies
         run: npm ci
 
-      - name: Run Lint Checks
+      - id: files
+        uses: Ana06/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run format Checks
+        run: |
+          npx prettier ${{steps.files.outputs.added_modified}} --write
+
+      - run: git diff --exit-code
+
+      - name: Formatting Error message
+        if: ${{ failure() }}
+        run: |
+          echo 'prettier formatting failure. Ensure you run `npm run format` and commit the files.'
+
+      - name: Run eslint Checks
         run: |
-          npm run lint
+          npx eslint ${{steps.files.outputs.added_modified}} --fix
 
       - run: git diff --exit-code
 
-      - name: Error message
+      - name: Eslint Error message
         if: ${{ failure() }}
         run: |
-          echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.'
+          echo 'Eslint failure. Ensure you run `npm run lint:fix` and commit the files.'

.vscode/settings.json

@@ -13,6 +13,10 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true
   },
+  "[json]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true
+  },
   "[yaml]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true

src/routerUtils.js

@@ -22,6 +22,7 @@ const userTransformHandler = () => {
 async function sendToDestination(destination, payload) {
   let parsedResponse;
   logger.info('Request recieved for destination', destination);
+
   const resp = await proxyRequest(payload);
 
   if (resp.success) {

src/services/comparator.ts

@@ -33,6 +33,7 @@ export class ComparatorService implements DestinationService {
 
   public init(): void {
     this.primaryService.init();
+
     this.secondaryService.init();
   }