From 01a483ababd0eddb08a2272c7deb0e3e72de0b5c Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Tue, 20 Aug 2024 12:43:46 +0530 Subject: [PATCH] chore: cicd improvements (#3585) * chore: cicd improvements * chore: verify formatting & lint errors to only check for added/modified files - chore: add json prettification on save in vscode settings * chore: remove test command while building image * fix: formatting in features.json * chore: update eslint in verify gha * chore: update latest image sha (#3622) * chore: update latest image sha * chore: add echo while setting a variable in run script * chore: update ut prod deployment workflow --------- Co-authored-by: Sai Sankeerth * chore: re-introduce execution of tests while building image - introduce a flag to skip tests - modify artifact building workflow to know when to skip tests * fix: update the correct workflows to skip tests in docker for hotfix release * fix: execute tests when dockerfile, package json is updated * chore: send slack message when prod deployment fails * chore: move slack notification to separate workflow * fix: format error * chore: add checkout step * chore: move notification to a separate job inside workflow - add capability to understand overall failure or just tests failure for notifying * chore: add missing lint step * fix: reusable notify workflow * chore: update notification job name --------- Co-authored-by: Sai Sankeerth --- .github/workflows/build-push-docker-image.yml | 63 ++++++++++++++++++- .../dt-test-and-report-code-coverage.yml | 26 +++++++- .github/workflows/prepare-for-dev-deploy.yml | 1 + .../workflows/prepare-for-prod-dt-deploy.yml | 2 + .../workflows/prepare-for-prod-ut-deploy.yml | 2 + .../workflows/prepare-for-staging-deploy.yml | 1 + .github/workflows/slack-notify.yml | 51 +++++++++++++++ .github/workflows/verify.yml | 24 +++++-- .vscode/settings.json | 4 ++ src/routerUtils.js | 1 + src/services/comparator.ts | 1 + 11 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/slack-notify.yml diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 1e7281479e..f0e33ff142 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -23,6 +23,15 @@ on: type: string build_type: type: string + use_merge_sha: + type: boolean + default: false + skip_tests: + type: boolean + default: false + description: if this option is true, we would skip tests while building docker image + workflow_url: + type: string secrets: DOCKERHUB_PROD_TOKEN: required: true @@ -31,14 +40,61 @@ env: DOCKERHUB_USERNAME: rudderlabs jobs: + get_sha: + runs-on: ubuntu-latest + name: Get SHA information + outputs: + sha: ${{steps.getSHA.outputs.SHA}} + steps: + - name: Checkout SHA + id: getSHA + run: | + if ${{inputs.use_merge_sha}} == true; then + sha=$(echo ${{github.sha}}) + else + sha=$(echo ${{ github.event.pull_request.head.sha }}) + fi + echo "SHA: $sha" + echo "SHA=$sha" >> $GITHUB_OUTPUT + + get_changed_files: + runs-on: ubuntu-latest + name: Get Changed files + outputs: + should_execute_tests: ${{ steps.processing.outputs.should_execute_tests }} + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 1 + - id: files + uses: Ana06/get-changed-files@v1.2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + format: 'json' + - id: processing + run: | + readarray -t modified_files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.modified }}')" + echo "Modified files: $modified_files" + found=false + for modified_file in "${modified_files[@]}"; do + if [[ "$modified_file" == "Dockerfile" || "$modified_file" == "docker-compose.yml" || "$modified_file" == "Dockerfile" || "$modified_file" == "Dockerfile-ut-func" ]]; then + found=true + break + fi + done + echo "Match Found: $found" + echo "::set-output name=should_execute_tests::$found" + build-transformer-image-arm64: name: Build Transformer Docker Image ARM64 runs-on: [self-hosted, Linux, ARM64] + needs: [get_sha, get_changed_files] steps: - name: Checkout uses: actions/checkout@v4.1.1 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - name: Setup Docker Buildx @@ -62,6 +118,7 @@ jobs: # cache-to: type=gha,mode=max - name: Run Tests + if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }} run: | docker run ${{ inputs.build_tag }} npm run test:js:ci docker run ${{ inputs.build_tag }} npm run test:ts:ci @@ -85,11 +142,12 @@ jobs: build-transformer-image-amd64: name: Build Transformer Docker Image AMD64 runs-on: [self-hosted, Linux, X64] + needs: [get_sha, get_changed_files] steps: - name: Checkout uses: actions/checkout@v4.1.1 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - name: Setup Docker Buildx @@ -113,6 +171,7 @@ jobs: # cache-to: type=gha,mode=max - name: Run Tests + if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }} run: | docker run ${{ inputs.build_tag }} npm run test:js:ci docker run ${{ inputs.build_tag }} npm run test:ts:ci diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 755eb24397..b1da46898a 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -13,10 +13,22 @@ concurrency: cancel-in-progress: true jobs: + get_workflow_url: + runs-on: ubuntu-latest + steps: + - id: get_url + run: | + curl -s https://api.github.com/repos/${{ github.repository }}/actions/workflows/${{ github.workflow }}/runs/${{ github.run_id }} | jq -r .html_url >> workflow_url.txt + echo "::set-output name=workflow_url::$(cat workflow_url.txt)" + outputs: + url: ${{ steps.get_url.outputs.workflow_url }} + coverage: name: Code Coverage runs-on: ubuntu-latest - + needs: [get_workflow_url] + outputs: + tests_run_outcome: ${{steps.run_tests.outcome}} steps: - name: Checkout uses: actions/checkout@v4.1.1 @@ -33,6 +45,8 @@ jobs: run: npm ci - name: Run Tests + id: run_tests + continue-on-error: true run: | # Supress logging in tests LOG_LEVEL=100 npm run test:js:ci @@ -70,3 +84,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.PAT }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + notify: + name: slack notification on failure + needs: [get_workflow_url, coverage] + if: needs.coverage.outputs.tests_run_outcome == 'failure' || failure() + uses: ./.github/workflows/slack-notify.yml + with: + workflow_url: ${{ needs.get_workflow_url.outputs.url }} + should_notify: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}} + secrets: inherit diff --git a/.github/workflows/prepare-for-dev-deploy.yml b/.github/workflows/prepare-for-dev-deploy.yml index 66020feef0..a1fdda8ccd 100644 --- a/.github/workflows/prepare-for-dev-deploy.yml +++ b/.github/workflows/prepare-for-dev-deploy.yml @@ -60,6 +60,7 @@ jobs: dockerfile: Dockerfile load_target: development push_target: production + use_merge_sha: true secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/prepare-for-prod-dt-deploy.yml b/.github/workflows/prepare-for-prod-dt-deploy.yml index 4a4d656e78..8e589f5b20 100644 --- a/.github/workflows/prepare-for-prod-dt-deploy.yml +++ b/.github/workflows/prepare-for-prod-dt-deploy.yml @@ -57,6 +57,8 @@ jobs: load_target: development push_target: production build_type: dt + use_merge_sha: true + skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}} secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/prepare-for-prod-ut-deploy.yml b/.github/workflows/prepare-for-prod-ut-deploy.yml index a5afed6dee..468307b94a 100644 --- a/.github/workflows/prepare-for-prod-ut-deploy.yml +++ b/.github/workflows/prepare-for-prod-ut-deploy.yml @@ -60,6 +60,8 @@ jobs: load_target: development push_target: production build_type: ut + use_merge_sha: true + skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}} secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/prepare-for-staging-deploy.yml b/.github/workflows/prepare-for-staging-deploy.yml index 4a4bf44f75..60a6238aa8 100644 --- a/.github/workflows/prepare-for-staging-deploy.yml +++ b/.github/workflows/prepare-for-staging-deploy.yml @@ -52,6 +52,7 @@ jobs: dockerfile: Dockerfile load_target: development push_target: production + use_merge_sha: true secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/slack-notify.yml b/.github/workflows/slack-notify.yml new file mode 100644 index 0000000000..bb5cac82f1 --- /dev/null +++ b/.github/workflows/slack-notify.yml @@ -0,0 +1,51 @@ +name: Notify workflow failure + +on: + workflow_call: + inputs: + should_notify: + type: boolean + default: true + workflow_url: + type: string + required: true + +jobs: + notify: + runs-on: ubuntu-latest + if: ${{ inputs.should_notify }} + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + + - name: notify + uses: slackapi/slack-github-action@v1.25.0 + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + PROJECT_NAME: 'Rudder Transformer' + with: + channel-id: ${{ secrets.SLACK_INTEGRATION_DEV_CHANNEL_ID }} + payload: | + { + "text": "**\nCC: ", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": ":siren2: prod release tests - Failed :siren2:" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*<${{inputs.workflow_url}}|failed workflow>*\nCC: " + } + } + ] + } diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index a03037d4ad..0d389f0e55 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -28,13 +28,29 @@ jobs: - name: Install Dependencies run: npm ci - - name: Run Lint Checks + - id: files + uses: Ana06/get-changed-files@v1.2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Run format Checks + run: | + npx prettier ${{steps.files.outputs.added_modified}} --write + + - run: git diff --exit-code + + - name: Formatting Error message + if: ${{ failure() }} + run: | + echo 'prettier formatting failure. Ensure you run `npm run format` and commit the files.' + + - name: Run eslint Checks run: | - npm run lint + npx eslint ${{steps.files.outputs.added_modified}} --fix - run: git diff --exit-code - - name: Error message + - name: Eslint Error message if: ${{ failure() }} run: | - echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.' + echo 'Eslint failure. Ensure you run `npm run lint:fix` and commit the files.' diff --git a/.vscode/settings.json b/.vscode/settings.json index 13c49c08f1..8d723306a9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,6 +13,10 @@ "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true }, + "[json]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true + }, "[yaml]": { "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true diff --git a/src/routerUtils.js b/src/routerUtils.js index ff9dd4b6f8..081070d78a 100644 --- a/src/routerUtils.js +++ b/src/routerUtils.js @@ -22,6 +22,7 @@ const userTransformHandler = () => { async function sendToDestination(destination, payload) { let parsedResponse; logger.info('Request recieved for destination', destination); + const resp = await proxyRequest(payload); if (resp.success) { diff --git a/src/services/comparator.ts b/src/services/comparator.ts index 0e28339797..1eb67cd597 100644 --- a/src/services/comparator.ts +++ b/src/services/comparator.ts @@ -33,6 +33,7 @@ export class ComparatorService implements DestinationService { public init(): void { this.primaryService.init(); + this.secondaryService.init(); }