Feat/snyk fixes test

.github/workflows/build-pr-artifacts.yml

@@ -43,7 +43,7 @@ jobs:
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
-      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
+      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name }}
       dockerfile: Dockerfile
       load_target: development
@@ -59,7 +59,7 @@ jobs:
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
-      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
       dockerfile: Dockerfile-ut-func
       load_target: development

.github/workflows/build-push-docker-image.yml

@@ -29,9 +29,9 @@ env:
   DOCKERHUB_USERNAME: rudderlabs
 
 jobs:
-  build-transformer-image:
-    name: Build Transformer Docker Image
-    runs-on: ubuntu-latest
+  build-transformer-image-arm64:
+    name: Build Transformer Docker Image ARM64
+    runs-on: [self-hosted, Linux, ARM64]
     steps:
       - name: Checkout
         uses: actions/[email protected]
@@ -55,12 +55,12 @@ jobs:
           target: ${{ inputs.load_target }}
           load: true
           tags: ${{ inputs.build_tag }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
 
       - name: Run Tests
         run: |
-          docker run ${{ inputs.build_tag }} npm run test:js:ci        
+          docker run ${{ inputs.build_tag }} npm run test:js:ci
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
@@ -70,12 +70,80 @@ jobs:
           file: ${{ inputs.dockerfile }}
           target: ${{ inputs.push_target }}
           push: true
-          tags: ${{ inputs.push_tags }}
+          tags: ${{ inputs.push_tags }}-arm64
           platforms: |
-            linux/amd64
             linux/arm64
           build-args: |
-            version=${{ inputs.img_tag }}
+            version=${{ inputs.img_tag }}-arm64
             GIT_COMMIT_SHA=${{ github.sha }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
+
+  build-transformer-image-amd64:
+    name: Build Transformer Docker Image AMD64
+    runs-on: [self-hosted, Linux, X64]
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+
+      - name: Setup Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Login to DockerHub
+        uses: docker/[email protected]
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+
+      - name: Build Docker Image
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          target: ${{ inputs.load_target }}
+          load: true
+          tags: ${{ inputs.build_tag }}
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
+
+      - name: Run Tests
+        run: |
+          docker run ${{ inputs.build_tag }} npm run test:js:ci
+          docker run ${{ inputs.build_tag }} npm run test:ts:ci
+
+      - name: Build and Push Multi-platform Images
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          target: ${{ inputs.push_target }}
+          push: true
+          tags: ${{ inputs.push_tags }}-amd64
+          platforms: |
+            linux/amd64
+          build-args: |
+            version=${{ inputs.img_tag }}-amd64
+            GIT_COMMIT_SHA=${{ github.sha }}
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
+
+  create-manifest:
+    name: Create multi-arch manifest
+    runs-on: ubuntu-latest
+    needs: [build-transformer-image-amd64, build-transformer-image-arm64]
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Login to DockerHub
+        uses: docker/[email protected]
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+
+      - name: Create multi-arch manifest
+        run: |
+          docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64

.github/workflows/prepare-for-dev-deploy.yml

@@ -51,7 +51,7 @@ jobs:
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
-      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
+      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name }}
       dockerfile: Dockerfile
       load_target: development
@@ -67,7 +67,7 @@ jobs:
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
-      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
       dockerfile: Dockerfile-ut-func
       load_target: development

.github/workflows/prepare-for-prod-deploy.yml

@@ -39,15 +39,31 @@ jobs:
           echo "Tag Name: $tag_name"
           echo "tag_name=$tag_name" >> $GITHUB_OUTPUT
 
-  build-transformer-image:
-    name: Build Transformer Docker Image - Prod
+  build-rudderstack-transformer-image:
+    name: Build Transformer Docker Image for Rudderstack org- Prod
     # Only merged pull requests from release candidate branches must trigger
     if: ((startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/')) && github.event.pull_request.merged == true)
     needs: [generate-tag-names]
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
-      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderstack/rudder-transformer:latest,rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderlabs/rudder-transformer:latest
+      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderstack/rudder-transformer:latest
+      img_tag: ${{ needs.generate-tag-names.outputs.tag_name }}
+      dockerfile: Dockerfile
+      load_target: development
+      push_target: production
+    secrets:
+      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+
+  build-rudderlabs-transformer-image:
+    name: Build Transformer Docker Image for Rudderlabs org - Prod
+    # Only merged pull requests from release candidate branches must trigger
+    if: ((startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/')) && github.event.pull_request.merged == true)
+    needs: [generate-tag-names]
+    uses: ./.github/workflows/build-push-docker-image.yml
+    with:
+      build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
+      push_tags: rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderlabs/rudder-transformer:latest
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name }}
       dockerfile: Dockerfile
       load_target: development
@@ -58,7 +74,7 @@ jobs:
   create-pull-request:
     name: Update Helm Charts For Production and Create Pull Request
     runs-on: ubuntu-latest
-    needs: [generate-tag-names, build-transformer-image]
+    needs: [generate-tag-names, build-rudderstack-transformer-image, build-rudderlabs-transformer-image]
     env:
       TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }}
       TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer

.github/workflows/prepare-for-prod-ut-deploy.yml

@@ -42,26 +42,43 @@ jobs:
           echo "UT Tag Name: $tag_name_ut"
           echo "tag_name_ut=$tag_name_ut" >> $GITHUB_OUTPUT
 
-  build-user-transformer-image:
-    name: Build User Transformer Docker Image - Prod
+  build-rudderstack-user-transformer-image:
+    name: Build User Transformer Docker Image for Rudderstack org - Prod
     # Only merged pull requests from release candidate branches must trigger
     if: ((startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/')) && github.event.pull_request.merged == true)
     needs: [generate-tag-names]
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
-      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderstack/rudder-transformer:ut-latest,rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderlabs/rudder-transformer:ut-latest
+      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderstack/rudder-transformer:ut-latest
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
       dockerfile: Dockerfile-ut-func
       load_target: development
       push_target: production
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
+  build-rudderlabs-user-transformer-image:
+    name: Build User Transformer Docker Image for Rudderlabs org - Prod
+    # Only merged pull requests from release candidate branches must trigger
+    if: ((startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/')) && github.event.pull_request.merged == true)
+    needs: [generate-tag-names]
+    uses: ./.github/workflows/build-push-docker-image.yml
+    with:
+      build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      push_tags: rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderlabs/rudder-transformer:ut-latest
+      img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      dockerfile: Dockerfile-ut-func
+      load_target: development
+      push_target: production
+    secrets:
+      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+
+
   create-pull-request:
     name: Update Helm Charts For Production and Create Pull Request
     runs-on: ubuntu-latest
-    needs: [generate-tag-names, build-user-transformer-image]
+    needs: [generate-tag-names, build-rudderstack-user-transformer-image, build-rudderlabs-user-transformer-image]
     env:
       UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
       TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer

.github/workflows/prepare-for-staging-deploy.yml

@@ -35,32 +35,65 @@ jobs:
           echo "UT Tag Name: $tag_name_ut"
           echo "tag_name_ut=$tag_name_ut" >> $GITHUB_OUTPUT
 
-  build-transformer-image:
-    name: Build Transformer Docker Image - Staging
+  build-rudderstack-transformer-image:
+    name: Build Transformer Docker Image For Rudderstack org - Staging
     # Only pull requests from release candidate branches must trigger
     if: (startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/'))
     needs: [generate-tag-names]
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
-      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
+      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name }}
       dockerfile: Dockerfile
       load_target: development
       push_target: production
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
-  build-user-transformer-image:
-    name: Build User Transformer Docker Image - Staging
+  build-rudderlabs-transformer-image:
+    name: Build Transformer Docker Image For Rudderlabs org - Staging
+    # Only pull requests from release candidate branches must trigger
+    if: (startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/'))
+    needs: [generate-tag-names]
+    uses: ./.github/workflows/build-push-docker-image.yml
+    with:
+      build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
+      push_tags: rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }}
+      img_tag: ${{ needs.generate-tag-names.outputs.tag_name }}
+      dockerfile: Dockerfile
+      load_target: development
+      push_target: production
+    secrets:
+      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}  
+
+  build-rudderstack-user-transformer-image:
+    name: Build User Transformer Docker Image For Rudderstack org - Staging
+    # Only pull requests from release candidate branches must trigger
+    if: (startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/'))
+
+    needs: [generate-tag-names]
+    uses: ./.github/workflows/build-push-docker-image.yml
+    with:
+      build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      dockerfile: Dockerfile-ut-func
+      load_target: development
+      push_target: production
+    secrets:
+      DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+
+  build-rudderlabs-user-transformer-image:
+    name: Build User Transformer Docker Image For Rudderlabs org - Staging
     # Only pull requests from release candidate branches must trigger
     if: (startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/'))
 
     needs: [generate-tag-names]
     uses: ./.github/workflows/build-push-docker-image.yml
     with:
       build_tag: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
-      push_tags: rudderstack/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      push_tags: rudderlabs/rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }}
       img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
       dockerfile: Dockerfile-ut-func
       load_target: development
@@ -71,7 +104,7 @@ jobs:
   create-pull-request:
     name: Update Helm Charts For Staging and Create Pull Request
     runs-on: ubuntu-latest
-    needs: [generate-tag-names, build-transformer-image, build-user-transformer-image]
+    needs: [generate-tag-names, build-rudderstack-transformer-image, build-rudderlabs-transformer-image, build-rudderstack-user-transformer-image, build-rudderlabs-user-transformer-image]
     env:
       TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }}
       UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }}

CHANGELOG.md

@@ -2,6 +2,36 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.41.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.41.0...v1.41.1) (2023-09-14)
+
+
+### Bug Fixes
+
+* **redis:** add destination_id in the key ([55629d6](https://github.com/rudderlabs/rudder-transformer/commit/55629d6d871d275de601a11ec223e0c283641d7b))
+
+## [1.41.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.40.2...v1.41.0) (2023-09-11)
+
+
+### Features
+
+* add support for updating profile properties through track call ([#2581](https://github.com/rudderlabs/rudder-transformer/issues/2581)) ([f0f20d6](https://github.com/rudderlabs/rudder-transformer/commit/f0f20d654ec5ee8eb078ce7f5610a4666d73fd8c))
+* **INT-580:** messageId to event_id mapping support ([#2570](https://github.com/rudderlabs/rudder-transformer/issues/2570)) ([b38843b](https://github.com/rudderlabs/rudder-transformer/commit/b38843bce9bc02d73dceedc6f751f402251fd23a))
+* **tiktok_ads:** messageId to event_id mapping support ([72f87bf](https://github.com/rudderlabs/rudder-transformer/commit/72f87bfa381ed7a5b74fb5907f932b78d0257ab9))
+
+
+### Bug Fixes
+
+* **bugsnag:** alerts ([266514b](https://github.com/rudderlabs/rudder-transformer/commit/266514bd56c150d6c88c1db0733a1da0a4367c02))
+* **bugsnag:** alerts ([#2580](https://github.com/rudderlabs/rudder-transformer/issues/2580)) ([9e9eeac](https://github.com/rudderlabs/rudder-transformer/commit/9e9eeacdf79cf8175f87f302242542060f668db8))
+* json paths for non tracks events for warehouse ([#2571](https://github.com/rudderlabs/rudder-transformer/issues/2571)) ([e455368](https://github.com/rudderlabs/rudder-transformer/commit/e45536805cf9545b73f4d5bf1be5fad1565ab075))
+
+### [1.40.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.40.1...v1.40.2) (2023-09-06)
+
+
+### Bug Fixes
+
+* marketo bulk upload import issue ([#2559](https://github.com/rudderlabs/rudder-transformer/issues/2559)) ([752f351](https://github.com/rudderlabs/rudder-transformer/commit/752f351f02b7f7611c702d7dbcb4804972bb0970))
+
 ### [1.40.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.40.0...v1.40.1) (2023-09-06)