Merge branch 'develop' into feat.vdm-next-poc

rudderlabs · Aug 29, 2024 · 9a9340f · 9a9340f
2 parents 42fbce5 + 9fb463e
commit 9a9340f
Show file tree

Hide file tree

Showing 18 changed files with 207 additions and 55 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -21,3 +21,5 @@ src/v0/destinations/personalize/scripts/
 test/integrations/destinations/testTypes.d.ts
 *.config*.js
 scripts/skipPrepareScript.js
+package-lock.json
+package.json
diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml
@@ -107,7 +107,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -124,7 +124,7 @@ jobs:
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -160,7 +160,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -177,7 +177,7 @@ jobs:
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}

diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml
@@ -80,7 +80,7 @@ jobs:
 
       - name: SonarCloud Scan
         if: always()
-        uses: SonarSource/sonarcloud-github-action@v2.1.1
+        uses: SonarSource/sonarcloud-github-action@v3.0.0
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

diff --git a/.github/workflows/ut-tests.yml b/.github/workflows/ut-tests.yml
@@ -66,6 +66,18 @@ jobs:
             --set gateway.image=rudderlabs/rudder-openfaas-gateway:0.25.2 \
             --set faasnetes.image=rudderlabs/rudder-openfaas-faas-netes:0.15.4
 
+      - name: Create regcred secret in openfaas
+        run: kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=${{ secrets.DOCKERHUB_USERNAME }} --docker-password=${{ secrets.DOCKERHUB_TOKEN }} --docker-email=${{ secrets.DOCKERHUB_EMAIL }} -n openfaas
+
+      - name: Create regcred secret in openfaas-fn
+        run: kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=${{ secrets.DOCKERHUB_USERNAME }} --docker-password=${{ secrets.DOCKERHUB_TOKEN }} --docker-email=${{ secrets.DOCKERHUB_EMAIL }} -n openfaas-fn
+
+      - name: Patch default service account in openfaas-fn
+        run: 'kubectl patch serviceaccount default -n openfaas-fn -p ''{"imagePullSecrets": [{"name": "regcred"}]}'''
+
+      - name: Patch deployment "gateway"
+        run: 'kubectl patch deployment gateway -n openfaas -p ''{"spec": {"template": {"spec": {"imagePullSecrets": [{"name": "regcred"}]}}}}'''
+
       - name: Wait for deployment "gateway" rollout
         run: kubectl rollout status deploy/gateway --timeout 120s -n openfaas
 

diff --git a/.prettierignore b/.prettierignore
@@ -8,3 +8,4 @@ test/**/*.js
 src/util/lodash-es-core.js
 src/util/url-search-params.min.js
 dist
+.eslintignore
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -71,7 +71,7 @@
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",
-    "axios": "^1.6.4",
+    "axios": "^1.7.3",
     "btoa": "^1.2.1",
     "component-each": "^0.2.6",
     "crypto-js": "^4.2.0",

diff --git a/src/cdk/v2/destinations/the_trade_desk/transformRecord.js b/src/cdk/v2/destinations/the_trade_desk/transformRecord.js
@@ -40,6 +40,9 @@ const batchResponseBuilder = (items, config) => {
   return response;
 };
 
+const checkNullUndefinedEmptyFields = (fields) =>
+  !!Object.entries(fields).some(([, value]) => !value);
+
 const processRecordInputs = (inputs, destination) => {
   const { Config } = destination;
   const items = [];
@@ -68,6 +71,11 @@ const processRecordInputs = (inputs, destination) => {
       return;
     }
 
+    if (checkNullUndefinedEmptyFields(fields)) {
+      errorResponseList.push(handleRtTfSingleEventError(input, emptyFieldsError, {}));
+      return;
+    }
+
     successMetadata.push(input.metadata);
     const data = [
       {
@@ -78,10 +86,7 @@ const processRecordInputs = (inputs, destination) => {
 
     Object.keys(fields).forEach((id) => {
       const value = fields[id];
-      if (value) {
-        // adding only non empty ID's
-        items.push({ [id]: value, Data: data });
-      }
+      items.push({ [id]: value, Data: data });
     });
   });
 

diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js
@@ -241,8 +241,9 @@ const deployFaasFunction = async (
     // To handle concurrent create requests,
     // throw retry error if deployment or service already exists so that request can be retried
     if (
-      (error.statusCode === 500 || error.statusCode === 400) &&
-      error.message.includes('already exists')
+      ((error.statusCode === 500 || error.statusCode === 400) &&
+        error.message.includes('already exists')) ||
+      (error.statusCode === 409 && error.message.includes('Conflict change already made'))
     ) {
       setFunctionInCache(functionName);
       throw new RetryRequestError(`${functionName} already exists`);

diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/config.js b/src/v0/destinations/google_adwords_enhanced_conversions/config.js
@@ -1,6 +1,8 @@
 const { getMappingConfig } = require('../../util');
 
-const BASE_ENDPOINT = 'https://googleads.googleapis.com/v15/customers';
+const API_VERSION = 'v17';
+
+const BASE_ENDPOINT = 'https://googleads.googleapis.com/v17/customers';
 
 const CONFIG_CATEGORIES = {
   TRACK_CONFIG: { type: 'track', name: 'trackConfig' },
@@ -17,4 +19,5 @@ module.exports = {
   hashAttributes,
   CONVERSION_ACTION_ID_CACHE_TTL,
   destType: 'google_adwords_enhanced_conversions',
+  API_VERSION,
 };
diff --git a/src/v0/sources/customerio/config.js b/src/v0/sources/customerio/config.js
@@ -8,6 +8,7 @@ const emailEventNameMap = {
   spammed: 'Email Marked as Spam',
   dropped: 'Email Dropped',
   bounced: 'Email Bounced',
+  subscribed: 'Email Subscribed',
   unsubscribed: 'Email Unsubscribed',
   converted: 'Email Converted',
   clicked: 'Email Link Clicked',

diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/dataDelivery/business.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/dataDelivery/business.ts
@@ -4,6 +4,7 @@ import {
   generateProxyV1Payload,
 } from '../../../testUtils';
 import { ProxyV1TestData } from '../../../testTypes';
+import { API_VERSION } from '../../../../../src/v0/destinations/google_adwords_enhanced_conversions/config';
 
 const headers = {
   Authorization: 'Bearer abcd1234',
@@ -82,8 +83,7 @@ export const testScenariosForV0API = [
       request: {
         body: generateProxyV0Payload({
           ...commonRequestParameters,
-          endpoint:
-            'https://googleads.googleapis.com/v15/customers/1234567899:uploadConversionAdjustments',
+          endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567899:uploadConversionAdjustments`,
         }),
         method: 'POST',
       },
@@ -138,8 +138,7 @@ export const testScenariosForV0API = [
             customerId: '1234567888',
             destination: 'google_adwords_enhanced_conversions',
           },
-          endpoint:
-            'https://googleads.googleapis.com/v15/customers/1234567888:uploadConversionAdjustments',
+          endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567888:uploadConversionAdjustments`,
         }),
         method: 'POST',
       },
@@ -203,8 +202,7 @@ export const testScenariosForV1API: ProxyV1TestData[] = [
         body: generateProxyV1Payload(
           {
             ...commonRequestParameters,
-            endpoint:
-              'https://googleads.googleapis.com/v15/customers/1234567899:uploadConversionAdjustments',
+            endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567899:uploadConversionAdjustments`,
           },
           [generateMetadata(1)],
         ),
@@ -251,8 +249,7 @@ export const testScenariosForV1API: ProxyV1TestData[] = [
               customerId: '1234567888',
               destination: 'google_adwords_enhanced_conversions',
             },
-            endpoint:
-              'https://googleads.googleapis.com/v15/customers/1234567888:uploadConversionAdjustments',
+            endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567888:uploadConversionAdjustments`,
           },
           [generateMetadata(1)],
         ),

diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/dataDelivery/oauth.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/dataDelivery/oauth.ts
@@ -1,4 +1,4 @@
-import { ProxyV1TestData } from '../../../testTypes';
+import { API_VERSION } from '../../../../../src/v0/destinations/google_adwords_enhanced_conversions/config';
 import {
   generateProxyV1Payload,
   generateProxyV0Payload,
@@ -83,8 +83,7 @@ export const v0oauthScenarios = [
       request: {
         body: generateProxyV0Payload({
           ...commonRequestParameters,
-          endpoint:
-            'https://googleads.googleapis.com/v15/customers/1234567890:uploadConversionAdjustments',
+          endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890:uploadConversionAdjustments`,
         }),
         method: 'POST',
       },
@@ -137,8 +136,7 @@ export const v0oauthScenarios = [
             customerId: '1234567910',
             destination: 'google_adwords_enhanced_conversions',
           },
-          endpoint:
-            'https://googleads.googleapis.com/v15/customers/1234567910/googleAds:searchStream',
+          endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567910/googleAds:searchStream`,
         }),
         method: 'POST',
       },
@@ -192,8 +190,7 @@ export const v1oauthScenarios = [
       request: {
         body: generateProxyV1Payload({
           ...commonRequestParameters,
-          endpoint:
-            'https://googleads.googleapis.com/v15/customers/1234567890:uploadConversionAdjustments',
+          endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890:uploadConversionAdjustments`,
         }),
         method: 'POST',
       },
@@ -244,8 +241,7 @@ export const v1oauthScenarios = [
             customerId: '1234567910',
             destination: 'google_adwords_enhanced_conversions',
           },
-          endpoint:
-            'https://googleads.googleapis.com/v15/customers/1234567910/googleAds:searchStream',
+          endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567910/googleAds:searchStream`,
         }),
         method: 'POST',
       },

diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/network.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/network.ts
@@ -1,7 +1,8 @@
+import { API_VERSION } from '../../../../src/v0/destinations/google_adwords_enhanced_conversions/config';
 export const networkCallsData = [
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567890/googleAds:searchStream',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890/googleAds:searchStream`,
       data: {
         query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Product Added'`,
       },
@@ -29,7 +30,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567899/googleAds:searchStream',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567899/googleAds:searchStream`,
       data: {
         query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Product Added'`,
       },
@@ -59,7 +60,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567899:uploadConversionAdjustments',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567899:uploadConversionAdjustments`,
       data: {
         conversionAdjustments: [
           {
@@ -123,7 +124,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567891/googleAds:searchStream',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567891/googleAds:searchStream`,
       data: {
         query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Product Added'`,
       },
@@ -153,7 +154,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567891:uploadConversionAdjustments',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567891:uploadConversionAdjustments`,
       data: {
         conversionAdjustments: [
           {
@@ -214,7 +215,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567891:uploadClickConversions',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567891:uploadClickConversions`,
       data: {
         conversionAdjustments: [
           {
@@ -275,7 +276,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567888/googleAds:searchStream',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567888/googleAds:searchStream`,
       data: {
         query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Product Added'`,
       },
@@ -305,7 +306,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567888:uploadConversionAdjustments',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567888:uploadConversionAdjustments`,
       data: {
         conversionAdjustments: [
           {
@@ -382,7 +383,7 @@ export const networkCallsData = [
   },
   {
     httpReq: {
-      url: 'https://googleads.googleapis.com/v15/customers/1234567910/googleAds:searchStream',
+      url: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567910/googleAds:searchStream`,
       data: {
         query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Product Added'`,
       },