fix: main into develop conflicts (#2686)

* fix(dm): updated the stats definition for the tracking plan service * fix(iterable): event id mapping (#2659) * chore(release): 1.43.1 * chore: update release owners (#2661) chore: update release stakeholder * chore(release): pull release/v1.44.0 into main (#2672) * fix(mixpanel): batch event ordering (#2608) * fix: bqstream event ordering fix (#2624) * fix: bqstream event ordering fix * fix: review comments addressed * fix: importing the common util function * fix: putting both error and successful events logic back * fix: test cases addition and code clean up * fix: code clean up * fix: resolving sonar error * fix: removing network handler for bqstream * fix: refactor process router dest * fix: use promise for prompts in personalize create tracking script (#2635) * fix(snyk): snyk updated versions merged to oen pr * chore: add ttl values for cache (#2593) * fix(hubspot): add validation for hubspotEvents (#2643) * fix: add optional chaining for error message in network handler of Intercom (#2648) fix bugsnag issue * fix(snyk): removed the new added dependencies * fix(snyk): removed aws-sdk dependency * feat: add circular reference check in flattern json (#2650) * chore: update fb_error message for invalid type of property value (#2656) fix: fb_error message for invalid type of property value * feat(iterable): user deletion support (#2621) * feat(iterable): user deletion support * feat(iterable): user deletion improvements * chore: added comments * feat(gaec): partial failures handling (#2631) * chore: partial failure handling * chore: code review changes * chore: move unit tests to correct place for mssql and postgres (#2657) * chore: move unit tests to correct place for mssql and postgres * chore: replace var to const * chore: replace var to const * feat: add axios mocking to component test-suite (#2638) * feat: tslint rules enable (#2604) * feat: remove eslint issues * fix: fixed es-lint tests * fix: merge conflicts * fix: add auth status inactive category handling to GA in userDeletion flow (#2669) * fix: refactor adjust using optional chaining (#2671) fix: refactor adj using opitonal chaining * chore(release): pull main into develop post release v1.43.1 (#2662) * fix(dm): updated the stats definition for the tracking plan service * fix(iterable): event id mapping (#2659) * chore(release): 1.43.1 * chore: update release owners (#2661) chore: update release stakeholder --------- Co-authored-by: Abhimanyu Babbar <[email protected]> Co-authored-by: Mihir Bhalala <[email protected]> Co-authored-by: GitHub Actions <[email protected]> Co-authored-by: Anant Jain <[email protected]> Co-authored-by: Sudip Paul <[email protected]> * chore(release): 1.44.0 * feat(ga4): added support of filter user traits (#2679) * fix: add optional chaining for traits (#2677) * fix: add optional chaing for traits * update changelog * feat: geolocation support for python transformations (#2678) * feat: geolocation support for python transformations * chore: updating changelog --------- Co-authored-by: Anant Jain <[email protected]> * update changelog * Update CHANGELOG.md --------- Co-authored-by: Gauravudia <[email protected]> Co-authored-by: shrouti1507 <[email protected]> Co-authored-by: Dilip Kola <[email protected]> Co-authored-by: Sudip Paul <[email protected]> Co-authored-by: Sanjay <[email protected]> Co-authored-by: Siva Shanmukh Vetcha <[email protected]> Co-authored-by: Ujjwal Abhishek <[email protected]> Co-authored-by: Mihir Bhalala <[email protected]> Co-authored-by: Sankeerth <[email protected]> Co-authored-by: AASHISH MALIK <[email protected]> Co-authored-by: Anant Jain <[email protected]> Co-authored-by: Abhimanyu Babbar <[email protected]> Co-authored-by: GitHub Actions <[email protected]> Co-authored-by: Jayachand <[email protected]> Co-authored-by: Anant Jain <[email protected]> --------- Co-authored-by: Abhimanyu Babbar <[email protected]> Co-authored-by: Mihir Bhalala <[email protected]> Co-authored-by: GitHub Actions <[email protected]> Co-authored-by: Sudip Paul <[email protected]> Co-authored-by: devops-github-rudderstack <[email protected]> Co-authored-by: Gauravudia <[email protected]> Co-authored-by: shrouti1507 <[email protected]> Co-authored-by: Dilip Kola <[email protected]> Co-authored-by: Sanjay <[email protected]> Co-authored-by: Siva Shanmukh Vetcha <[email protected]> Co-authored-by: Ujjwal Abhishek <[email protected]> Co-authored-by: Sankeerth <[email protected]> Co-authored-by: AASHISH MALIK <[email protected]> Co-authored-by: Jayachand <[email protected]>
rudderlabs · Oct 5, 2023 · e37d693 · e37d693
1 parent d4cac26
commit e37d693
Show file tree

Hide file tree

Showing 11 changed files with 447 additions and 59 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,33 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.44.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.43.1...v1.44.0) (2023-09-29)
+
+
+### Features
+
+* add geolocation support for python transformations ([#2678](https://github.com/rudderlabs/rudder-transformer/issues/2678)) ([b688c4a](https://github.com/rudderlabs/rudder-transformer/commit/b688c4a6889250bacfd2a81c9de8a1436281daa9))
+* add axios mocking to component test-suite ([#2638](https://github.com/rudderlabs/rudder-transformer/issues/2638)) ([100c808](https://github.com/rudderlabs/rudder-transformer/commit/100c80881eb99716198231deb9ec0e69dc3fa2a0))
+* add circular reference check in flattern json ([#2650](https://github.com/rudderlabs/rudder-transformer/issues/2650)) ([a21ea3a](https://github.com/rudderlabs/rudder-transformer/commit/a21ea3aaecfc3d10de3f4eaa34425dda388032ff))
+* **gaec:** partial failures handling ([#2631](https://github.com/rudderlabs/rudder-transformer/issues/2631)) ([0008277](https://github.com/rudderlabs/rudder-transformer/commit/0008277fe43d20ced4066f8f1f1b0f1d12345f01))
+* **iterable:** user deletion support ([#2621](https://github.com/rudderlabs/rudder-transformer/issues/2621)) ([c0ab19a](https://github.com/rudderlabs/rudder-transformer/commit/c0ab19a293a72ea0a7575b863356aea9a0e6762f))
+* tslint rules enable ([#2604](https://github.com/rudderlabs/rudder-transformer/issues/2604)) ([4d6c4b8](https://github.com/rudderlabs/rudder-transformer/commit/4d6c4b834de005484dc1099a52c69ab6212273e8))
+
+
+### Bug Fixes
+* add optional chaining for traits ([#2677](https://github.com/rudderlabs/rudder-transformer/issues/2677)) ([b80bfbc](https://github.com/rudderlabs/rudder-transformer/commit/b80bfbc8a6afdef57ad5f2560bb803c8e00bcbfc))
+* add auth status inactive category handling to GA in userDeletion flow ([#2669](https://github.com/rudderlabs/rudder-transformer/issues/2669)) ([b784800](https://github.com/rudderlabs/rudder-transformer/commit/b7848004e2679ec7278c8381560cb1a77da6c642))
+* add optional chaining for error message in network handler of Intercom ([#2648](https://github.com/rudderlabs/rudder-transformer/issues/2648)) ([10a13bc](https://github.com/rudderlabs/rudder-transformer/commit/10a13bcf7f987de45540b52d414883664e43f9cd))
+* bqstream event ordering fix ([#2624](https://github.com/rudderlabs/rudder-transformer/issues/2624)) ([e97e7ca](https://github.com/rudderlabs/rudder-transformer/commit/e97e7caed20ffb007f1c543e15c15c6e89e2dfb7))
+* **hubspot:** add validation for hubspotEvents ([#2643](https://github.com/rudderlabs/rudder-transformer/issues/2643)) ([b49d0ee](https://github.com/rudderlabs/rudder-transformer/commit/b49d0ee576f55e5f95e3d02b7111e995fc9b8ada))
+* **mixpanel:** batch event ordering ([#2608](https://github.com/rudderlabs/rudder-transformer/issues/2608)) ([36c7f06](https://github.com/rudderlabs/rudder-transformer/commit/36c7f06ccfa534b751ca9b3f5c41e9bb6d2e9f00))
+* refactor adjust using optional chaining ([#2671](https://github.com/rudderlabs/rudder-transformer/issues/2671)) ([f63b493](https://github.com/rudderlabs/rudder-transformer/commit/f63b4932d5ba467a82a36d3e8b80578f81f729cf))
+* **snyk:** fix dependencies version from snyk ([#2640](https://github.com/rudderlabs/rudder-transformer/issues/2640)) ([8422744](https://github.com/rudderlabs/rudder-transformer/commit/8422744b3af48f71358f7acbe2df7b3688ce04a8))
+* **snyk:** removed aws-sdk dependency ([08a3535](https://github.com/rudderlabs/rudder-transformer/commit/08a3535257e5d84566c95cebc701613478f0e039))
+* **snyk:** removed the new added dependencies ([364779c](https://github.com/rudderlabs/rudder-transformer/commit/364779ce959d7548e5cf1cc8c718b71331dcb378))
+* **snyk:** snyk updated versions merged to oen pr ([16a9b8e](https://github.com/rudderlabs/rudder-transformer/commit/16a9b8e6cc26a8df1331cb1af8774df7cc05c959))
+* use promise for prompts in personalize create tracking script ([#2635](https://github.com/rudderlabs/rudder-transformer/issues/2635)) ([5fbddd1](https://github.com/rudderlabs/rudder-transformer/commit/5fbddd1490b16dbda6bca19df4cac5bef6ff78c5))
+
 ### [1.43.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.43.0...v1.43.1) (2023-09-26)
 
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.43.1",
+  "version": "1.44.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js
@@ -20,6 +20,7 @@ const FAAS_MAX_INFLIGHT = process.env.FAAS_MAX_INFLIGHT || '4';
 const FAAS_EXEC_TIMEOUT = process.env.FAAS_EXEC_TIMEOUT || '4s';
 const FAAS_ENABLE_WATCHDOG_ENV_VARS = process.env.FAAS_ENABLE_WATCHDOG_ENV_VARS || 'true';
 const CONFIG_BACKEND_URL = process.env.CONFIG_BACKEND_URL || 'https://api.rudderlabs.com';
+const GEOLOCATION_URL = process.env.GEOLOCATION_URL || '';
 const FAAS_AST_VID = 'ast';
 const FAAS_AST_FN_NAME = 'fn-ast';
 
@@ -128,6 +129,9 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI
       envVars.max_inflight = FAAS_MAX_INFLIGHT;
       envVars.exec_timeout = FAAS_EXEC_TIMEOUT;
     }
+    if (GEOLOCATION_URL) {
+      envVars.geolocation_url = GEOLOCATION_URL;
+    }
     // TODO: investigate and add more required labels and annotations
     const payload = {
       service: functionName,

diff --git a/src/v0/destinations/clevertap/transform.js b/src/v0/destinations/clevertap/transform.js
@@ -232,7 +232,7 @@ const getClevertapProfile = (message, category) => {
   if (message.traits?.overrideFields) {
     const { overrideFields } = message.traits;
     Object.assign(profile, overrideFields);
-  } else if (message.context.traits?.overrideFields) {
+  } else if (message.context?.traits?.overrideFields) {
     const { overrideFields } = message.context.traits;
     Object.assign(profile, overrideFields);
   }

diff --git a/src/v0/destinations/ga4/transform.js b/src/v0/destinations/ga4/transform.js
@@ -106,7 +106,7 @@ const responseBuilder = (message, { Config }) => {
       }
       break;
     default:
-      throw ConfigurationError('Invalid type of client');
+      throw new ConfigurationError('Invalid type of client');
   }
 
   let payload = {};
@@ -119,7 +119,7 @@ const responseBuilder = (message, { Config }) => {
     payload.name = evConfigEvent;
     payload.params = constructPayload(message, mappingConfig[name]);
 
-    const { items, mapRootLevelPropertiesToGA4ItemsArray } = getItemsArray(message, item, itemList)
+    const { items, mapRootLevelPropertiesToGA4ItemsArray } = getItemsArray(message, item, itemList);
 
     if (items.length > 0) {
       payload.params.items = items;
@@ -233,7 +233,7 @@ const responseBuilder = (message, { Config }) => {
   }
 
   // Prepare GA4 user properties
-  const userProperties = prepareUserProperties(message);
+  const userProperties = prepareUserProperties(message, Config.piiPropertiesToIgnore);
   if (!isEmptyObject(userProperties)) {
     rawPayload.user_properties = userProperties;
   }

diff --git a/src/v0/destinations/ga4/utils.js b/src/v0/destinations/ga4/utils.js
@@ -166,13 +166,15 @@ const GA4_ITEM_EXCLUSION = [
 ];
 
 /**
- * Remove arrays and objects from transformed payload 
- * @param {*} params 
- * @returns 
+ * Remove arrays and objects from transformed payload
+ * @param {*} params
+ * @returns
  */
 const removeInvalidParams = (params) =>
   Object.fromEntries(
-    Object.entries(params).filter(([key, value]) => key === 'items' || (typeof value !== 'object' && !isEmpty(value))),
+    Object.entries(params).filter(
+      ([key, value]) => key === 'items' || (typeof value !== 'object' && !isEmpty(value)),
+    ),
   );
 
 /**
@@ -244,10 +246,10 @@ const getItem = (message, isItemsRequired) => {
 
 /**
  * Returns items array for ga4 event payload
- * @param {*} message 
- * @param {*} item 
- * @param {*} itemList 
- * @returns 
+ * @param {*} message
+ * @param {*} item
+ * @param {*} itemList
+ * @returns
  */
 const getItemsArray = (message, item, itemList) => {
   let items = [];
@@ -269,7 +271,7 @@ const getItemsArray = (message, item, itemList) => {
   }
 
   return { items, mapRootLevelPropertiesToGA4ItemsArray };
-}
+};
 /**
  * get exclusion list for a particular event
  * ga4ExclusionList contains the sourceKeys that are already mapped
@@ -398,13 +400,25 @@ const isValidUserProperty = (key, value) => {
 /**
  * Function to validate and prepare user_properties
  * @param {*} message
+ * @param {*} piiPropertiesToIgnore
+ * @returns
  */
-const prepareUserProperties = (message) => {
+const prepareUserProperties = (message, piiPropertiesToIgnore = []) => {
+  // Exclude PII user traits
+  const piiProperties = [];
+  if (piiPropertiesToIgnore.length > 0) {
+    piiPropertiesToIgnore.forEach((property) => {
+      if (typeof property.piiProperty === 'string' && property.piiProperty.trim() !== '') {
+        piiProperties.push(property.piiProperty.trim());
+      }
+    });
+  }
+
   const userProperties = extractCustomFields(
     message,
     {},
     ['properties.user_properties', 'context.traits'],
-    GA4_RESERVED_USER_PROPERTY_EXCLUSION,
+    [...GA4_RESERVED_USER_PROPERTY_EXCLUSION, ...piiProperties],
   );
 
   const validatedUserProperties = Object.entries(userProperties)