Merge branch 'develop' into feature/dat-1207-javascript-transformatio…

…n-changes
rudderlabs · Jun 11, 2024 · ce2d62d · ce2d62d
2 parents 8808508 + a39df2b
commit ce2d62d
Show file tree

Hide file tree

Showing 54 changed files with 5,396 additions and 52,956 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -46,7 +46,7 @@ N/A
 
 - [ ] Is the PR limited to one linear task?
 
-- [ ] Are relevant unit and component test-cases added?
+- [ ] Are relevant unit and component test-cases added in **new readability format**?
 
 ### Reviewer checklist
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,48 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.68.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.68.1...v1.68.2) (2024-06-06)
+
+
+### Bug Fixes
+
+* **user-transformation:** pass tf id in common metadata ([d2b0779](https://github.com/rudderlabs/rudder-transformer/commit/d2b0779d3a0145c0088903edfe328c1c7554cbd2))
+
+### [1.68.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.68.0...v1.68.1) (2024-05-29)
+
+
+### Bug Fixes
+
+* tiktok_v2 assigning value to undefined properties ([#3426](https://github.com/rudderlabs/rudder-transformer/issues/3426)) ([323396b](https://github.com/rudderlabs/rudder-transformer/commit/323396b09fd6b7fda3cce53cc4f1cc443d7a78c1))
+
+## [1.68.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.67.0...v1.68.0) (2024-05-27)
+
+
+### Features
+
+* add json-data type support in redis ([#3336](https://github.com/rudderlabs/rudder-transformer/issues/3336)) ([0196f20](https://github.com/rudderlabs/rudder-transformer/commit/0196f20cc79e1f470d96a649dd9404c3c9284329))
+* facebook custom audience app secret support ([#3357](https://github.com/rudderlabs/rudder-transformer/issues/3357)) ([fce4ef9](https://github.com/rudderlabs/rudder-transformer/commit/fce4ef973500411c7ad812e7949bb1b73dabc3ba))
+* filtering unknown events in awin ([#3392](https://github.com/rudderlabs/rudder-transformer/issues/3392)) ([d842da8](https://github.com/rudderlabs/rudder-transformer/commit/d842da87a34cb63023eba288e0c5258e29997dcf))
+* **ga4:** component test refactor ([#3220](https://github.com/rudderlabs/rudder-transformer/issues/3220)) ([3ff9a5e](https://github.com/rudderlabs/rudder-transformer/commit/3ff9a5e8e955b929a1b04a89dcf0ccbc49e18648))
+* **integrations/auth0:** include Auth0 event type in Rudderstack message ([#3370](https://github.com/rudderlabs/rudder-transformer/issues/3370)) ([e9409fd](https://github.com/rudderlabs/rudder-transformer/commit/e9409fde6063d7eaa8558396b85b5fdf99f964e1))
+* onboard koddi destination ([#3359](https://github.com/rudderlabs/rudder-transformer/issues/3359)) ([f74c4a0](https://github.com/rudderlabs/rudder-transformer/commit/f74c4a0bc92ae6ccb0c00ac5b21745e496a015bc))
+* onboarding adjust source ([#3395](https://github.com/rudderlabs/rudder-transformer/issues/3395)) ([668d331](https://github.com/rudderlabs/rudder-transformer/commit/668d3311aadacbb92b1873bf43919db7d341afbb))
+
+
+### Bug Fixes
+
+* fb custom audience html response ([#3402](https://github.com/rudderlabs/rudder-transformer/issues/3402)) ([d1a2bd6](https://github.com/rudder
+* standardise hashing for all CAPI integrations ([#3379](https://github.com/rudderlabs/rudder-transformer/issues/3379)) ([c249a69](https://github.com/rudderlabs/rudder-transformer/commit/c249a694d735f6d241a35b6e21f493c54890ac84))
+* tiktok_v2 remove default value for content-type for custom events ([#3383](https://github.com/rudderlabs/rudder-transformer/issues/3383)) ([6e7b5a0](https://github.com/rudderlabs/rudder-transformer/commit/6e7b5a0d8bf2c859dfb15b9cad7ed6070bd0892b))
+* added step for reconciling openfaas functions for python transformations ([#3420](https://github.com/rudderlabs/rudder-transformer/issues/3420)) ([7a2ab63](https://github.com/rudderlabs/rudder-transformer/commit/7a2ab63674d40870af4d16f0673a2a2594c899e9))
+
+## [1.67.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.66.1...v1.67.0) (2024-05-23)
+
+
+### Features
+
+* sre 456 ut move high cardinality histogram metrics to summaries cp ([#3409](https://github.com/rudderlabs/rudder-transformer/issues/3409)) ([be20dc2](https://github.com/rudderlabs/rudder-transformer/commit/be20dc26ade2fa0212dc91126cf42087a84a07c9))
+
 ### [1.66.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.66.0...v1.66.1) (2024-05-20)
 
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -49,5 +49,5 @@ We look forward to your feedback on improving this project.
 
 [slack]: https://resources.rudderstack.com/join-rudderstack-slack
 [issue]: https://github.com/rudderlabs/rudder-transformer/issues/new
-[cla]: https://rudderlabs.wufoo.com/forms/rudderlabs-contributor-license-agreement
+[cla]: https://forms.gle/845JRGVZaC6kPZy68
 [config-generator]: https://github.com/rudderlabs/config-generator
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.66.1",
+  "version": "1.68.2",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -65,7 +65,7 @@
     "@ndhoule/extend": "^2.0.0",
     "@pyroscope/nodejs": "^0.2.9",
     "@rudderstack/integrations-lib": "^0.2.8",
-    "@rudderstack/workflow-engine": "^0.7.5",
+    "@rudderstack/workflow-engine": "^0.7.9",
     "@shopify/jest-koa-mocks": "^5.1.1",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",

diff --git a/src/cdk/v2/destinations/emarsys/utils.js b/src/cdk/v2/destinations/emarsys/utils.js
@@ -162,7 +162,7 @@ const deduceEventId = (message, destConfig) => {
   const { eventsMapping } = destConfig;
   const { event } = message;
   validateEventName(event);
-  if (eventsMapping.length > 0) {
+  if (Array.isArray(eventsMapping) && eventsMapping.length > 0) {
     const keyMap = getHashFromArray(eventsMapping, 'from', 'to', false);
     eventId = keyMap[event];
   }

diff --git a/src/legacy/router.js b/src/legacy/router.js
@@ -649,6 +649,9 @@ if (startDestTransformer) {
       stats.timing('user_transform_request_latency', startTime, {
         processSessions,
       });
+      stats.timingSummary('user_transform_request_latency_summary', startTime, {
+        processSessions,
+      });
       stats.increment('user_transform_requests', { processSessions });
       stats.histogram('user_transform_output_events', transformedEvents.length, {
         processSessions,

diff --git a/src/services/userTransform.ts b/src/services/userTransform.ts
@@ -67,6 +67,7 @@ export class UserTransformService {
           destinationId: eventsToProcess[0]?.metadata.destinationId,
           destinationType: eventsToProcess[0]?.metadata.destinationType,
           workspaceId: eventsToProcess[0]?.metadata.workspaceId,
+          transformationId: eventsToProcess[0]?.metadata.transformationId,
           messageIds,
         };
 
@@ -173,7 +174,17 @@ export class UserTransformService {
             ...getTransformationMetadata(eventsToProcess[0]?.metadata),
           });
 
-          stats.histogram('user_transform_batch_size', requestSize, {
+          stats.timing('user_transform_batch_size', requestSize, {
+            ...metaTags,
+            ...getTransformationMetadata(eventsToProcess[0]?.metadata),
+          });
+
+          stats.timingSummary('user_transform_request_latency_summary', userFuncStartTime, {
+            ...metaTags,
+            ...getTransformationMetadata(eventsToProcess[0]?.metadata),
+          });
+
+          stats.timingSummary('user_transform_batch_size_summary', requestSize, {
             ...metaTags,
             ...getTransformationMetadata(eventsToProcess[0]?.metadata),
           });

diff --git a/src/types/index.ts b/src/types/index.ts
@@ -131,6 +131,8 @@ type Destination = {
   WorkspaceID: string;
   Transformations: UserTransformationInput[];
   RevisionID?: string;
+  IsProcessorEnabled?: boolean;
+  IsConnectionEnabled?: boolean;
 };
 
 type UserTransformationLibrary = {

diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js
@@ -11,20 +11,26 @@ const {
 } = require('./openfaas');
 const { getLibraryCodeV1 } = require('./customTransforrmationsStore-v1');
 
+const HASH_SECRET = process.env.OPENFAAS_FN_HASH_SECRET || '';
 const libVersionIdsCache = new NodeCache();
 
-function generateFunctionName(userTransformation, libraryVersionIds, testMode) {
+function generateFunctionName(userTransformation, libraryVersionIds, testMode, hashSecret = '') {
   if (userTransformation.versionId === FAAS_AST_VID) return FAAS_AST_FN_NAME;
 
   if (testMode) {
     const funcName = `fn-test-${uuidv4()}`;
     return funcName.substring(0, 63).toLowerCase();
   }
 
-  const ids = [userTransformation.workspaceId, userTransformation.versionId].concat(
+  let ids = [userTransformation.workspaceId, userTransformation.versionId].concat(
     (libraryVersionIds || []).sort(),
   );
 
+  if (hashSecret !== '') {
+    ids = ids.concat([hashSecret]);
+  }
+
+  // FIXME: Why the id's are sorted ?!
   const hash = crypto.createHash('md5').update(`${ids}`).digest('hex');
   return `fn-${userTransformation.workspaceId}-${hash}`.substring(0, 63).toLowerCase();
 }
@@ -90,7 +96,13 @@ async function setOpenFaasUserTransform(
     testMode,
   };
   const functionName =
-    pregeneratedFnName || generateFunctionName(userTransformation, libraryVersionIds, testMode);
+    pregeneratedFnName ||
+    generateFunctionName(
+      userTransformation,
+      libraryVersionIds,
+      testMode,
+      process.env.OPENFAAS_FN_HASH_SECRET,
+    );
   const setupTime = new Date();
 
   await setupFaasFunction(
@@ -130,7 +142,13 @@ async function runOpenFaasUserTransform(
 
   const trMetadata = events[0].metadata ? getTransformationMetadata(events[0].metadata) : {};
   // check and deploy faas function if not exists
-  const functionName = generateFunctionName(userTransformation, libraryVersionIds, testMode);
+  const functionName = generateFunctionName(
+    userTransformation,
+    libraryVersionIds,
+    testMode,
+    process.env.OPENFAAS_FN_HASH_SECRET,
+  );
+
   if (testMode) {
     await setOpenFaasUserTransform(
       userTransformation,

diff --git a/src/util/customTransformer-v1.js b/src/util/customTransformer-v1.js
@@ -95,6 +95,7 @@ async function userTransformHandlerV1(
     };
     stats.counter('user_transform_function_input_events', events.length, tags);
     stats.timing('user_transform_function_latency', invokeTime, tags);
+    stats.timingSummary('user_transform_function_latency_summary', invokeTime, tags);
   }
 
   return { transformedEvents, logs };

diff --git a/src/util/customTransformer.js b/src/util/customTransformer.js
@@ -261,6 +261,7 @@ async function runUserTransform(
 
     stats.counter('user_transform_function_input_events', events.length, tags);
     stats.timing('user_transform_function_latency', invokeTime, tags);
+    stats.timingSummary('user_transform_function_latency_summary', invokeTime, tags);
   }
 
   return {

diff --git a/src/util/customTransforrmationsStore-v1.js b/src/util/customTransforrmationsStore-v1.js
@@ -31,6 +31,7 @@ async function getTransformationCodeV1(versionId) {
     responseStatusHandler(response.status, 'Transformation', versionId, url);
     stats.increment('get_transformation_code', { success: 'true', ...tags });
     stats.timing('get_transformation_code_time', startTime, tags);
+    stats.timingSummary('get_transformation_code_time_summary', startTime, tags);
     const myJson = await response.json();
     transformationCache[versionId] = myJson;
     return myJson;
@@ -56,6 +57,7 @@ async function getLibraryCodeV1(versionId) {
     responseStatusHandler(response.status, 'Transformation Library', versionId, url);
     stats.increment('get_libraries_code', { success: 'true', ...tags });
     stats.timing('get_libraries_code_time', startTime, tags);
+    stats.timingSummary('get_libraries_code_time_summary', startTime, tags);
     const myJson = await response.json();
     libraryCache[versionId] = myJson;
     return myJson;
@@ -83,6 +85,7 @@ async function getRudderLibByImportName(importName) {
     responseStatusHandler(response.status, 'Rudder Library', importName, url);
     stats.increment('get_libraries_code', { success: 'true', ...tags });
     stats.timing('get_libraries_code_time', startTime, tags);
+    stats.timingSummary('get_libraries_code_time_summary', startTime, tags);
     const myJson = await response.json();
     rudderLibraryCache[importName] = myJson;
     return myJson;

diff --git a/src/util/customTransforrmationsStore.js b/src/util/customTransforrmationsStore.js
@@ -24,6 +24,7 @@ async function getTransformationCode(versionId) {
     responseStatusHandler(response.status, 'Transformation', versionId, url);
     stats.increment('get_transformation_code', { versionId, success: 'true' });
     stats.timing('get_transformation_code_time', startTime, { versionId });
+    stats.timingSummary('get_transformation_code_time_summary', startTime, { versionId });
     const myJson = await response.json();
     myCache.set(versionId, myJson);
     return myJson;