feat: onboard smartly destination (#3660)

* feat: onboard smartly destination

* chore: updated processor changes

* chore: router changes added

* chore: mock test added

* chore: batch builder func and test case updated

* chore: update for auth token

* fix: edit after primary dev testing

* fix: test

* fix: editing test caes

* fix: smartly test cases

* chore: updated test cases and function

* chore: ignore eslint of yaml files

* chore: add .eslintignore to .prettierignore

* chore: ignore eslint of yaml files

* chore: update .prettierignore

* chore: update .eslintignore

* chore: updated enpoint for processor

---------

Co-authored-by: shrouti1507 <[email protected]>
Co-authored-by: Dilip Kola <[email protected]>
Co-authored-by: Dilip Kola <[email protected]>

.eslintignore

@@ -21,3 +21,8 @@ src/v0/destinations/personalize/scripts/
 test/integrations/destinations/testTypes.d.ts
 *.config*.js
 scripts/skipPrepareScript.js
+*.yaml
+*.yml
+.eslintignore
+.prettierignore
+*.json

.eslintrc.json

@@ -19,7 +19,8 @@
   "parserOptions": {
     "ecmaVersion": 12,
     "sourceType": "module",
-    "project": "./tsconfig.json"
+    "project": "./tsconfig.json",
+    "extraFileExtensions": [".yaml"]
   },
   "rules": {
     "unicorn/filename-case": [

.prettierignore

@@ -8,3 +8,5 @@ test/**/*.js
 src/util/lodash-es-core.js
 src/util/url-search-params.min.js
 dist
+.eslintignore
+.prettierignore

src/cdk/v2/destinations/smartly/config.js

@@ -0,0 +1,21 @@
+const { getMappingConfig } = require('../../../../v0/util');
+
+const ConfigCategories = {
+  TRACK: {
+    type: 'track',
+    name: 'trackMapping',
+  },
+};
+
+const mappingConfig = getMappingConfig(ConfigCategories, __dirname);
+const singleEventEndpoint = 'https://s2s.smartly.io/events';
+const batchEndpoint = 'https://s2s.smartly.io/events/batch';
+
+module.exports = {
+  ConfigCategories,
+  mappingConfig,
+  singleEventEndpoint,
+  batchEndpoint,
+  TRACK_CONFIG: mappingConfig[ConfigCategories.TRACK.name],
+  MAX_BATCH_SIZE: 1000,
+};

src/cdk/v2/destinations/smartly/data/trackMapping.json

@@ -0,0 +1,76 @@
+[
+  {
+    "destKey": "value",
+    "sourceKeys": [
+      "properties.total",
+      "properties.value",
+      "properties.revenue",
+      {
+        "operation": "multiplication",
+        "args": [
+          {
+            "sourceKeys": "properties.price"
+          },
+          {
+            "sourceKeys": "properties.quantity",
+            "default": 1
+          }
+        ]
+      }
+    ],
+    "metadata": {
+      "type": "toNumber"
+    },
+    "required": false
+  },
+  {
+    "sourceKeys": ["properties.conversions", "properties.products.length"],
+    "required": false,
+    "metadata": {
+      "defaultValue": "1"
+    },
+    "destKey": "conversions"
+  },
+  {
+    "sourceKeys": ["properties.adUnitId", "properties.ad_unit_id"],
+    "required": true,
+    "destKey": "ad_unit_id",
+    "metadata": {
+      "type": "toString"
+    }
+  },
+  {
+    "sourceKeys": ["properties.platform"],
+    "required": true,
+    "destKey": "platform"
+  },
+  {
+    "sourceKeys": ["properties.adInteractionTime", "properties.ad_interaction_time"],
+    "required": true,
+    "metadata": {
+      "type": "secondTimestamp"
+    },
+    "destKey": "ad_interaction_time"
+  },
+  {
+    "sourceKeys": ["properties.installTime"],
+    "required": false,
+    "metadata": {
+      "type": "secondTimestamp"
+    },
+    "destKey": "installTime"
+  },
+  {
+    "sourceKeys": ["originalTimestamp", "timestamp"],
+    "required": false,
+    "metadata": {
+      "type": "secondTimestamp"
+    },
+    "destKey": "event_time"
+  },
+  {
+    "sourceKeys": ["properties.currency"],
+    "required": false,
+    "destKey": "value_currency"
+  }
+]

src/cdk/v2/destinations/smartly/procWorkflow.yaml

@@ -0,0 +1,31 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - name: removeUndefinedAndNullValues
+    path: ../../../../v0/util
+  - name: constructPayload
+    path: ../../../../v0/util
+  - path: ./config
+  - path: ./utils
+steps:
+  - name: messageType
+    template: |
+      .message.type.toLowerCase();
+  - name: validateInput
+    template: |
+      let messageType = $.outputs.messageType;
+      $.assert(messageType, "message Type is not present. Aborting");
+      $.assert(messageType in {{$.EventType.([.TRACK])}}, "message type " + messageType + " is not supported");
+      $.assertConfig(.destination.Config.apiToken, "API Token is not present. Aborting");
+  - name: preparePayload
+    template: |
+      const payload = $.removeUndefinedAndNullValues($.constructPayload(.message, $.TRACK_CONFIG));
+      $.verifyAdInteractionTime(payload.ad_interaction_time);
+      $.context.payloadList = $.getPayloads(.message.event, .destination.Config, payload)
+  - name: buildResponse
+    template: |
+      const response = $.buildResponseList($.context.payloadList)
+      response

src/cdk/v2/destinations/smartly/rtWorkflow.yaml

@@ -0,0 +1,35 @@
+bindings:
+  - path: ./config
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - path: ./utils
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "output": .body.JSON,
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata
+      })[]
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      $.batchResponseBuilder($.outputs.successfulEvents);
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents]

src/cdk/v2/destinations/smartly/utils.js

@@ -0,0 +1,108 @@
+const { BatchUtils } = require('@rudderstack/workflow-engine');
+const { InstrumentationError } = require('@rudderstack/integrations-lib');
+const moment = require('moment');
+const config = require('./config');
+const {
+  getHashFromArrayWithDuplicate,
+  defaultRequestConfig,
+  isDefinedAndNotNull,
+} = require('../../../../v0/util');
+
+// docs reference : https://support.smartly.io/hc/en-us/articles/4406049685788-S2S-integration-API-description#01H8HBXZF6WSKSYBW1C6NY8A88
+
+/**
+ * This function generates an array of payload objects, each with the event property set
+ * to different values associated with the given event name according to eventsMapping
+ * @param {*} event
+ * @param {*} eventsMapping
+ * @param {*} payload
+ * @returns
+ */
+const getPayloads = (event, Config, payload) => {
+  if (!isDefinedAndNotNull(event) || typeof event !== 'string') {
+    throw new InstrumentationError('Event is not defined or is not String');
+  }
+  const eventsMap = getHashFromArrayWithDuplicate(Config.eventsMapping);
+  // eventsMap = hashmap {"prop1":["val1","val2"],"prop2":["val2"]}
+  const eventList = Array.isArray(eventsMap[event.toLowerCase()])
+    ? eventsMap[event.toLowerCase()]
+    : Array.from(eventsMap[event.toLowerCase()] || [event]);
+
+  const payloadLists = eventList.map((ev) => ({ ...payload, event_name: ev }));
+  return payloadLists;
+};
+
+//  ad_interaction_time must be within one year in the future and three years in the past from the current date
+// Example : "1735680000"
+const verifyAdInteractionTime = (adInteractionTime) => {
+  if (isDefinedAndNotNull(adInteractionTime)) {
+    const now = moment();
+    const threeYearAgo = now.clone().subtract(3, 'year');
+    const oneYearFromNow = now.clone().add(1, 'year');
+    const inputMoment = moment(adInteractionTime * 1000); // Convert to milliseconds
+    if (!inputMoment.isAfter(threeYearAgo) || !inputMoment.isBefore(oneYearFromNow)) {
+      throw new InstrumentationError(
+        'ad_interaction_time must be within one year in the future and three years in the past.',
+      );
+    }
+  }
+};
+
+const buildResponseList = (payloadList) =>
+  payloadList.map((payload) => {
+    const response = defaultRequestConfig();
+    response.body.JSON = payload;
+    response.endpoint = config.singleEventEndpoint;
+    response.method = 'POST';
+    return response;
+  });
+
+const batchBuilder = (batch, destination) => ({
+  batchedRequest: {
+    body: {
+      JSON: { events: batch.map((event) => event.output) },
+      JSON_ARRAY: {},
+      XML: {},
+      FORM: {},
+    },
+    version: '1',
+    type: 'REST',
+    method: 'POST',
+    endpoint: config.batchEndpoint,
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${destination.Config.apiToken}`,
+    },
+    params: {},
+    files: {},
+  },
+  metadata: batch
+    .map((event) => event.metadata)
+    .filter((metadata, index, self) => self.findIndex((m) => m.jobId === metadata.jobId) === index), // handling jobId duplication for multiplexed events
+  batched: true,
+  statusCode: 200,
+  destination: batch[0].destination,
+});
+
+/**
+ * This fucntions make chunk of successful events based on MAX_BATCH_SIZE
+ * and then build the response for each chunk to be returned as object of an array
+ * @param {*} events
+ * @returns
+ */
+const batchResponseBuilder = (events) => {
+  if (events.length === 0) {
+    return [];
+  }
+  const { destination } = events[0];
+  const batches = BatchUtils.chunkArrayBySizeAndLength(events, { maxItems: config.MAX_BATCH_SIZE });
+
+  const response = [];
+  batches.items.forEach((batch) => {
+    const batchedResponse = batchBuilder(batch, destination);
+    response.push(batchedResponse);
+  });
+  return response;
+};
+
+module.exports = { batchResponseBuilder, getPayloads, buildResponseList, verifyAdInteractionTime };

src/cdk/v2/destinations/smartly/utils.test.js

@@ -0,0 +1,59 @@
+const moment = require('moment');
+const { verifyAdInteractionTime } = require('./utils');
+
+describe('verifyAdInteractionTime', () => {
+  it('should pass when adInteractionTime is 2 years in the past (UNIX timestamp)', () => {
+    // 2 years ago from now
+    const adInteractionTime = moment().subtract(2, 'years').unix();
+    expect(() => verifyAdInteractionTime(adInteractionTime)).not.toThrow();
+  });
+
+  it('should pass when adInteractionTime is 10 months in the future (UNIX timestamp)', () => {
+    // 10 months in the future from now
+    const adInteractionTime = moment().add(10, 'months').unix();
+    expect(() => verifyAdInteractionTime(adInteractionTime)).not.toThrow();
+  });
+
+  it('should fail when adInteractionTime is 4 years in the past (UNIX timestamp)', () => {
+    // 4 years ago from now
+    const adInteractionTime = moment().subtract(4, 'years').unix();
+    expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow(
+      'ad_interaction_time must be within one year in the future and three years in the past.',
+    );
+  });
+
+  it('should fail when adInteractionTime is 2 years in the future (UNIX timestamp)', () => {
+    // 2 years in the future from now
+    const adInteractionTime = moment().add(2, 'years').unix();
+    expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow(
+      'ad_interaction_time must be within one year in the future and three years in the past.',
+    );
+  });
+
+  it('should pass when adInteractionTime is exactly 1 year in the future (UTC date string)', () => {
+    // Exactly 1 year in the future from now
+    const adInteractionTime = moment.utc().add(1, 'year').toISOString();
+    expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow();
+  });
+
+  it('should fail when adInteractionTime is 4 years in the past (UTC date string)', () => {
+    // 4 years ago from now
+    const adInteractionTime = moment.utc().subtract(4, 'years').toISOString();
+    expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow(
+      'ad_interaction_time must be within one year in the future and three years in the past.',
+    );
+  });
+
+  it('should fail when adInteractionTime is 2 years in the future (UTC date string)', () => {
+    // 2 years in the future from now
+    const adInteractionTime = moment.utc().add(2, 'years').toISOString();
+    expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow(
+      'ad_interaction_time must be within one year in the future and three years in the past.',
+    );
+  });
+
+  it('should not throw an error if adInteractionTime is null or undefined', () => {
+    expect(() => verifyAdInteractionTime(null)).not.toThrow();
+    expect(() => verifyAdInteractionTime(undefined)).not.toThrow();
+  });
+});

src/features.json

@@ -77,7 +77,8 @@
     "CLICKSEND": true,
     "ZOHO": true,
     "CORDIAL": true,
-    "BLOOMREACH_CATALOG": true
+    "BLOOMREACH_CATALOG": true,
+    "SMARTLY": true
   },
   "regulations": [
     "BRAZE",