chore: onboard script to generate testdata and test integration (#3112)

.gitignore

@@ -137,4 +137,5 @@ dist
 .idea
 
 # component test report
-test_reports/
+test_reports/
+temp/

test/integrations/destinations/salesforce/network.ts

@@ -466,5 +466,5 @@ export const networkCallsData = [
   ...tfProxyMocksData,
   ...transformationMocksData,
   ...businessMockData,
-  ...otherMocksData
+  ...otherMocksData,
 ];

test/integrations/testTypes.ts

@@ -37,6 +37,7 @@ export interface mockType {
 }
 
 export interface TestCaseData {
+  id?: string;
   name: string;
   description: string;
   scenario?: string;

test/integrations/testUtils.ts

@@ -1,29 +1,22 @@
-import { z } from 'zod';
 import { globSync } from 'glob';
 import { join } from 'path';
 import { MockHttpCallsData, TestCaseData } from './testTypes';
 import MockAdapter from 'axios-mock-adapter';
 import isMatch from 'lodash/isMatch';
 import { OptionValues } from 'commander';
 import { removeUndefinedAndNullValues } from '@rudderstack/integrations-lib';
-import {
-  Destination,
-  Metadata,
-  ProxyMetdata,
-  ProxyV0Request,
-  ProxyV1Request,
-} from '../../src/types';
+import tags from '../../src/v0/util/tags';
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
+import { Destination, ProxyMetdata, ProxyV0Request, ProxyV1Request } from '../../src/types';
 import {
   DeliveryV0ResponseSchema,
   DeliveryV0ResponseSchemaForOauth,
   DeliveryV1ResponseSchema,
   DeliveryV1ResponseSchemaForOauth,
   ProcessorTransformationResponseListSchema,
-  ProcessorTransformationResponseSchema,
   ProxyV0RequestSchema,
   ProxyV1RequestSchema,
   RouterTransformationResponseListSchema,
-  RouterTransformationResponseSchema,
 } from '../../src/types/zodTypes';
 
 const generateAlphanumericId = (size = 36) =>
@@ -104,6 +97,49 @@ export const overrideDestination = (destination: Destination, overrideConfigValu
   });
 };
 
+export const produceTestData = (testData: TestCaseData[], filterKeys = []) => {
+  const result: any = [];
+  testData.forEach((tcData) => {
+    let events;
+    try {
+      switch (tcData.feature) {
+        case tags.FEATURES.PROCESSOR:
+          events = tcData.input.request.body;
+          break;
+        case tags.FEATURES.BATCH:
+          events = tcData.input.request.body.input;
+          break;
+        case tags.FEATURES.ROUTER:
+          events = tcData.input.request.body.input;
+          break;
+      }
+    } catch (e) {
+      throw new Error(
+        `Error in producing test data for destination:${tcData.name}, id:${tcData.id}: ${e}`,
+      );
+    }
+
+    events.forEach((event) => {
+      const { message } = event;
+      // remove unwanted keys
+      filterKeys.forEach((key) => {
+        delete message[key];
+      });
+      result.push(message);
+    });
+  });
+
+  // write the data to a file
+
+  // create directory if not exists
+  const dir = join(__dirname, '../../temp');
+  if (!existsSync(dir)) {
+    mkdirSync(dir);
+  }
+  writeFileSync(join(__dirname, '../../temp/test_data.json'), JSON.stringify(result, null, 2));
+  console.log('Data generated successfully at temp/test_data.json');
+};
+
 export const generateIndentifyPayload: any = (parametersOverride: any) => {
   const payload = {
     type: 'identify',

test/scripts/testDataGenerator.ts

@@ -0,0 +1,88 @@
+import path from 'path';
+import { TestCaseData } from '../integrations/testTypes';
+import { getTestData, getTestDataFilePaths, produceTestData } from '../integrations/testUtils';
+import { Command } from 'commander';
+import axios from 'axios';
+import * as fs from 'fs';
+
+// Produces test data for a given destination
+// Example usage
+// npx ts-node test/scripts/testDataGenerator.ts --destination=klaviyo --feature=processor
+
+const command = new Command();
+command
+  .allowUnknownOption()
+  .option('-d, --destination <string>', 'Enter Destination Name')
+  .option('-f, --feature <string>', 'Enter Feature Name(processor, router)')
+  .option('-i, --index <number>', 'Enter Test index')
+  .option('-id, --id <string>', 'Enter unique "Id" of the test case you want to run')
+  .option('-dp, --dataPlane <string>', 'Enter Data Plane URL')
+  .option('-wk, --writeKey <string>', 'Enter Write Key')
+  .option(
+    '-fk, --filterKeys <string>',
+    'Enter Keys to filter from the test data(originalTimestamp, timestamp, messageId etc)',
+  )
+  .parse();
+
+const opts = command.opts();
+
+if (opts.destination === undefined) {
+  throw new Error('Destination is not provided');
+}
+
+const filterKeys = opts.filterKeys ? opts.filterKeys.split(',') : [];
+
+const rootDir = __dirname;
+const resolvedpath = path.resolve(rootDir, '../integrations');
+const destinationTestDataPaths = getTestDataFilePaths(resolvedpath, opts);
+
+destinationTestDataPaths.forEach((testDataPath) => {
+  let testData: TestCaseData[] = getTestData(testDataPath);
+  if (opts.index !== undefined) {
+    testData = [testData[parseInt(opts.index)]];
+  }
+  if (opts.id) {
+    testData = testData.filter((data) => {
+      if (data['id'] === opts.id) {
+        return true;
+      }
+      return false;
+    });
+  }
+  console.log('Writing test data to ../../temp/test_data.json');
+  produceTestData(testData, filterKeys);
+
+  if (opts.dataPlane && opts.writeKey) {
+    // read file ../../temp/test_data.json
+    console.log('Sending data to data plane URL: ', opts.dataPlane);
+
+    const resolvedpathForData = path.resolve(rootDir, '../../temp/test_data.json');
+
+    fs.readFile(resolvedpathForData, 'utf8', function (err, data) {
+      if (err) {
+        console.log(err);
+      } else {
+        const parsedData = JSON.parse(data);
+        axios
+          .post(
+            `${opts.dataPlane}/v1/batch`,
+            {
+              batch: parsedData,
+            },
+            {
+              headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Basic ${Buffer.from(opts.writeKey + ':').toString('base64')}`,
+              },
+            },
+          )
+          .then((response) => {
+            console.log(response);
+          })
+          .catch((error) => {
+            console.log(error);
+          });
+      }
+    });
+  }
+});